Add virtual file system (VFS) replacing the low level parts of the old resource system

apps/nifosgtest/test.cpp

@@ -6,6 +6,9 @@
 
 #include <components/nifosg/nifloader.hpp>
 
+#include <components/vfs/manager.hpp>
+#include <components/vfs/bsaarchive.hpp>
+
 #include <osgGA/TrackballManipulator>
 
 #include <osgDB/Registry>
@@ -57,10 +60,11 @@ int main(int argc, char** argv)
         return 1;
     }
 
-    Bsa::BSAFile bsa;
+    VFS::Manager resourceMgr (false);
-    bsa.open(argv[1]);
+    resourceMgr.addArchive(new VFS::BsaArchive(argv[1]));
+    resourceMgr.buildIndex();
 
-    Nif::NIFFilePtr nif(new Nif::NIFFile(bsa.getFile(argv[2]), std::string(argv[2])));
+    Nif::NIFFilePtr nif(new Nif::NIFFile(resourceMgr.get(argv[2]), std::string(argv[2])));
 
     osgViewer::Viewer viewer;
 
@@ -75,7 +79,7 @@ int main(int argc, char** argv)
     std::vector<NifOsg::Controller >  controllers;
     osg::Group* newNode = new osg::Group;
     NifOsg::Loader loader;
-    loader.resourceManager = &bsa;
+    loader.resourceManager = &resourceMgr;
     loader.loadAsSkeleton(nif, newNode);
 
     for (unsigned int i=0; i<loader.mControllers.size(); ++i)

components/CMakeLists.txt

@@ -34,6 +34,10 @@ add_component_dir (bsa
     bsa_file
     )
 
+add_component_dir (vfs
+    manager archive bsaarchive filesystemarchive
+    )
+
 add_component_dir (nif
     controlled effect niftypes record controller extra node record_ptr data niffile property nifkey data node base nifstream
     )

components/bsa/bsa_file.cpp

@@ -179,3 +179,8 @@ Files::IStreamPtr BSAFile::getFile(const char *file)
 
     return Files::openConstrainedFileStream (filename.c_str (), fs.offset, fs.fileSize);
 }
+
+Files::IStreamPtr BSAFile::getFile(const FileStruct *file)
+{
+    return Files::openConstrainedFileStream (filename.c_str (), file->offset, file->fileSize);
+}

components/bsa/bsa_file.hpp

@@ -120,6 +120,8 @@ public:
     */
     Files::IStreamPtr getFile(const char *file);
 
+    Files::IStreamPtr getFile(const FileStruct* file);
+
     /// Get a list of all files
     const FileList &getList() const
     { return files; }

components/nifosg/nifloader.cpp

@@ -465,7 +465,7 @@ namespace NifOsg
                     osgDB::Options* opts = new osgDB::Options;
                     opts->setOptionString("dds_dxt1_detect_rgba");
                     osgDB::ReaderWriter* reader = osgDB::Registry::instance()->getReaderWriterForExtension("dds");
-                    osgDB::ReaderWriter::ReadResult result = reader->readImage(*resourceManager->getFile(filename.c_str()), opts);
+                    osgDB::ReaderWriter::ReadResult result = reader->readImage(*resourceManager->get(filename.c_str()), opts);
                     textures.push_back(osg::ref_ptr<osg::Image>(result.getImage()));
                 }
                 boost::shared_ptr<ControllerValue> dest(new FlipControllerValue(stateset, flipctrl, textures));
@@ -843,7 +843,7 @@ namespace NifOsg
                     osgDB::Options* opts = new osgDB::Options;
                     opts->setOptionString("dds_dxt1_detect_rgba");
                     osgDB::ReaderWriter* reader = osgDB::Registry::instance()->getReaderWriterForExtension("dds");
-                    osgDB::ReaderWriter::ReadResult result = reader->readImage(*resourceManager->getFile(filename.c_str()), opts);
+                    osgDB::ReaderWriter::ReadResult result = reader->readImage(*resourceManager->get(filename.c_str()), opts);
                     osg::Image* image = result.getImage();
                     osg::Texture2D* texture2d = new osg::Texture2D;
                     texture2d->setImage(image);

components/nifosg/nifloader.hpp

@@ -5,6 +5,8 @@
 
 #include <components/nifcache/nifcache.hpp> // NIFFilePtr
 
+#include <components/vfs/manager.hpp>
+
 #include <osg/Group>
 
 #include "controller.hpp"
@@ -24,10 +26,6 @@ namespace Nif
     class NiTriShape;
     class Property;
 }
-namespace Bsa
-{
-    class BSAFile;
-}
 
 namespace NifOsg
 {
@@ -41,8 +39,7 @@ namespace NifOsg
 
         void loadAsSkeleton(Nif::NIFFilePtr file, osg::Group* parentNode);
 
-        // FIXME replace with resource system
+        VFS::Manager* resourceManager;
-        Bsa::BSAFile* resourceManager;
 
         // FIXME move
         std::vector<Controller> mControllers;

components/vfs/archive.hpp

@@ -0,0 +1,30 @@
+#ifndef OPENMW_COMPONENTS_RESOURCE_ARCHIVE_H
+#define OPENMW_COMPONENTS_RESOURCE_ARCHIVE_H
+
+#include <map>
+
+#include <components/files/constrainedfilestream.hpp>
+
+namespace VFS
+{
+
+    class File
+    {
+    public:
+        virtual ~File() {}
+
+        virtual Files::IStreamPtr open() = 0;
+    };
+
+    class Archive
+    {
+    public:
+        virtual ~Archive() {}
+
+        /// List all resources contained in this archive, and run the resource names through the given normalize function.
+        virtual void listResources(std::map<std::string, File*>& out, char (*normalize_function) (char)) = 0;
+    };
+
+}
+
+#endif

components/vfs/bsaarchive.cpp

@@ -0,0 +1,43 @@
+#include "bsaarchive.hpp"
+
+namespace VFS
+{
+
+
+BsaArchive::BsaArchive(const std::string &filename)
+{
+    mFile.open(filename);
+
+    const Bsa::BSAFile::FileList &filelist = mFile.getList();
+    for(Bsa::BSAFile::FileList::const_iterator it = filelist.begin();it != filelist.end();++it)
+    {
+        mResources.push_back(BsaArchiveFile(&*it, &mFile));
+    }
+}
+
+void BsaArchive::listResources(std::map<std::string, File *> &out, char (*normalize_function)(char))
+{
+    for (std::vector<BsaArchiveFile>::iterator it = mResources.begin(); it != mResources.end(); ++it)
+    {
+        std::string ent = it->mInfo->name;
+        std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function);
+
+        out[ent] = &*it;
+    }
+}
+
+// ------------------------------------------------------------------------------
+
+BsaArchiveFile::BsaArchiveFile(const Bsa::BSAFile::FileStruct *info, Bsa::BSAFile* bsa)
+    : mInfo(info)
+    , mFile(bsa)
+{
+
+}
+
+Files::IStreamPtr BsaArchiveFile::open()
+{
+    return mFile->getFile(mInfo);
+}
+
+}

components/vfs/bsaarchive.hpp

@@ -0,0 +1,32 @@
+#include "archive.hpp"
+
+#include <components/bsa/bsa_file.hpp>
+
+namespace VFS
+{
+
+    class BsaArchiveFile : public File
+    {
+    public:
+        BsaArchiveFile(const Bsa::BSAFile::FileStruct* info, Bsa::BSAFile* bsa);
+
+        virtual Files::IStreamPtr open();
+
+        const Bsa::BSAFile::FileStruct* mInfo;
+        Bsa::BSAFile* mFile;
+    };
+
+    class BsaArchive : public Archive
+    {
+    public:
+        BsaArchive(const std::string& filename);
+
+        virtual void listResources(std::map<std::string, File*>& out, char (*normalize_function) (char));
+
+    private:
+        Bsa::BSAFile mFile;
+
+        std::vector<BsaArchiveFile> mResources;
+    };
+
+}

components/vfs/filesystemarchive.cpp

@@ -0,0 +1,65 @@
+#include "filesystemarchive.hpp"
+
+#include <boost/filesystem.hpp>
+
+namespace VFS
+{
+
+    FileSystemArchive::FileSystemArchive(const std::string &path)
+        : mBuiltIndex(false)
+        , mPath(path)
+    {
+
+    }
+
+    void FileSystemArchive::listResources(std::map<std::string, File *> &out, char (*normalize_function)(char))
+    {
+        if (!mBuiltIndex)
+        {
+            typedef boost::filesystem::recursive_directory_iterator directory_iterator;
+
+            directory_iterator end;
+
+            size_t prefix = mPath.size ();
+
+            if (mPath.size () > 0 && mPath [prefix - 1] != '\\' && mPath [prefix - 1] != '/')
+                ++prefix;
+
+            for (directory_iterator i (mPath); i != end; ++i)
+            {
+                if(boost::filesystem::is_directory (*i))
+                    continue;
+
+                std::string proper = i->path ().string ();
+
+                FileSystemArchiveFile file(proper);
+
+                std::string searchable;
+
+                std::transform(proper.begin() + prefix, proper.end(), std::back_inserter(searchable), normalize_function);
+
+                mIndex.insert (std::make_pair (searchable, file));
+            }
+
+            mBuiltIndex = true;
+        }
+
+        for (index::iterator it = mIndex.begin(); it != mIndex.end(); ++it)
+        {
+            out[it->first] = &it->second;
+        }
+    }
+
+    // ----------------------------------------------------------------------------------
+
+    FileSystemArchiveFile::FileSystemArchiveFile(const std::string &path)
+        : mPath(path)
+    {
+    }
+
+    Files::IStreamPtr FileSystemArchiveFile::open()
+    {
+        return Files::openConstrainedFileStream(mPath.c_str());
+    }
+
+}

components/vfs/filesystemarchive.hpp

@@ -0,0 +1,40 @@
+#ifndef OPENMW_COMPONENTS_RESOURCE_FILESYSTEMARCHIVE_H
+#define OPENMW_COMPONENTS_RESOURCE_FILESYSTEMARCHIVE_H
+
+#include "archive.hpp"
+
+namespace VFS
+{
+
+    class FileSystemArchiveFile : public File
+    {
+    public:
+        FileSystemArchiveFile(const std::string& path);
+
+        virtual Files::IStreamPtr open();
+
+    private:
+        std::string mPath;
+
+    };
+
+    class FileSystemArchive : public Archive
+    {
+    public:
+        FileSystemArchive(const std::string& path);
+
+        virtual void listResources(std::map<std::string, File*>& out, char (*normalize_function) (char));
+
+
+    private:
+        typedef std::map <std::string, FileSystemArchiveFile> index;
+        index mIndex;
+
+        bool mBuiltIndex;
+        std::string mPath;
+
+    };
+
+}
+
+#endif

components/vfs/manager.cpp

@@ -0,0 +1,76 @@
+#include "manager.hpp"
+
+#include <stdexcept>
+
+#include "archive.hpp"
+
+namespace
+{
+
+    char strict_normalize_char(char ch)
+    {
+        return ch == '\\' ? '/' : ch;
+    }
+
+    char nonstrict_normalize_char(char ch)
+    {
+        return ch == '\\' ? '/' : std::tolower(ch,std::locale::classic());
+    }
+
+    void normalize_path(std::string& path, bool strict)
+    {
+        char (*normalize_char)(char) = strict ? &strict_normalize_char : &nonstrict_normalize_char;
+        std::transform(path.begin(), path.end(), path.begin(), normalize_char);
+    }
+
+}
+
+namespace VFS
+{
+
+    Manager::Manager(bool strict)
+        : mStrict(strict)
+    {
+
+    }
+
+    Manager::~Manager()
+    {
+        for (std::vector<Archive*>::iterator it = mArchives.begin(); it != mArchives.end(); ++it)
+            delete *it;
+        mArchives.clear();
+    }
+
+    void Manager::addArchive(Archive *archive)
+    {
+        mArchives.push_back(archive);
+    }
+
+    void Manager::buildIndex()
+    {
+        mIndex.clear();
+
+        for (std::vector<Archive*>::const_iterator it = mArchives.begin(); it != mArchives.end(); ++it)
+            (*it)->listResources(mIndex, mStrict ? &strict_normalize_char : &nonstrict_normalize_char);
+    }
+
+    Files::IStreamPtr Manager::get(const std::string &name) const
+    {
+        std::string normalized = name;
+        normalize_path(normalized, mStrict);
+
+        std::map<std::string, File*>::const_iterator found = mIndex.find(normalized);
+        if (found == mIndex.end())
+            throw std::runtime_error("Resource '" + name + "' not found");
+        return found->second->open();
+    }
+
+    bool Manager::exists(const std::string &name) const
+    {
+        std::string normalized = name;
+        normalize_path(normalized, mStrict);
+
+        return mIndex.find(normalized) != mIndex.end();
+    }
+
+}

components/vfs/manager.hpp

@@ -0,0 +1,52 @@
+#ifndef OPENMW_COMPONENTS_RESOURCEMANAGER_H
+#define OPENMW_COMPONENTS_RESOURCEMANAGER_H
+
+#include <components/files/constrainedfilestream.hpp>
+
+#include <vector>
+#include <map>
+
+namespace VFS
+{
+
+    class Archive;
+    class File;
+
+    /// @brief The main class responsible for loading files from a virtual file system.
+    /// @par Various archive types (e.g. directories on the filesystem, or compressed archives)
+    /// can be registered, and will be merged into a single file tree. If the same filename is
+    /// contained in multiple archives, the last added archive will have priority.
+    class Manager
+    {
+    public:
+        /// @param strict Use strict path handling? If enabled, no case folding will
+        /// be done, but slash/backslash conversions are always done.
+        Manager(bool strict);
+
+        ~Manager();
+
+        /// Register the given archive. All files contained in it will be added to the index on the next buildIndex() call.
+        /// @note Takes ownership of the given pointer.
+        void addArchive(Archive* archive);
+
+        /// Build the file index. Should be called when all archives have been registered.
+        void buildIndex();
+
+        /// Does a file with this name exist?
+        bool exists(const std::string& name) const;
+
+        /// Retrieve a file by name.
+        /// @note Throws an exception if the file can not be found.
+        Files::IStreamPtr get(const std::string& name) const;
+
+    private:
+        bool mStrict;
+
+        std::vector<Archive*> mArchives;
+
+        std::map<std::string, File*> mIndex;
+    };
+
+}
+
+#endif