From c04a0ca3a583e4c64c546c075de46d4c75516c99 Mon Sep 17 00:00:00 2001 From: Andrei Kortunov Date: Tue, 22 Aug 2023 18:04:14 +0400 Subject: [PATCH 1/3] Implement Lua API for VFS --- CHANGELOG.md | 1 + CMakeLists.txt | 2 +- apps/openmw/CMakeLists.txt | 2 +- apps/openmw/mwlua/luabindings.cpp | 2 + apps/openmw/mwlua/vfsbindings.cpp | 354 ++++++++++++++++++ apps/openmw/mwlua/vfsbindings.hpp | 13 + components/vfs/manager.cpp | 13 + components/vfs/manager.hpp | 40 ++ docs/source/reference/lua-scripting/api.rst | 1 + .../reference/lua-scripting/openmw_vfs.rst | 7 + .../lua-scripting/tables/packages.rst | 2 + files/lua_api/CMakeLists.txt | 7 +- files/lua_api/openmw/vfs.lua | 159 ++++++++ 13 files changed, 598 insertions(+), 5 deletions(-) create mode 100644 apps/openmw/mwlua/vfsbindings.cpp create mode 100644 apps/openmw/mwlua/vfsbindings.hpp create mode 100644 docs/source/reference/lua-scripting/openmw_vfs.rst create mode 100644 files/lua_api/openmw/vfs.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index 21c1c55275..81223fd270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ Feature #6491: Add support for Qt6 Feature #6556: Lua API for sounds Feature #6726: Lua API for creating new objects + Feature #6864: Lua file access API Feature #6922: Improve launcher appearance Feature #6933: Support high-resolution cursor textures Feature #6945: Support S3TC-compressed and BGR/BGRA NiPixelData diff --git a/CMakeLists.txt b/CMakeLists.txt index f887cb181e..adfb7ca7f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,7 @@ message(STATUS "Configuring OpenMW...") set(OPENMW_VERSION_MAJOR 0) set(OPENMW_VERSION_MINOR 49) set(OPENMW_VERSION_RELEASE 0) -set(OPENMW_LUA_API_REVISION 45) +set(OPENMW_LUA_API_REVISION 46) set(OPENMW_VERSION_COMMITHASH "") set(OPENMW_VERSION_TAGHASH "") diff --git a/apps/openmw/CMakeLists.txt b/apps/openmw/CMakeLists.txt index 3df00f1be0..a8c233ff56 100644 --- a/apps/openmw/CMakeLists.txt +++ b/apps/openmw/CMakeLists.txt @@ -61,7 +61,7 @@ add_openmw_dir (mwscript add_openmw_dir (mwlua luamanagerimp object objectlists userdataserializer luaevents engineevents objectvariant context globalscripts localscripts playerscripts luabindings objectbindings cellbindings mwscriptbindings - camerabindings uibindings soundbindings inputbindings nearbybindings postprocessingbindings stats debugbindings + camerabindings vfsbindings uibindings soundbindings inputbindings nearbybindings postprocessingbindings stats debugbindings types/types types/door types/item types/actor types/container types/lockable types/weapon types/npc types/creature types/player types/activator types/book types/lockpick types/probe types/apparatus types/potion types/ingredient types/misc types/repair types/armor types/light types/static types/clothing types/levelledlist types/terminal worker magicbindings ) diff --git a/apps/openmw/mwlua/luabindings.cpp b/apps/openmw/mwlua/luabindings.cpp index 2bace90614..d183d86e36 100644 --- a/apps/openmw/mwlua/luabindings.cpp +++ b/apps/openmw/mwlua/luabindings.cpp @@ -43,6 +43,7 @@ #include "soundbindings.hpp" #include "types/types.hpp" #include "uibindings.hpp" +#include "vfsbindings.hpp" namespace MWLua { @@ -331,6 +332,7 @@ namespace MWLua { "openmw.core", initCorePackage(context) }, { "openmw.types", initTypesPackage(context) }, { "openmw.util", LuaUtil::initUtilPackage(lua) }, + { "openmw.vfs", initVFSPackage(context) }, }; } diff --git a/apps/openmw/mwlua/vfsbindings.cpp b/apps/openmw/mwlua/vfsbindings.cpp new file mode 100644 index 0000000000..d91f3f669c --- /dev/null +++ b/apps/openmw/mwlua/vfsbindings.cpp @@ -0,0 +1,354 @@ +#include "vfsbindings.hpp" + +#include +#include +#include +#include +#include + +#include "../mwbase/environment.hpp" + +#include "context.hpp" +#include "luamanagerimp.hpp" + +namespace MWLua +{ + namespace + { + // Too many arguments may cause stack corruption and crash. + constexpr std::size_t sMaximumReadArguments = 20; + + // Print a message if we read a large chunk of file to string. + constexpr std::size_t sFileSizeWarningThreshold = 1024 * 1024; + + struct FileHandle + { + public: + FileHandle(Files::IStreamPtr stream, std::string_view fileName) + { + mFilePtr = std::move(stream); + mFileName = fileName; + } + + Files::IStreamPtr mFilePtr; + std::string mFileName; + }; + + std::ios_base::seekdir getSeekDir(FileHandle& self, std::string_view whence) + { + if (whence == "cur") + return std::ios_base::cur; + if (whence == "set") + return std::ios_base::beg; + if (whence == "end") + return std::ios_base::end; + + throw std::runtime_error( + "Error when handling '" + self.mFileName + "': invalid seek direction: '" + std::string(whence) + "'."); + } + + size_t getBytesLeftInStream(Files::IStreamPtr& file) + { + auto oldPos = file->tellg(); + file->seekg(0, std::ios_base::end); + auto newPos = file->tellg(); + file->seekg(oldPos, std::ios_base::beg); + + return newPos - oldPos; + } + + void printLargeDataMessage(FileHandle& file, size_t size) + { + if (!file.mFilePtr || !Settings::lua().mLuaDebug || size < sFileSizeWarningThreshold) + return; + + Log(Debug::Verbose) << "Read a large data chunk (" << size << " bytes) from '" << file.mFileName << "'."; + } + + sol::object readFile(LuaUtil::LuaState* lua, FileHandle& file) + { + std::ostringstream os; + if (file.mFilePtr && file.mFilePtr->peek() != EOF) + os << file.mFilePtr->rdbuf(); + + auto result = os.str(); + printLargeDataMessage(file, result.size()); + return sol::make_object(lua->sol(), std::move(result)); + } + + sol::object readLineFromFile(LuaUtil::LuaState* lua, FileHandle& file) + { + std::string result; + if (file.mFilePtr && std::getline(*file.mFilePtr, result)) + { + printLargeDataMessage(file, result.size()); + return sol::make_object(lua->sol(), result); + } + + return sol::nil; + } + + sol::object readNumberFromFile(LuaUtil::LuaState* lua, Files::IStreamPtr& file) + { + double number = 0; + if (file && *file >> number) + return sol::make_object(lua->sol(), number); + + return sol::nil; + } + + sol::object readCharactersFromFile(LuaUtil::LuaState* lua, FileHandle& file, size_t count) + { + if (count <= 0 && file.mFilePtr->peek() != EOF) + return sol::make_object(lua->sol(), std::string()); + + auto bytesLeft = getBytesLeftInStream(file.mFilePtr); + if (bytesLeft <= 0) + return sol::nil; + + if (count > bytesLeft) + count = bytesLeft; + + std::string result(count, '\0'); + if (file.mFilePtr->read(&result[0], count)) + { + printLargeDataMessage(file, result.size()); + return sol::make_object(lua->sol(), result); + } + + return sol::nil; + } + + void validateFile(const FileHandle& self) + { + if (self.mFilePtr) + return; + + throw std::runtime_error("Error when handling '" + self.mFileName + "': attempt to use a closed file."); + } + + sol::variadic_results seek( + LuaUtil::LuaState* lua, FileHandle& self, std::ios_base::seekdir dir, std::streamoff off) + { + sol::variadic_results values; + try + { + self.mFilePtr->seekg(off, dir); + if (self.mFilePtr->fail() || self.mFilePtr->bad()) + { + auto msg = "Failed to seek in file '" + self.mFileName + "'"; + values.push_back(sol::nil); + values.push_back(sol::make_object(lua->sol(), msg)); + } + else + values.push_back(sol::make_object(lua->sol(), self.mFilePtr->tellg())); + } + catch (std::exception& e) + { + auto msg = "Failed to seek in file '" + self.mFileName + "': " + std::string(e.what()); + values.push_back(sol::nil); + values.push_back(sol::make_object(lua->sol(), msg)); + } + + return values; + } + } + + sol::table initVFSPackage(const Context& context) + { + sol::table api(context.mLua->sol(), sol::create); + + auto vfs = MWBase::Environment::get().getResourceSystem()->getVFS(); + + sol::usertype vfsIterator + = context.mLua->sol().new_usertype("VFSIterator"); + vfsIterator[sol::meta_function::to_string] = [](const VFS::Manager::StatefulIterator& vfsIterator) { + return "VFSIterator{'" + vfsIterator.getPath() + "'}"; + }; + vfsIterator["path"] = sol::readonly_property( + [](const VFS::Manager::StatefulIterator& vfsIterator) { return vfsIterator.getPath(); }); + + auto createIter = [](VFS::Manager::StatefulIterator& vfsIterator) { + return sol::as_function([vfsIterator, i = 1]() mutable { + if (auto v = vfsIterator.next()) + return std::tuple, sol::optional>(i++, *v); + else + return std::tuple, sol::optional>(sol::nullopt, sol::nullopt); + }); + }; + vfsIterator["__pairs"] = createIter; + vfsIterator["__ipairs"] = createIter; + + sol::usertype handle = context.mLua->sol().new_usertype("FileHandle"); + handle["fileName"] = sol::readonly_property([](const FileHandle& self) { return self.mFileName; }); + handle[sol::meta_function::to_string] = [](const FileHandle& self) { + return "FileHandle{'" + self.mFileName + "'" + (!self.mFilePtr ? ", closed" : "") + "}"; + }; + handle["seek"] = sol::overload( + [lua = context.mLua](FileHandle& self, std::string_view whence, sol::optional offset) { + validateFile(self); + + auto off = static_cast(offset.value_or(0)); + auto dir = getSeekDir(self, whence); + + return seek(lua, self, dir, off); + }, + [lua = context.mLua](FileHandle& self, sol::optional offset) { + validateFile(self); + + auto off = static_cast(offset.value_or(0)); + + return seek(lua, self, std::ios_base::cur, off); + }); + handle["lines"] = [lua = context.mLua](FileHandle& self) { + return sol::as_function([&lua, &self]() mutable { + validateFile(self); + return readLineFromFile(lua, self); + }); + }; + + api["lines"] = [lua = context.mLua, vfs](std::string_view fileName) { + auto normalizedName = VFS::Path::normalizeFilename(fileName); + return sol::as_function( + [lua, file = FileHandle(vfs->getNormalized(normalizedName), normalizedName)]() mutable { + validateFile(file); + auto result = readLineFromFile(lua, file); + if (result == sol::nil) + file.mFilePtr.reset(); + + return result; + }); + }; + + handle["close"] = [lua = context.mLua](FileHandle& self) { + sol::variadic_results values; + try + { + self.mFilePtr.reset(); + if (self.mFilePtr) + { + auto msg = "Can not close file '" + self.mFileName + "': file handle is still opened."; + values.push_back(sol::nil); + values.push_back(sol::make_object(lua->sol(), msg)); + } + else + values.push_back(sol::make_object(lua->sol(), true)); + } + catch (std::exception& e) + { + auto msg = "Can not close file '" + self.mFileName + "': " + std::string(e.what()); + values.push_back(sol::nil); + values.push_back(sol::make_object(lua->sol(), msg)); + } + + return values; + }; + + handle["read"] = [lua = context.mLua](FileHandle& self, const sol::variadic_args args) { + validateFile(self); + + if (args.size() > sMaximumReadArguments) + throw std::runtime_error( + "Error when handling '" + self.mFileName + "': too many arguments for 'read'."); + + sol::variadic_results values; + // If there are no arguments, read a string + if (args.size() == 0) + { + values.push_back(readLineFromFile(lua, self)); + return values; + } + + bool success = true; + size_t i = 0; + for (i = 0; i < args.size() && success; i++) + { + if (args[i].is()) + { + auto format = args[i].as(); + + if (format == "*a" || format == "*all") + { + values.push_back(readFile(lua, self)); + continue; + } + + if (format == "*n" || format == "*number") + { + auto result = readNumberFromFile(lua, self.mFilePtr); + values.push_back(result); + if (result == sol::nil) + success = false; + continue; + } + + if (format == "*l" || format == "*line") + { + auto result = readLineFromFile(lua, self); + values.push_back(result); + if (result == sol::nil) + success = false; + continue; + } + + throw std::runtime_error("Error when handling '" + self.mFileName + "': bad argument #" + + std::to_string(i + 1) + " to 'read' (invalid format)"); + } + else if (args[i].is()) + { + int number = args[i].as(); + auto result = readCharactersFromFile(lua, self, number); + values.push_back(result); + if (result == sol::nil) + success = false; + } + } + + // We should return nil if we just reached the end of stream + if (!success && self.mFilePtr->eof()) + return values; + + if (!success && (self.mFilePtr->fail() || self.mFilePtr->bad())) + { + auto msg = "Error when handling '" + self.mFileName + "': can not read data for argument #" + + std::to_string(i); + values.push_back(sol::make_object(lua->sol(), msg)); + } + + return values; + }; + + api["open"] = [lua = context.mLua, vfs](std::string_view fileName) { + sol::variadic_results values; + try + { + auto normalizedName = VFS::Path::normalizeFilename(fileName); + auto handle = FileHandle(vfs->getNormalized(normalizedName), normalizedName); + values.push_back(sol::make_object(lua->sol(), std::move(handle))); + } + catch (std::exception& e) + { + auto msg = "Can not open file: " + std::string(e.what()); + values.push_back(sol::nil); + values.push_back(sol::make_object(lua->sol(), msg)); + } + + return values; + }; + + api["type"] = sol::overload( + [](const FileHandle& handle) -> std::string { + if (handle.mFilePtr) + return "file"; + + return "closed file"; + }, + [](const sol::object&) -> sol::object { return sol::nil; }); + + api["fileExists"] = [vfs](std::string_view fileName) -> bool { return vfs->exists(fileName); }; + api["getIterator"] + = [vfs](std::string_view path) -> VFS::Manager::StatefulIterator { return vfs->getStatefulIterator(path); }; + + return LuaUtil::makeReadOnly(api); + } +} diff --git a/apps/openmw/mwlua/vfsbindings.hpp b/apps/openmw/mwlua/vfsbindings.hpp new file mode 100644 index 0000000000..b251db6fd4 --- /dev/null +++ b/apps/openmw/mwlua/vfsbindings.hpp @@ -0,0 +1,13 @@ +#ifndef MWLUA_VFSBINDINGS_H +#define MWLUA_VFSBINDINGS_H + +#include + +#include "context.hpp" + +namespace MWLua +{ + sol::table initVFSPackage(const Context&); +} + +#endif // MWLUA_VFSBINDINGS_H diff --git a/components/vfs/manager.cpp b/components/vfs/manager.cpp index bfc001e4f2..0484e06c54 100644 --- a/components/vfs/manager.cpp +++ b/components/vfs/manager.cpp @@ -89,4 +89,17 @@ namespace VFS ++normalized.back(); return { it, mIndex.lower_bound(normalized) }; } + + Manager::StatefulIterator Manager::getStatefulIterator(std::string_view path) const + { + if (path.empty()) + return { mIndex.begin(), mIndex.end(), std::string() }; + std::string normalized = Path::normalizeFilename(path); + const auto it = mIndex.lower_bound(normalized); + if (it == mIndex.end() || !startsWith(it->first, normalized)) + return { it, it, normalized }; + std::string upperBound = normalized; + ++upperBound.back(); + return { it, mIndex.lower_bound(upperBound), normalized }; + } } diff --git a/components/vfs/manager.hpp b/components/vfs/manager.hpp index db38e4b240..bfb44c3fc2 100644 --- a/components/vfs/manager.hpp +++ b/components/vfs/manager.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -48,11 +49,18 @@ namespace VFS const std::string& operator*() const { return mIt->first; } const std::string* operator->() const { return &mIt->first; } bool operator!=(const RecursiveDirectoryIterator& other) { return mIt != other.mIt; } + bool operator==(const RecursiveDirectoryIterator& other) const { return mIt == other.mIt; } RecursiveDirectoryIterator& operator++() { ++mIt; return *this; } + RecursiveDirectoryIterator operator++(int) + { + RecursiveDirectoryIterator old = *this; + mIt++; + return old; + } private: std::map::const_iterator mIt; @@ -61,6 +69,31 @@ namespace VFS using RecursiveDirectoryRange = IteratorPair; public: + class StatefulIterator : RecursiveDirectoryRange + { + public: + StatefulIterator(RecursiveDirectoryIterator first, RecursiveDirectoryIterator last, const std::string& path) + : RecursiveDirectoryRange(first, last) + , mCurrent(first) + , mPath(path) + { + } + + const std::string& getPath() const { return mPath; } + + std::optional next() + { + if (mCurrent == end()) + return std::nullopt; + + return *mCurrent++; + } + + private: + RecursiveDirectoryIterator mCurrent; + std::string mPath; + }; + // Empty the file index and unregister archives. void reset(); @@ -93,6 +126,13 @@ namespace VFS /// @note May be called from any thread once the index has been built. RecursiveDirectoryRange getRecursiveDirectoryIterator(std::string_view path) const; + /// Recursively iterate over the elements of the given path + /// In practice it return all files of the VFS starting with the given path + /// Stores iterator to current element. + /// @note the path is normalized + /// @note May be called from any thread once the index has been built. + StatefulIterator getStatefulIterator(std::string_view path) const; + /// Retrieve the absolute path to the file /// @note Throws an exception if the file can not be found. /// @note May be called from any thread once the index has been built. diff --git a/docs/source/reference/lua-scripting/api.rst b/docs/source/reference/lua-scripting/api.rst index 77d4c9b14b..9000ef1188 100644 --- a/docs/source/reference/lua-scripting/api.rst +++ b/docs/source/reference/lua-scripting/api.rst @@ -17,6 +17,7 @@ Lua API reference openmw_core openmw_types openmw_async + openmw_vfs openmw_world openmw_self openmw_nearby diff --git a/docs/source/reference/lua-scripting/openmw_vfs.rst b/docs/source/reference/lua-scripting/openmw_vfs.rst new file mode 100644 index 0000000000..407459e7e0 --- /dev/null +++ b/docs/source/reference/lua-scripting/openmw_vfs.rst @@ -0,0 +1,7 @@ +Package openmw.vfs +================== + +.. include:: version.rst + +.. raw:: html + :file: generated_html/openmw_vfs.html diff --git a/docs/source/reference/lua-scripting/tables/packages.rst b/docs/source/reference/lua-scripting/tables/packages.rst index e746274e6d..67709bbf7b 100644 --- a/docs/source/reference/lua-scripting/tables/packages.rst +++ b/docs/source/reference/lua-scripting/tables/packages.rst @@ -15,6 +15,8 @@ +------------------------------------------------------------+--------------------+---------------------------------------------------------------+ |:ref:`openmw.async ` | everywhere | | Timers and callbacks. | +------------------------------------------------------------+--------------------+---------------------------------------------------------------+ +|:ref:`openmw.vfs ` | everywhere | | Read-only access to data directories via VFS. | ++------------------------------------------------------------+--------------------+---------------------------------------------------------------+ |:ref:`openmw.world ` | by global scripts | | Read-write access to the game world. | +------------------------------------------------------------+--------------------+---------------------------------------------------------------+ |:ref:`openmw.self ` | by local scripts | | Full access to the object the script is attached to. | diff --git a/files/lua_api/CMakeLists.txt b/files/lua_api/CMakeLists.txt index 87f5bfe91b..96409e803e 100644 --- a/files/lua_api/CMakeLists.txt +++ b/files/lua_api/CMakeLists.txt @@ -12,14 +12,15 @@ set(LUA_API_FILES openmw/ambient.lua openmw/async.lua openmw/core.lua + openmw/debug.lua openmw/nearby.lua + openmw/postprocessing.lua openmw/self.lua + openmw/types.lua openmw/ui.lua openmw/util.lua + openmw/vfs.lua openmw/world.lua - openmw/types.lua - openmw/postprocessing.lua - openmw/debug.lua ) foreach (f ${LUA_API_FILES}) diff --git a/files/lua_api/openmw/vfs.lua b/files/lua_api/openmw/vfs.lua new file mode 100644 index 0000000000..f432273810 --- /dev/null +++ b/files/lua_api/openmw/vfs.lua @@ -0,0 +1,159 @@ +--- +-- `openmw.vfs` provides read-only access to data directories via VFS. +-- Interface is very similar to "io" library. +-- @module vfs +-- @usage local vfs = require('openmw.vfs') + + + +--- +-- @type VFSIterator +-- @field #string path VFS prefix path + +--- +-- @type FileHandle +-- @field #string fileName VFS path to related file + +--- +-- Close a file handle +-- @function [parent=#FileHandle] close +-- @param self +-- @return #boolean true if a call succeeds without errors. +-- @return #nil, #string nil plus the error message in case of any error. + +--- +-- Get an iterator function to fetch the next line from given file. +-- Throws an exception if file is closed. +-- +-- Hint: since garbage collection works once per frame, +-- you will get the whole file in RAM if you read it in one frame. +-- So if you need to read a really large file, it is better to split reading +-- between different frames (e.g. by keeping a current position in file +-- and using a "seek" to read from saved position). +-- @function [parent=#FileHandle] lines +-- @param self +-- @return #function Iterator function to get next line +-- @usage f = vfs.open("Test\\test.txt"); +-- for line in f:lines() do +-- print(line); +-- end + +--- +-- Set new position in file. +-- Throws an exception if file is closed or seek base is incorrect. +-- @function [parent=#FileHandle] seek +-- @param self +-- @param #string whence Seek base (optional, "cur" by default). Can be: +-- +-- * "set" - seek from beginning of file; +-- * "cur" - seek from current position; +-- * "end" - seek from end of file (offset needs to be <= 0); +-- @param #number offset Offset from given base (optional, 0 by default) +-- @return #number new position in file if a call succeeds without errors. +-- @return #nil, #string nil plus the error message in case of any error. +-- @usage -- set pointer to beginning of file +-- f = vfs.open("Test\\test.txt"); +-- f:seek("set"); +-- @usage -- print current position in file +-- f = vfs.open("Test\\test.txt"); +-- print(f:seek()); +-- @usage -- print file size +-- f = vfs.open("Test\\test.txt"); +-- print(f:seek("end")); + +--- +-- Read data from file to strings. +-- Throws an exception if file is closed, if there is too many arguments or if an invalid format encountered. +-- +-- Hint: since garbage collection works once per frame, +-- you will get the whole file in RAM if you read it in one frame. +-- So if you need to read a really large file, it is better to split reading +-- between different frames (e.g. by keeping a current position in file +-- and using a "seek" to read from saved position). +-- @function [parent=#FileHandle] read +-- @param self +-- @param ... Read formats (up to 20 arguments, default value is one "*l"). Can be: +-- +-- * "\*a" (or "*all") - reads the whole file, starting at the current position as #string. On end of file, it returns the empty string. +-- * "\*l" (or "*line") - reads the next line (skipping the end of line), returning nil on end of file (nil and error message if error occured); +-- * "\*n" (or "*number") - read a floating point value as #number (nil and error message if error occured); +-- * number - reads a #string with up to this number of characters, returning nil on end of file (nil and error message if error occured). If number is 0 and end of file is not reached, it reads nothing and returns an empty string; +-- @return #string One #string for every format if a call succeeds without errors. One #string for every successfully handled format, nil for first failed format. +-- @usage -- read three numbers from file +-- f = vfs.open("Test\\test.txt"); +-- local n1, n2, n3 = f:read("*number", "*number", "*number"); +-- @usage -- read 10 bytes from file +-- f = vfs.open("Test\\test.txt"); +-- local n4 = f:read(10); +-- @usage -- read until end of file +-- f = vfs.open("Test\\test.txt"); +-- local n5 = f:read("*all"); +-- @usage -- read a line from file +-- f = vfs.open("Test\\test.txt"); +-- local n6 = f:read(); +-- @usage -- try to read three numbers from file with "1" content +-- f = vfs.open("one.txt"); +-- print(f:read("*number", "*number", "*number")); +-- -- prints(1, nil) + +--- +-- Check if file exists in VFS +-- @function [parent=#vfs] fileExists +-- @param #string fileName Path to file in VFS +-- @return #boolean (true - exists, false - does not exist) +-- @usage local exists = vfs.fileExists("Test\\test.txt"); + +--- +-- Open a file +-- @function [parent=#vfs] open +-- @param #string fileName Path to file in VFS +-- @return #FileHandle Opened file handle if a call succeeds without errors. +-- @return #nil, #string nil plus the error message in case of any error. +-- @usage f, msg = vfs.open("Test\\test.txt"); +-- -- print file name or error message +-- if (f == nil) +-- print(msg); +-- else +-- print(f.fileName); +-- end + +--- +-- Get an iterator function to fetch the next line from file with given path. +-- Throws an exception if file is closed or file with given path does not exist. +-- Closes file automatically when it fails to read any more bytes. +-- +-- Hint: since garbage collection works once per frame, +-- you will get the whole file in RAM if you read it in one frame. +-- So if you need to read a really large file, it is better to split reading +-- between different frames (e.g. by keeping a current position in file +-- and using a "seek" to read from saved position). +-- @function [parent=#vfs] lines +-- @param #string fileName Path to file in VFS +-- @return #function Iterator function to get next line +-- @usage for line in vfs.lines("Test\\test.txt") do +-- print(line); +-- end + +--- +-- Get iterator to fetch file names with given path prefix from VFS +-- @function [parent=#vfs] getIterator +-- @param #string path Path prefix +-- @return #VFSIterator Opened iterator +-- @usage local dir = vfs.getIterator("Music\\Explore"); +-- for _, fileName in pairs(dir) do +-- print(fileName); +-- end + +--- +-- Detect a file handle type +-- @function [parent=#vfs] type +-- @param #any handle Object to check +-- @return #string File handle type. Can be: +-- +-- * "file" - an argument is a valid opened @{openmw.vfs#FileHandle}; +-- * "closed file" - an argument is a valid closed @{openmw.vfs#FileHandle}; +-- * nil - an argument is not a @{openmw.vfs#FileHandle}; +-- @usage f = vfs.open("Test\\test.txt"); +-- print(vfs.type(f)); + +return nil From 65109b3822f74af4ef22e6783f89c434d62bee8f Mon Sep 17 00:00:00 2001 From: Andrei Kortunov Date: Sat, 2 Sep 2023 17:32:22 +0400 Subject: [PATCH 2/3] Simplify VFS index iteration --- apps/openmw/mwlua/vfsbindings.cpp | 34 ++++++++++---------------- components/vfs/manager.cpp | 13 ---------- components/vfs/manager.hpp | 40 ------------------------------- files/lua_api/openmw/vfs.lua | 18 +++++++------- 4 files changed, 22 insertions(+), 83 deletions(-) diff --git a/apps/openmw/mwlua/vfsbindings.cpp b/apps/openmw/mwlua/vfsbindings.cpp index d91f3f669c..ad32520649 100644 --- a/apps/openmw/mwlua/vfsbindings.cpp +++ b/apps/openmw/mwlua/vfsbindings.cpp @@ -160,25 +160,6 @@ namespace MWLua auto vfs = MWBase::Environment::get().getResourceSystem()->getVFS(); - sol::usertype vfsIterator - = context.mLua->sol().new_usertype("VFSIterator"); - vfsIterator[sol::meta_function::to_string] = [](const VFS::Manager::StatefulIterator& vfsIterator) { - return "VFSIterator{'" + vfsIterator.getPath() + "'}"; - }; - vfsIterator["path"] = sol::readonly_property( - [](const VFS::Manager::StatefulIterator& vfsIterator) { return vfsIterator.getPath(); }); - - auto createIter = [](VFS::Manager::StatefulIterator& vfsIterator) { - return sol::as_function([vfsIterator, i = 1]() mutable { - if (auto v = vfsIterator.next()) - return std::tuple, sol::optional>(i++, *v); - else - return std::tuple, sol::optional>(sol::nullopt, sol::nullopt); - }); - }; - vfsIterator["__pairs"] = createIter; - vfsIterator["__ipairs"] = createIter; - sol::usertype handle = context.mLua->sol().new_usertype("FileHandle"); handle["fileName"] = sol::readonly_property([](const FileHandle& self) { return self.mFileName; }); handle[sol::meta_function::to_string] = [](const FileHandle& self) { @@ -346,8 +327,19 @@ namespace MWLua [](const sol::object&) -> sol::object { return sol::nil; }); api["fileExists"] = [vfs](std::string_view fileName) -> bool { return vfs->exists(fileName); }; - api["getIterator"] - = [vfs](std::string_view path) -> VFS::Manager::StatefulIterator { return vfs->getStatefulIterator(path); }; + api["pathsWithPrefix"] = [vfs](std::string_view prefix) { + auto iterator = vfs->getRecursiveDirectoryIterator(prefix); + return sol::as_function([iterator, current = iterator.begin()]() mutable -> sol::optional { + if (current != iterator.end()) + { + const std::string& result = *current; + ++current; + return result; + } + + return sol::nullopt; + }); + }; return LuaUtil::makeReadOnly(api); } diff --git a/components/vfs/manager.cpp b/components/vfs/manager.cpp index 0484e06c54..bfc001e4f2 100644 --- a/components/vfs/manager.cpp +++ b/components/vfs/manager.cpp @@ -89,17 +89,4 @@ namespace VFS ++normalized.back(); return { it, mIndex.lower_bound(normalized) }; } - - Manager::StatefulIterator Manager::getStatefulIterator(std::string_view path) const - { - if (path.empty()) - return { mIndex.begin(), mIndex.end(), std::string() }; - std::string normalized = Path::normalizeFilename(path); - const auto it = mIndex.lower_bound(normalized); - if (it == mIndex.end() || !startsWith(it->first, normalized)) - return { it, it, normalized }; - std::string upperBound = normalized; - ++upperBound.back(); - return { it, mIndex.lower_bound(upperBound), normalized }; - } } diff --git a/components/vfs/manager.hpp b/components/vfs/manager.hpp index bfb44c3fc2..db38e4b240 100644 --- a/components/vfs/manager.hpp +++ b/components/vfs/manager.hpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -49,18 +48,11 @@ namespace VFS const std::string& operator*() const { return mIt->first; } const std::string* operator->() const { return &mIt->first; } bool operator!=(const RecursiveDirectoryIterator& other) { return mIt != other.mIt; } - bool operator==(const RecursiveDirectoryIterator& other) const { return mIt == other.mIt; } RecursiveDirectoryIterator& operator++() { ++mIt; return *this; } - RecursiveDirectoryIterator operator++(int) - { - RecursiveDirectoryIterator old = *this; - mIt++; - return old; - } private: std::map::const_iterator mIt; @@ -69,31 +61,6 @@ namespace VFS using RecursiveDirectoryRange = IteratorPair; public: - class StatefulIterator : RecursiveDirectoryRange - { - public: - StatefulIterator(RecursiveDirectoryIterator first, RecursiveDirectoryIterator last, const std::string& path) - : RecursiveDirectoryRange(first, last) - , mCurrent(first) - , mPath(path) - { - } - - const std::string& getPath() const { return mPath; } - - std::optional next() - { - if (mCurrent == end()) - return std::nullopt; - - return *mCurrent++; - } - - private: - RecursiveDirectoryIterator mCurrent; - std::string mPath; - }; - // Empty the file index and unregister archives. void reset(); @@ -126,13 +93,6 @@ namespace VFS /// @note May be called from any thread once the index has been built. RecursiveDirectoryRange getRecursiveDirectoryIterator(std::string_view path) const; - /// Recursively iterate over the elements of the given path - /// In practice it return all files of the VFS starting with the given path - /// Stores iterator to current element. - /// @note the path is normalized - /// @note May be called from any thread once the index has been built. - StatefulIterator getStatefulIterator(std::string_view path) const; - /// Retrieve the absolute path to the file /// @note Throws an exception if the file can not be found. /// @note May be called from any thread once the index has been built. diff --git a/files/lua_api/openmw/vfs.lua b/files/lua_api/openmw/vfs.lua index f432273810..ba381a1249 100644 --- a/files/lua_api/openmw/vfs.lua +++ b/files/lua_api/openmw/vfs.lua @@ -6,10 +6,6 @@ ---- --- @type VFSIterator --- @field #string path VFS prefix path - --- -- @type FileHandle -- @field #string fileName VFS path to related file @@ -135,14 +131,18 @@ -- end --- --- Get iterator to fetch file names with given path prefix from VFS --- @function [parent=#vfs] getIterator +-- Get iterator function to fetch file names with given path prefix from VFS +-- @function [parent=#vfs] pathsWithPrefix -- @param #string path Path prefix --- @return #VFSIterator Opened iterator --- @usage local dir = vfs.getIterator("Music\\Explore"); --- for _, fileName in pairs(dir) do +-- @return #function Function to get next file name +-- @usage -- get all files with given prefix from VFS index +-- for fileName in vfs.pathsWithPrefix("Music\\Explore") do -- print(fileName); -- end +-- @usage -- get some first files +-- local getNextFile = vfs.pathsWithPrefix("Music\\Explore"); +-- local firstFile = getNextFile(); +-- local secondFile = getNextFile(); --- -- Detect a file handle type From 66bf7be373a2d705c9819966e9cad1b2f7da39af Mon Sep 17 00:00:00 2001 From: Andrei Kortunov Date: Sun, 3 Sep 2023 08:24:51 +0400 Subject: [PATCH 3/3] Preload new packages to console --- files/data/scripts/omw/console/global.lua | 1 + files/data/scripts/omw/console/local.lua | 1 + files/data/scripts/omw/console/player.lua | 2 ++ 3 files changed, 4 insertions(+) diff --git a/files/data/scripts/omw/console/global.lua b/files/data/scripts/omw/console/global.lua index e3b4375b1e..bba0cbc7b3 100644 --- a/files/data/scripts/omw/console/global.lua +++ b/files/data/scripts/omw/console/global.lua @@ -22,6 +22,7 @@ local env = { storage = require('openmw.storage'), core = require('openmw.core'), types = require('openmw.types'), + vfs = require('openmw.vfs'), async = require('openmw.async'), world = require('openmw.world'), aux_util = require('openmw_aux.util'), diff --git a/files/data/scripts/omw/console/local.lua b/files/data/scripts/omw/console/local.lua index adcad3d6cb..6962b9e798 100644 --- a/files/data/scripts/omw/console/local.lua +++ b/files/data/scripts/omw/console/local.lua @@ -24,6 +24,7 @@ local env = { storage = require('openmw.storage'), core = require('openmw.core'), types = require('openmw.types'), + vfs = require('openmw.vfs'), async = require('openmw.async'), nearby = require('openmw.nearby'), self = require('openmw.self'), diff --git a/files/data/scripts/omw/console/player.lua b/files/data/scripts/omw/console/player.lua index 464482e7ad..c614d2d962 100644 --- a/files/data/scripts/omw/console/player.lua +++ b/files/data/scripts/omw/console/player.lua @@ -71,6 +71,8 @@ local env = { storage = require('openmw.storage'), core = require('openmw.core'), types = require('openmw.types'), + vfs = require('openmw.vfs'), + ambient = require('openmw.ambient'), async = require('openmw.async'), nearby = require('openmw.nearby'), self = require('openmw.self'),