From 715efe892f8429960258d3b953c4c77781e200c4 Mon Sep 17 00:00:00 2001 From: Andrei Kortunov Date: Tue, 5 Mar 2024 10:07:35 +0400 Subject: [PATCH] Load YAML files via Lua (feature 7590) --- CHANGELOG.md | 1 + CI/file_name_exceptions.txt | 1 + CMakeLists.txt | 2 +- apps/openmw/CMakeLists.txt | 2 +- apps/openmw/mwlua/luabindings.cpp | 2 + apps/openmw/mwlua/markupbindings.cpp | 32 ++ apps/openmw/mwlua/markupbindings.hpp | 13 + apps/openmw_test_suite/CMakeLists.txt | 1 + apps/openmw_test_suite/lua/test_yaml.cpp | 354 ++++++++++++++++++ components/CMakeLists.txt | 2 +- components/lua/yamlloader.cpp | 241 ++++++++++++ components/lua/yamlloader.hpp | 50 +++ docs/source/reference/lua-scripting/api.rst | 1 + .../reference/lua-scripting/openmw_markup.rst | 7 + .../lua-scripting/tables/packages.rst | 2 + files/data/scripts/omw/console/global.lua | 1 + files/data/scripts/omw/console/local.lua | 1 + files/data/scripts/omw/console/menu.lua | 1 + files/data/scripts/omw/console/player.lua | 1 + files/lua_api/CMakeLists.txt | 1 + files/lua_api/openmw/markup.lua | 37 ++ 21 files changed, 750 insertions(+), 3 deletions(-) create mode 100644 apps/openmw/mwlua/markupbindings.cpp create mode 100644 apps/openmw/mwlua/markupbindings.hpp create mode 100644 apps/openmw_test_suite/lua/test_yaml.cpp create mode 100644 components/lua/yamlloader.cpp create mode 100644 components/lua/yamlloader.hpp create mode 100644 docs/source/reference/lua-scripting/openmw_markup.rst create mode 100644 files/lua_api/openmw/markup.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index fe081224ce..f1f36e594c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -194,6 +194,7 @@ Feature #7546: Start the game on Fredas Feature #7554: Controller binding for tab for menu navigation Feature #7568: Uninterruptable scripted music + Feature #7590: [Lua] Ability to deserialize YAML data from scripts Feature #7606: Launcher: allow Shift-select in Archives tab Feature #7608: Make the missing dependencies warning when loading a savegame more helpful Feature #7618: Show the player character's health in the save details diff --git a/CI/file_name_exceptions.txt b/CI/file_name_exceptions.txt index c3bcee8661..dff3527348 100644 --- a/CI/file_name_exceptions.txt +++ b/CI/file_name_exceptions.txt @@ -20,6 +20,7 @@ apps/openmw_test_suite/lua/test_storage.cpp apps/openmw_test_suite/lua/test_ui_content.cpp apps/openmw_test_suite/lua/test_utilpackage.cpp apps/openmw_test_suite/lua/test_inputactions.cpp +apps/openmw_test_suite/lua/test_yaml.cpp apps/openmw_test_suite/misc/test_endianness.cpp apps/openmw_test_suite/misc/test_resourcehelpers.cpp apps/openmw_test_suite/misc/test_stringops.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ca25fd05ff..cfcd167cd5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,7 +80,7 @@ message(STATUS "Configuring OpenMW...") set(OPENMW_VERSION_MAJOR 0) set(OPENMW_VERSION_MINOR 49) set(OPENMW_VERSION_RELEASE 0) -set(OPENMW_LUA_API_REVISION 56) +set(OPENMW_LUA_API_REVISION 57) set(OPENMW_POSTPROCESSING_API_REVISION 1) set(OPENMW_VERSION_COMMITHASH "") diff --git a/apps/openmw/CMakeLists.txt b/apps/openmw/CMakeLists.txt index 5fb06881ec..08bf11d194 100644 --- a/apps/openmw/CMakeLists.txt +++ b/apps/openmw/CMakeLists.txt @@ -64,7 +64,7 @@ add_openmw_dir (mwlua context menuscripts globalscripts localscripts playerscripts luabindings objectbindings cellbindings mwscriptbindings camerabindings vfsbindings uibindings soundbindings inputbindings nearbybindings postprocessingbindings stats debugbindings corebindings worldbindings worker magicbindings factionbindings - classbindings itemdata inputprocessor animationbindings birthsignbindings racebindings + classbindings itemdata inputprocessor animationbindings birthsignbindings racebindings markupbindings types/types types/door types/item types/actor types/container types/lockable types/weapon types/npc types/creature types/player types/activator types/book types/lockpick types/probe types/apparatus types/potion types/ingredient types/misc types/repair types/armor types/light types/static diff --git a/apps/openmw/mwlua/luabindings.cpp b/apps/openmw/mwlua/luabindings.cpp index 0de10827e0..553b8af8f6 100644 --- a/apps/openmw/mwlua/luabindings.cpp +++ b/apps/openmw/mwlua/luabindings.cpp @@ -14,6 +14,7 @@ #include "debugbindings.hpp" #include "inputbindings.hpp" #include "localscripts.hpp" +#include "markupbindings.hpp" #include "menuscripts.hpp" #include "nearbybindings.hpp" #include "objectbindings.hpp" @@ -35,6 +36,7 @@ namespace MWLua { "openmw.async", LuaUtil::getAsyncPackageInitializer( lua, [tm] { return tm->getSimulationTime(); }, [tm] { return tm->getGameTime(); }) }, + { "openmw.markup", initMarkupPackage(context) }, { "openmw.util", LuaUtil::initUtilPackage(lua) }, { "openmw.vfs", initVFSPackage(context) }, }; diff --git a/apps/openmw/mwlua/markupbindings.cpp b/apps/openmw/mwlua/markupbindings.cpp new file mode 100644 index 0000000000..997674b45d --- /dev/null +++ b/apps/openmw/mwlua/markupbindings.cpp @@ -0,0 +1,32 @@ +#include "markupbindings.hpp" + +#include +#include +#include +#include +#include + +#include "../mwbase/environment.hpp" + +#include "context.hpp" + +namespace MWLua +{ + sol::table initMarkupPackage(const Context& context) + { + sol::table api(context.mLua->sol(), sol::create); + + auto vfs = MWBase::Environment::get().getResourceSystem()->getVFS(); + + api["loadYaml"] = [lua = context.mLua, vfs](std::string_view fileName) { + auto normalizedName = VFS::Path::normalizeFilename(fileName); + auto file = vfs->getNormalized(normalizedName); + return LuaUtil::YamlLoader::load(*file, lua->sol()); + }; + api["decodeYaml"] = [lua = context.mLua](std::string_view inputData) { + return LuaUtil::YamlLoader::load(std::string(inputData), lua->sol()); + }; + + return LuaUtil::makeReadOnly(api); + } +} diff --git a/apps/openmw/mwlua/markupbindings.hpp b/apps/openmw/mwlua/markupbindings.hpp new file mode 100644 index 0000000000..9105ab5edf --- /dev/null +++ b/apps/openmw/mwlua/markupbindings.hpp @@ -0,0 +1,13 @@ +#ifndef MWLUA_MARKUPBINDINGS_H +#define MWLUA_MARKUPBINDINGS_H + +#include + +#include "context.hpp" + +namespace MWLua +{ + sol::table initMarkupPackage(const Context&); +} + +#endif // MWLUA_MARKUPBINDINGS_H diff --git a/apps/openmw_test_suite/CMakeLists.txt b/apps/openmw_test_suite/CMakeLists.txt index 71da2de590..f3f50cea71 100644 --- a/apps/openmw_test_suite/CMakeLists.txt +++ b/apps/openmw_test_suite/CMakeLists.txt @@ -29,6 +29,7 @@ file(GLOB UNITTEST_SRC_FILES lua/test_storage.cpp lua/test_async.cpp lua/test_inputactions.cpp + lua/test_yaml.cpp lua/test_ui_content.cpp diff --git a/apps/openmw_test_suite/lua/test_yaml.cpp b/apps/openmw_test_suite/lua/test_yaml.cpp new file mode 100644 index 0000000000..c7d484cf51 --- /dev/null +++ b/apps/openmw_test_suite/lua/test_yaml.cpp @@ -0,0 +1,354 @@ +#include "gmock/gmock.h" +#include + +#include + +#include "../testing_util.hpp" + +namespace +{ + template + bool checkNumber(sol::state_view& lua, const std::string& inputData, T requiredValue) + { + sol::object result = LuaUtil::YamlLoader::load(inputData, lua); + if (result.get_type() != sol::type::number) + return false; + + return result.as() == requiredValue; + } + + bool checkBool(sol::state_view& lua, const std::string& inputData, bool requiredValue) + { + sol::object result = LuaUtil::YamlLoader::load(inputData, lua); + if (result.get_type() != sol::type::boolean) + return false; + + return result.as() == requiredValue; + } + + bool checkNil(sol::state_view& lua, const std::string& inputData) + { + sol::object result = LuaUtil::YamlLoader::load(inputData, lua); + return result == sol::nil; + } + + bool checkNan(sol::state_view& lua, const std::string& inputData) + { + sol::object result = LuaUtil::YamlLoader::load(inputData, lua); + if (result.get_type() != sol::type::number) + return false; + + return std::isnan(result.as()); + } + + bool checkString(sol::state_view& lua, const std::string& inputData, const std::string& requiredValue) + { + sol::object result = LuaUtil::YamlLoader::load(inputData, lua); + if (result.get_type() != sol::type::string) + return false; + + return result.as() == requiredValue; + } + + bool checkString(sol::state_view& lua, const std::string& inputData) + { + sol::object result = LuaUtil::YamlLoader::load(inputData, lua); + if (result.get_type() != sol::type::string) + return false; + + return result.as() == inputData; + } + + TEST(LuaUtilYamlLoader, ScalarTypeDeduction) + { + sol::state lua; + + ASSERT_TRUE(checkNil(lua, "null")); + ASSERT_TRUE(checkNil(lua, "Null")); + ASSERT_TRUE(checkNil(lua, "NULL")); + ASSERT_TRUE(checkNil(lua, "~")); + ASSERT_TRUE(checkNil(lua, "")); + ASSERT_FALSE(checkNil(lua, "NUll")); + ASSERT_TRUE(checkString(lua, "NUll")); + ASSERT_TRUE(checkString(lua, "'null'", "null")); + + ASSERT_TRUE(checkNumber(lua, "017", 17)); + ASSERT_TRUE(checkNumber(lua, "-017", -17)); + ASSERT_TRUE(checkNumber(lua, "+017", 17)); + ASSERT_TRUE(checkNumber(lua, "17", 17)); + ASSERT_TRUE(checkNumber(lua, "-17", -17)); + ASSERT_TRUE(checkNumber(lua, "+17", 17)); + ASSERT_TRUE(checkNumber(lua, "0o17", 15)); + ASSERT_TRUE(checkString(lua, "-0o17")); + ASSERT_TRUE(checkString(lua, "+0o17")); + ASSERT_TRUE(checkString(lua, "0b1")); + ASSERT_TRUE(checkString(lua, "1:00")); + ASSERT_TRUE(checkString(lua, "'17'", "17")); + ASSERT_TRUE(checkNumber(lua, "0x17", 23)); + ASSERT_TRUE(checkString(lua, "'-0x17'", "-0x17")); + ASSERT_TRUE(checkString(lua, "'+0x17'", "+0x17")); + + ASSERT_TRUE(checkNumber(lua, "2.1e-05", 2.1e-5)); + ASSERT_TRUE(checkNumber(lua, "-2.1e-05", -2.1e-5)); + ASSERT_TRUE(checkNumber(lua, "+2.1e-05", 2.1e-5)); + ASSERT_TRUE(checkNumber(lua, "2.1e+5", 210000)); + ASSERT_TRUE(checkNumber(lua, "-2.1e+5", -210000)); + ASSERT_TRUE(checkNumber(lua, "+2.1e+5", 210000)); + ASSERT_TRUE(checkNumber(lua, "0.27", 0.27)); + ASSERT_TRUE(checkNumber(lua, "-0.27", -0.27)); + ASSERT_TRUE(checkNumber(lua, "+0.27", 0.27)); + ASSERT_TRUE(checkNumber(lua, "2.7", 2.7)); + ASSERT_TRUE(checkNumber(lua, "-2.7", -2.7)); + ASSERT_TRUE(checkNumber(lua, "+2.7", 2.7)); + ASSERT_TRUE(checkNumber(lua, ".27", 0.27)); + ASSERT_TRUE(checkNumber(lua, "-.27", -0.27)); + ASSERT_TRUE(checkNumber(lua, "+.27", 0.27)); + ASSERT_TRUE(checkNumber(lua, "27.", 27.0)); + ASSERT_TRUE(checkNumber(lua, "-27.", -27.0)); + ASSERT_TRUE(checkNumber(lua, "+27.", 27.0)); + + ASSERT_TRUE(checkNan(lua, ".nan")); + ASSERT_TRUE(checkNan(lua, ".NaN")); + ASSERT_TRUE(checkNan(lua, ".NAN")); + ASSERT_FALSE(checkNan(lua, "nan")); + ASSERT_FALSE(checkNan(lua, ".nAn")); + ASSERT_TRUE(checkString(lua, "'.nan'", ".nan")); + ASSERT_TRUE(checkString(lua, ".nAn")); + + ASSERT_TRUE(checkNumber(lua, "1.7976931348623157E+308", std::numeric_limits::max())); + ASSERT_TRUE(checkNumber(lua, "-1.7976931348623157E+308", std::numeric_limits::lowest())); + ASSERT_TRUE(checkNumber(lua, "2.2250738585072014e-308", std::numeric_limits::min())); + ASSERT_TRUE(checkNumber(lua, ".inf", std::numeric_limits::infinity())); + ASSERT_TRUE(checkNumber(lua, "+.inf", std::numeric_limits::infinity())); + ASSERT_TRUE(checkNumber(lua, "-.inf", -std::numeric_limits::infinity())); + ASSERT_TRUE(checkNumber(lua, ".Inf", std::numeric_limits::infinity())); + ASSERT_TRUE(checkNumber(lua, "+.Inf", std::numeric_limits::infinity())); + ASSERT_TRUE(checkNumber(lua, "-.Inf", -std::numeric_limits::infinity())); + ASSERT_TRUE(checkNumber(lua, ".INF", std::numeric_limits::infinity())); + ASSERT_TRUE(checkNumber(lua, "+.INF", std::numeric_limits::infinity())); + ASSERT_TRUE(checkNumber(lua, "-.INF", -std::numeric_limits::infinity())); + ASSERT_TRUE(checkString(lua, ".INf")); + ASSERT_TRUE(checkString(lua, "-.INf")); + ASSERT_TRUE(checkString(lua, "+.INf")); + + ASSERT_TRUE(checkBool(lua, "true", true)); + ASSERT_TRUE(checkBool(lua, "false", false)); + ASSERT_TRUE(checkBool(lua, "True", true)); + ASSERT_TRUE(checkBool(lua, "False", false)); + ASSERT_TRUE(checkBool(lua, "TRUE", true)); + ASSERT_TRUE(checkBool(lua, "FALSE", false)); + ASSERT_TRUE(checkString(lua, "y")); + ASSERT_TRUE(checkString(lua, "n")); + ASSERT_TRUE(checkString(lua, "On")); + ASSERT_TRUE(checkString(lua, "Off")); + ASSERT_TRUE(checkString(lua, "YES")); + ASSERT_TRUE(checkString(lua, "NO")); + ASSERT_TRUE(checkString(lua, "TrUe")); + ASSERT_TRUE(checkString(lua, "FaLsE")); + ASSERT_TRUE(checkString(lua, "'true'", "true")); + } + + TEST(LuaUtilYamlLoader, DepthLimit) + { + sol::state lua; + + const std::string input = R"( + array1: &array1_alias + [ + <: *array1_alias, + foo + ] + )"; + + bool depthExceptionThrown = false; + try + { + YAML::Node root = YAML::Load(input); + sol::object result = LuaUtil::YamlLoader::load(input, lua); + } + catch (const std::runtime_error& e) + { + ASSERT_EQ(std::string(e.what()), "Maximum layers depth exceeded, probably caused by a circular reference"); + depthExceptionThrown = true; + } + + ASSERT_TRUE(depthExceptionThrown); + } + + TEST(LuaUtilYamlLoader, Collections) + { + sol::state lua; + + sol::object map = LuaUtil::YamlLoader::load("{ x: , y: 2, 4: 5 }", lua); + ASSERT_EQ(map.as()["x"], sol::nil); + ASSERT_EQ(map.as()["y"], 2); + ASSERT_EQ(map.as()[4], 5); + + sol::object array = LuaUtil::YamlLoader::load("[ 3, 4 ]", lua); + ASSERT_EQ(array.as()[1], 3); + + sol::object emptyTable = LuaUtil::YamlLoader::load("{}", lua); + ASSERT_TRUE(emptyTable.as().empty()); + + sol::object emptyArray = LuaUtil::YamlLoader::load("[]", lua); + ASSERT_TRUE(emptyArray.as().empty()); + + ASSERT_THROW(LuaUtil::YamlLoader::load("{ null: 1 }", lua), std::runtime_error); + ASSERT_THROW(LuaUtil::YamlLoader::load("{ .nan: 1 }", lua), std::runtime_error); + + const std::string scalarArrayInput = R"( + - First Scalar + - 1 + - true)"; + + sol::object scalarArray = LuaUtil::YamlLoader::load(scalarArrayInput, lua); + ASSERT_EQ(scalarArray.as()[1], std::string("First Scalar")); + ASSERT_EQ(scalarArray.as()[2], 1); + ASSERT_EQ(scalarArray.as()[3], true); + + const std::string scalarMapWithCommentsInput = R"( + string: 'str' # String value + integer: 65 # Integer value + float: 0.278 # Float value + bool: false # Boolean value)"; + + sol::object scalarMapWithComments = LuaUtil::YamlLoader::load(scalarMapWithCommentsInput, lua); + ASSERT_EQ(scalarMapWithComments.as()["string"], std::string("str")); + ASSERT_EQ(scalarMapWithComments.as()["integer"], 65); + ASSERT_EQ(scalarMapWithComments.as()["float"], 0.278); + ASSERT_EQ(scalarMapWithComments.as()["bool"], false); + + const std::string mapOfArraysInput = R"( + x: + - 2 + - 7 + - true + y: + - aaa + - false + - 1)"; + + sol::object mapOfArrays = LuaUtil::YamlLoader::load(mapOfArraysInput, lua); + ASSERT_EQ(mapOfArrays.as()["x"][3], true); + ASSERT_EQ(mapOfArrays.as()["y"][1], std::string("aaa")); + + const std::string arrayOfMapsInput = R"( + - + name: Name1 + hr: 65 + avg: 0.278 + - + name: Name2 + hr: 63 + avg: 0.288)"; + + sol::object arrayOfMaps = LuaUtil::YamlLoader::load(arrayOfMapsInput, lua); + ASSERT_EQ(arrayOfMaps.as()[1]["avg"], 0.278); + ASSERT_EQ(arrayOfMaps.as()[2]["name"], std::string("Name2")); + + const std::string arrayOfArraysInput = R"( + - [Name1, 65, 0.278] + - [Name2 , 63, 0.288])"; + + sol::object arrayOfArrays = LuaUtil::YamlLoader::load(arrayOfArraysInput, lua); + ASSERT_EQ(arrayOfArrays.as()[1][2], 65); + ASSERT_EQ(arrayOfArrays.as()[2][1], std::string("Name2")); + + const std::string mapOfMapsInput = R"( + Name1: {hr: 65, avg: 0.278} + Name2 : { + hr: 63, + avg: 0.288, + })"; + + sol::object mapOfMaps = LuaUtil::YamlLoader::load(mapOfMapsInput, lua); + ASSERT_EQ(mapOfMaps.as()["Name1"]["hr"], 65); + ASSERT_EQ(mapOfMaps.as()["Name2"]["avg"], 0.288); + } + + TEST(LuaUtilYamlLoader, Structures) + { + sol::state lua; + + const std::string twoDocumentsInput + = "---\n" + " - First Scalar\n" + " - 2\n" + " - true\n" + "\n" + "---\n" + " - Second Scalar\n" + " - 3\n" + " - false"; + + sol::object twoDocuments = LuaUtil::YamlLoader::load(twoDocumentsInput, lua); + ASSERT_EQ(twoDocuments.as()[1][1], std::string("First Scalar")); + ASSERT_EQ(twoDocuments.as()[2][3], false); + + const std::string anchorInput = R"(--- + x: + - Name1 + # Following node labeled as "a" + - &a Value1 + y: + - *a # Subsequent occurrence + - Name2)"; + + sol::object anchor = LuaUtil::YamlLoader::load(anchorInput, lua); + ASSERT_EQ(anchor.as()["y"][1], std::string("Value1")); + + const std::string compoundKeyInput = R"( + ? - String1 + - String2 + : - 1 + + ? [ String3, + String4 ] + : [ 2, 3, 4 ])"; + + ASSERT_THROW(LuaUtil::YamlLoader::load(compoundKeyInput, lua), std::runtime_error); + + const std::string compactNestedMappingInput = R"( + - item : Item1 + quantity: 2 + - item : Item2 + quantity: 4 + - item : Item3 + quantity: 11)"; + + sol::object compactNestedMapping = LuaUtil::YamlLoader::load(compactNestedMappingInput, lua); + ASSERT_EQ(compactNestedMapping.as()[2]["quantity"], 4); + } + + TEST(LuaUtilYamlLoader, Scalars) + { + sol::state lua; + + const std::string literalScalarInput = R"(--- | + a + b + c)"; + + ASSERT_TRUE(checkString(lua, literalScalarInput, "a\nb\nc")); + + const std::string foldedScalarInput = R"(--- > + a + b + c)"; + + ASSERT_TRUE(checkString(lua, foldedScalarInput, "a b c")); + + const std::string multiLinePlanarScalarsInput = R"( + plain: + This unquoted scalar + spans many lines. + + quoted: "So does this + quoted scalar.\n")"; + + sol::object multiLinePlanarScalars = LuaUtil::YamlLoader::load(multiLinePlanarScalarsInput, lua); + ASSERT_TRUE( + multiLinePlanarScalars.as()["plain"] == std::string("This unquoted scalar spans many lines.")); + ASSERT_TRUE(multiLinePlanarScalars.as()["quoted"] == std::string("So does this quoted scalar.\n")); + } +} diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 01efcd7c05..f593e0f0f2 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -65,7 +65,7 @@ list(APPEND COMPONENT_FILES "${OpenMW_BINARY_DIR}/${OSG_PLUGIN_CHECKER_CPP_FILE} add_component_dir (lua luastate scriptscontainer asyncpackage utilpackage serialization configuration l10n storage utf8 - shapes/box inputactions + shapes/box inputactions yamlloader ) add_component_dir (l10n diff --git a/components/lua/yamlloader.cpp b/components/lua/yamlloader.cpp new file mode 100644 index 0000000000..14553cfac4 --- /dev/null +++ b/components/lua/yamlloader.cpp @@ -0,0 +1,241 @@ +#include "yamlloader.hpp" + +#include +#include + +#include +#include + +namespace LuaUtil +{ + namespace + { + constexpr uint64_t maxDepth = 250; + } + + sol::object YamlLoader::load(const std::string& input, const sol::state_view& lua) + { + std::vector rootNodes = YAML::LoadAll(input); + return LuaUtil::YamlLoader::load(rootNodes, lua); + } + + sol::object YamlLoader::load(std::istream& input, const sol::state_view& lua) + { + std::vector rootNodes = YAML::LoadAll(input); + return load(rootNodes, lua); + } + + sol::object YamlLoader::load(const std::vector rootNodes, const sol::state_view& lua) + { + if (rootNodes.empty()) + return sol::nil; + + if (rootNodes.size() == 1) + return getNode(rootNodes[0], lua, 0); + + sol::table documentsTable(lua, sol::create); + for (const auto& root : rootNodes) + { + documentsTable.add(getNode(root, lua, 1)); + } + + return documentsTable; + } + + sol::object YamlLoader::getNode(const YAML::Node& node, const sol::state_view& lua, uint64_t depth) + { + if (depth >= maxDepth) + throw std::runtime_error("Maximum layers depth exceeded, probably caused by a circular reference"); + + ++depth; + + if (node.IsMap()) + return getMap(node, lua, depth); + else if (node.IsSequence()) + return getArray(node, lua, depth); + else if (node.IsScalar()) + return getScalar(node, lua); + else if (node.IsNull()) + return sol::nil; + + nodeError(node, "An unknown YAML node encountered"); + } + + sol::table YamlLoader::getMap(const YAML::Node& node, const sol::state_view& lua, uint64_t depth) + { + sol::table childTable(lua, sol::create); + + for (const auto& pair : node) + { + if (pair.first.IsMap()) + nodeError(pair.first, "Only scalar nodes can be used as keys, encountered map instead"); + if (pair.first.IsSequence()) + nodeError(pair.first, "Only scalar nodes can be used as keys, encountered array instead"); + if (pair.first.IsNull()) + nodeError(pair.first, "Only scalar nodes can be used as keys, encountered null instead"); + + auto key = getNode(pair.first, lua, depth); + if (key.get_type() == sol::type::number && std::isnan(key.as())) + nodeError(pair.first, "Only scalar nodes can be used as keys, encountered nan instead"); + + childTable[key] = getNode(pair.second, lua, depth); + } + + return childTable; + } + + sol::table YamlLoader::getArray(const YAML::Node& node, const sol::state_view& lua, uint64_t depth) + { + sol::table childTable(lua, sol::create); + + for (const auto& child : node) + { + childTable.add(getNode(child, lua, depth)); + } + + return childTable; + } + + YamlLoader::ScalarType YamlLoader::getScalarType(const YAML::Node& node) + { + const auto& tag = node.Tag(); + const auto& value = node.Scalar(); + if (tag == "!") + return ScalarType::String; + + // Note that YAML allows to explicitely specify a scalar type via tag (e.g. "!!bool"), but it makes no + // sense in Lua: + // 1. Both integers and floats use the "number" type prior to Lua 5.3 + // 2. Strings can be quoted, which is more readable than "!!str" + // 3. Most of possible conversions are invalid or their result is unclear + // So ignore this feature for now. + if (tag != "?") + nodeError(node, "An invalid tag'" + tag + "' encountered"); + + if (value.empty()) + return ScalarType::Null; + + // Resolve type according to YAML 1.2 Core Schema (see https://yaml.org/spec/1.2.2/#103-core-schema) + static const std::regex boolRegex("true|True|TRUE|false|False|FALSE", std::regex_constants::extended); + if (std::regex_match(node.Scalar(), boolRegex)) + return ScalarType::Boolean; + + static const std::regex decimalRegex("[-+]?[0-9]+", std::regex_constants::extended); + if (std::regex_match(node.Scalar(), decimalRegex)) + return ScalarType::Decimal; + + static const std::regex floatRegex( + "[-+]?([.][0-9]+|[0-9]+([.][0-9]*)?)([eE][-+]?[0-9]+)?", std::regex_constants::extended); + if (std::regex_match(node.Scalar(), floatRegex)) + return ScalarType::Float; + + static const std::regex octalRegex("0o[0-7]+", std::regex_constants::extended); + if (std::regex_match(node.Scalar(), octalRegex)) + return ScalarType::Octal; + + static const std::regex hexdecimalRegex("0x[0-9a-fA-F]+", std::regex_constants::extended); + if (std::regex_match(node.Scalar(), hexdecimalRegex)) + return ScalarType::Hexadecimal; + + static const std::regex infinityRegex("[-+]?([.]inf|[.]Inf|[.]INF)", std::regex_constants::extended); + if (std::regex_match(node.Scalar(), infinityRegex)) + return ScalarType::Infinity; + + static const std::regex nanRegex("[.]nan|[.]NaN|[.]NAN", std::regex_constants::extended); + if (std::regex_match(node.Scalar(), nanRegex)) + return ScalarType::NotNumber; + + static const std::regex nullRegex("null|Null|NULL|~", std::regex_constants::extended); + if (std::regex_match(node.Scalar(), nullRegex)) + return ScalarType::Null; + + return ScalarType::String; + } + + sol::object YamlLoader::getScalar(const YAML::Node& node, const sol::state_view& lua) + { + auto type = getScalarType(node); + const auto& value = node.Scalar(); + + switch (type) + { + case ScalarType::Null: + return sol::nil; + case ScalarType::String: + return sol::make_object(lua, value); + case ScalarType::NotNumber: + return sol::make_object(lua, std::nan("")); + case ScalarType::Infinity: + { + if (!value.empty() && value[0] == '-') + return sol::make_object(lua, -std::numeric_limits::infinity()); + + return sol::make_object(lua, std::numeric_limits::infinity()); + } + case ScalarType::Boolean: + { + if (Misc::StringUtils::lowerCase(value) == "true") + return sol::make_object(lua, true); + + if (Misc::StringUtils::lowerCase(value) == "false") + return sol::make_object(lua, false); + + nodeError(node, "Can not read a boolean value '" + value + "'"); + } + case ScalarType::Decimal: + { + int offset = 0; + + // std::from_chars does not support "+" sign + if (!value.empty() && value[0] == '+') + ++offset; + + int result = 0; + const auto status = std::from_chars(value.data() + offset, value.data() + value.size(), result); + if (status.ec == std::errc()) + return sol::make_object(lua, result); + + nodeError(node, "Can not read a decimal value '" + value + "'"); + } + case ScalarType::Float: + { + // Not all compilers support std::from_chars for floats + double result = 0.0; + bool success = YAML::convert::decode(node, result); + if (success) + return sol::make_object(lua, result); + + nodeError(node, "Can not read a float value '" + value + "'"); + } + case ScalarType::Hexadecimal: + { + int result = 0; + const auto status = std::from_chars(value.data() + 2, value.data() + value.size(), result, 16); + if (status.ec == std::errc()) + return sol::make_object(lua, result); + + nodeError(node, "Can not read a hexadecimal value '" + value + "'"); + } + case ScalarType::Octal: + { + int result = 0; + const auto status = std::from_chars(value.data() + 2, value.data() + value.size(), result, 8); + if (status.ec == std::errc()) + return sol::make_object(lua, result); + + nodeError(node, "Can not read an octal value '" + value + "'"); + } + default: + nodeError(node, "An unknown scalar '" + value + "' encountered"); + } + } + + [[noreturn]] void YamlLoader::nodeError(const YAML::Node& node, const std::string& message) + { + const auto& mark = node.Mark(); + std::string error = Misc::StringUtils::format( + " at line=%d column=%d position=%d", mark.line + 1, mark.column + 1, mark.pos + 1); + throw std::runtime_error(message + error); + } + +} diff --git a/components/lua/yamlloader.hpp b/components/lua/yamlloader.hpp new file mode 100644 index 0000000000..1ca95223cd --- /dev/null +++ b/components/lua/yamlloader.hpp @@ -0,0 +1,50 @@ +#ifndef COMPONENTS_LUA_YAMLLOADER_H +#define COMPONENTS_LUA_YAMLLOADER_H + +#include +#include +#include +#include + +namespace LuaUtil +{ + + class YamlLoader + { + public: + static sol::object load(const std::string& input, const sol::state_view& lua); + + static sol::object load(std::istream& input, const sol::state_view& lua); + + private: + enum class ScalarType + { + Boolean, + Decimal, + Float, + Hexadecimal, + Infinity, + NotNumber, + Null, + Octal, + String + }; + + static sol::object load(const std::vector rootNodes, const sol::state_view& lua); + + static sol::object getNode(const YAML::Node& node, const sol::state_view& lua, uint64_t depth); + + static sol::table getMap(const YAML::Node& node, const sol::state_view& lua, uint64_t depth); + + static sol::table getArray(const YAML::Node& node, const sol::state_view& lua, uint64_t depth); + + static ScalarType getScalarType(const YAML::Node& node); + + static sol::object getScalar(const YAML::Node& node, const sol::state_view& lua); + + [[noreturn]] static void nodeError(const YAML::Node& node, const std::string& message); + }; + +} + +#endif // COMPONENTS_LUA_YAMLLOADER_H diff --git a/docs/source/reference/lua-scripting/api.rst b/docs/source/reference/lua-scripting/api.rst index 0b700d46a3..fb354a10a7 100644 --- a/docs/source/reference/lua-scripting/api.rst +++ b/docs/source/reference/lua-scripting/api.rst @@ -19,6 +19,7 @@ Lua API reference openmw_animation openmw_async openmw_vfs + openmw_markup openmw_world openmw_self openmw_nearby diff --git a/docs/source/reference/lua-scripting/openmw_markup.rst b/docs/source/reference/lua-scripting/openmw_markup.rst new file mode 100644 index 0000000000..b37afec88f --- /dev/null +++ b/docs/source/reference/lua-scripting/openmw_markup.rst @@ -0,0 +1,7 @@ +Package openmw.markup +===================== + +.. include:: version.rst + +.. raw:: html + :file: generated_html/openmw_markup.html diff --git a/docs/source/reference/lua-scripting/tables/packages.rst b/docs/source/reference/lua-scripting/tables/packages.rst index 247bd7eacc..fd82608aed 100644 --- a/docs/source/reference/lua-scripting/tables/packages.rst +++ b/docs/source/reference/lua-scripting/tables/packages.rst @@ -19,6 +19,8 @@ +------------------------------------------------------------+--------------------+---------------------------------------------------------------+ |:ref:`openmw.vfs ` | everywhere | | Read-only access to data directories via VFS. | +------------------------------------------------------------+--------------------+---------------------------------------------------------------+ +|:ref:`openmw.markup ` | everywhere | | API to work with markup languages. | ++------------------------------------------------------------+--------------------+---------------------------------------------------------------+ |:ref:`openmw.world ` | by global scripts | | Read-write access to the game world. | +------------------------------------------------------------+--------------------+---------------------------------------------------------------+ |:ref:`openmw.self ` | by local scripts | | Full access to the object the script is attached to. | diff --git a/files/data/scripts/omw/console/global.lua b/files/data/scripts/omw/console/global.lua index bba0cbc7b3..d1d5ae423a 100644 --- a/files/data/scripts/omw/console/global.lua +++ b/files/data/scripts/omw/console/global.lua @@ -23,6 +23,7 @@ local env = { core = require('openmw.core'), types = require('openmw.types'), vfs = require('openmw.vfs'), + markup = require('openmw.markup'), async = require('openmw.async'), world = require('openmw.world'), aux_util = require('openmw_aux.util'), diff --git a/files/data/scripts/omw/console/local.lua b/files/data/scripts/omw/console/local.lua index 6962b9e798..1acd18df0c 100644 --- a/files/data/scripts/omw/console/local.lua +++ b/files/data/scripts/omw/console/local.lua @@ -25,6 +25,7 @@ local env = { core = require('openmw.core'), types = require('openmw.types'), vfs = require('openmw.vfs'), + markup = require('openmw.markup'), async = require('openmw.async'), nearby = require('openmw.nearby'), self = require('openmw.self'), diff --git a/files/data/scripts/omw/console/menu.lua b/files/data/scripts/omw/console/menu.lua index 9d6dbaf1d7..b6851bc646 100644 --- a/files/data/scripts/omw/console/menu.lua +++ b/files/data/scripts/omw/console/menu.lua @@ -47,6 +47,7 @@ local env = { core = require('openmw.core'), storage = require('openmw.storage'), vfs = require('openmw.vfs'), + markup = require('openmw.markup'), ambient = require('openmw.ambient'), async = require('openmw.async'), ui = require('openmw.ui'), diff --git a/files/data/scripts/omw/console/player.lua b/files/data/scripts/omw/console/player.lua index 6d0ee790a9..9d2e372a93 100644 --- a/files/data/scripts/omw/console/player.lua +++ b/files/data/scripts/omw/console/player.lua @@ -72,6 +72,7 @@ local env = { core = require('openmw.core'), types = require('openmw.types'), vfs = require('openmw.vfs'), + markup = require('openmw.markup'), ambient = require('openmw.ambient'), async = require('openmw.async'), nearby = require('openmw.nearby'), diff --git a/files/lua_api/CMakeLists.txt b/files/lua_api/CMakeLists.txt index 0b960ea259..526ee90955 100644 --- a/files/lua_api/CMakeLists.txt +++ b/files/lua_api/CMakeLists.txt @@ -17,6 +17,7 @@ set(LUA_API_FILES openmw/debug.lua openmw/input.lua openmw/interfaces.lua + openmw/markup.lua openmw/menu.lua openmw/nearby.lua openmw/postprocessing.lua diff --git a/files/lua_api/openmw/markup.lua b/files/lua_api/openmw/markup.lua new file mode 100644 index 0000000000..c8776281d3 --- /dev/null +++ b/files/lua_api/openmw/markup.lua @@ -0,0 +1,37 @@ +--- +-- `openmw.markup` allows to work with markup languages. +-- @module markup +-- @usage local markup = require('openmw.markup') + + + +--- +-- Convert YAML data to Lua object +-- @function [parent=#markup] decodeYaml +-- @param #string inputData Data to decode. It has such limitations: +-- +-- 1. YAML format of [version 1.2](https://yaml.org/spec/1.2.2) is used. +-- 2. Map keys should be scalar values (strings, booleans, numbers). +-- 3. YAML tag system is not supported. +-- 4. If scalar is quoted, it is treated like a string. +-- Othewise type deduction works according to YAML 1.2 [Core Schema](https://yaml.org/spec/1.2.2/#103-core-schema). +-- 5. Circular dependencies between YAML nodes are not allowed. +-- 6. Lua 5.1 does not have integer numbers - all numeric scalars use a #number type (which use a floating point). +-- 7. Integer scalars numbers values are limited by the "int" range. Use floating point notation for larger number in YAML files. +-- @return #any Lua object (can be table or scalar value). +-- @usage local result = markup.decodeYaml('{ "x": 1 }'); +-- -- prints 1 +-- print(result["x"]) + +--- +-- Load YAML file from VFS to Lua object. Conventions are the same as in @{#markup.decodeYaml}. +-- @function [parent=#markup] loadYaml +-- @param #string fileName YAML file path in VFS. +-- @return #any Lua object (can be table or scalar value). +-- @usage -- file contains '{ "x": 1 }' data +-- local result = markup.loadYaml('test.yaml'); +-- -- prints 1 +-- print(result["x"]) + + +return nil