Use a separate instance of Lua i18n for every context

lua_on_mac
Petr Mikheev 2 years ago
parent f91a5499d3
commit 0f246e7365

@ -737,7 +737,7 @@ void OMW::Engine::prepareEngine (Settings::Manager & settings)
mViewer->addEventHandler(mScreenCaptureHandler);
mLuaManager = new MWLua::LuaManager(mVFS.get());
mLuaManager = new MWLua::LuaManager(mVFS.get(), (mResDir / "lua_libs").string());
mEnvironment.setLuaManager(mLuaManager);
// Create input and UI first to set up a bootstrapping environment for

@ -7,6 +7,7 @@ namespace LuaUtil
{
class LuaState;
class UserdataSerializer;
class I18nManager;
}
namespace MWLua
@ -20,6 +21,7 @@ namespace MWLua
LuaManager* mLuaManager;
LuaUtil::LuaState* mLua;
LuaUtil::UserdataSerializer* mSerializer;
LuaUtil::I18nManager* mI18n;
WorldView* mWorldView;
LocalEventQueue* mLocalEventQueue;
GlobalEventQueue* mGlobalEventQueue;

@ -1,6 +1,7 @@
#include "luabindings.hpp"
#include <components/lua/luastate.hpp>
#include <components/lua/i18n.hpp>
#include <components/queries/luabindings.hpp>
#include "../mwbase/environment.hpp"
@ -25,7 +26,7 @@ namespace MWLua
{
auto* lua = context.mLua;
sol::table api(lua->sol(), sol::create);
api["API_REVISION"] = 11;
api["API_REVISION"] = 12;
api["quit"] = [lua]()
{
Log(Debug::Warning) << "Quit requested by a Lua script.\n" << lua->debugTraceback();
@ -64,6 +65,7 @@ namespace MWLua
{"CarriedLeft", MWWorld::InventoryStore::Slot_CarriedLeft},
{"Ammunition", MWWorld::InventoryStore::Slot_Ammunition}
}));
api["i18n"] = [i18n=context.mI18n](const std::string& context) { return i18n->getContext(context); };
return LuaUtil::makeReadOnly(api);
}

@ -6,6 +6,8 @@
#include <components/esm/esmwriter.hpp>
#include <components/esm/luascripts.hpp>
#include <components/settings/settings.hpp>
#include <components/lua/utilpackage.hpp>
#include "../mwbase/windowmanager.hpp"
@ -20,9 +22,10 @@
namespace MWLua
{
LuaManager::LuaManager(const VFS::Manager* vfs) : mLua(vfs, &mConfiguration)
LuaManager::LuaManager(const VFS::Manager* vfs, const std::string& libsDir) : mLua(vfs, &mConfiguration), mI18n(vfs, &mLua)
{
Log(Debug::Info) << "Lua version: " << LuaUtil::getLuaVersion();
mLua.addInternalLibSearchPath(libsDir);
mGlobalSerializer = createUserdataSerializer(false, mWorldView.getObjectRegistry());
mLocalSerializer = createUserdataSerializer(true, mWorldView.getObjectRegistry());
@ -46,6 +49,7 @@ namespace MWLua
context.mIsGlobal = true;
context.mLuaManager = this;
context.mLua = &mLua;
context.mI18n = &mI18n;
context.mWorldView = &mWorldView;
context.mLocalEventQueue = &mLocalEvents;
context.mGlobalEventQueue = &mGlobalEvents;
@ -55,6 +59,11 @@ namespace MWLua
localContext.mIsGlobal = false;
localContext.mSerializer = mLocalSerializer.get();
mI18n.init();
std::vector<std::string> preferredLanguages;
Misc::StringUtils::split(Settings::Manager::getString("i18n preferred languages", "Lua"), preferredLanguages, ", ");
mI18n.setPreferredLanguages(preferredLanguages);
initObjectBindingsForGlobalScripts(context);
initCellBindingsForGlobalScripts(context);
initObjectBindingsForLocalScripts(localContext);

@ -5,6 +5,7 @@
#include <set>
#include <components/lua/luastate.hpp>
#include <components/lua/i18n.hpp>
#include "../mwbase/luamanager.hpp"
@ -22,7 +23,7 @@ namespace MWLua
class LuaManager : public MWBase::LuaManager
{
public:
LuaManager(const VFS::Manager* vfs);
LuaManager(const VFS::Manager* vfs, const std::string& libsDir);
// Called by engine.cpp when the environment is fully initialized.
void init();
@ -91,6 +92,7 @@ namespace MWLua
bool mGlobalScriptsStarted = false;
LuaUtil::ScriptsConfiguration mConfiguration;
LuaUtil::LuaState mLua;
LuaUtil::I18nManager mI18n;
sol::table mNearbyPackage;
sol::table mUserInterfacePackage;
sol::table mCameraPackage;

@ -23,6 +23,7 @@ if (GTEST_FOUND AND GMOCK_FOUND)
lua/test_serialization.cpp
lua/test_querypackage.cpp
lua/test_configuration.cpp
lua/test_i18n.cpp
lua/test_ui_content.cpp

@ -0,0 +1,110 @@
#include "gmock/gmock.h"
#include <gtest/gtest.h>
#include <components/files/fixedpath.hpp>
#include <components/lua/luastate.hpp>
#include <components/lua/i18n.hpp>
#include "testing_util.hpp"
namespace
{
using namespace testing;
TestFile invalidScript("not a script");
TestFile incorrectScript("return { incorrectSection = {}, engineHandlers = { incorrectHandler = function() end } }");
TestFile emptyScript("");
TestFile test1En(R"X(
return {
good_morning = "Good morning.",
you_have_arrows = {
one = "You have one arrow.",
other = "You have %{count} arrows.",
},
}
)X");
TestFile test1De(R"X(
return {
good_morning = "Guten Morgen.",
you_have_arrows = {
one = "Du hast ein Pfeil.",
other = "Du hast %{count} Pfeile.",
},
["Hello %{name}!"] = "Hallo %{name}!",
}
)X");
TestFile test2En(R"X(
return {
good_morning = "Morning!",
you_have_arrows = "Arrows count: %{count}",
}
)X");
TestFile invalidTest2De(R"X(
require('math')
return {}
)X");
struct LuaI18nTest : Test
{
std::unique_ptr<VFS::Manager> mVFS = createTestVFS({
{"i18n/Test1/en.lua", &test1En},
{"i18n/Test1/de.lua", &test1De},
{"i18n/Test2/en.lua", &test2En},
{"i18n/Test2/de.lua", &invalidTest2De},
});
LuaUtil::ScriptsConfiguration mCfg;
std::string mLibsPath = (Files::TargetPathType("openmw_test_suite").getLocalPath() / "resources" / "lua_libs").string();
};
TEST_F(LuaI18nTest, I18n)
{
internal::CaptureStdout();
LuaUtil::LuaState lua{mVFS.get(), &mCfg};
sol::state& l = lua.sol();
LuaUtil::I18nManager i18n(mVFS.get(), &lua);
lua.addInternalLibSearchPath(mLibsPath);
i18n.init();
i18n.setPreferredLanguages({"de", "en"});
EXPECT_THAT(internal::GetCapturedStdout(), "I18n preferred languages: de en\n");
internal::CaptureStdout();
l["t1"] = i18n.getContext("Test1");
EXPECT_THAT(internal::GetCapturedStdout(), "Language file \"i18n/Test1/de.lua\" is enabled\n");
internal::CaptureStdout();
l["t2"] = i18n.getContext("Test2");
{
std::string output = internal::GetCapturedStdout();
EXPECT_THAT(output, HasSubstr("Can not load i18n/Test2/de.lua"));
EXPECT_THAT(output, HasSubstr("Language file \"i18n/Test2/en.lua\" is enabled"));
}
EXPECT_EQ(get<std::string>(l, "t1('good_morning')"), "Guten Morgen.");
EXPECT_EQ(get<std::string>(l, "t1('you_have_arrows', {count=1})"), "Du hast ein Pfeil.");
EXPECT_EQ(get<std::string>(l, "t1('you_have_arrows', {count=5})"), "Du hast 5 Pfeile.");
EXPECT_EQ(get<std::string>(l, "t1('Hello %{name}!', {name='World'})"), "Hallo World!");
EXPECT_EQ(get<std::string>(l, "t2('good_morning')"), "Morning!");
EXPECT_EQ(get<std::string>(l, "t2('you_have_arrows', {count=3})"), "Arrows count: 3");
internal::CaptureStdout();
i18n.setPreferredLanguages({"en", "de"});
EXPECT_THAT(internal::GetCapturedStdout(),
"I18n preferred languages: en de\n"
"Language file \"i18n/Test1/en.lua\" is enabled\n"
"Language file \"i18n/Test2/en.lua\" is enabled\n");
EXPECT_EQ(get<std::string>(l, "t1('good_morning')"), "Good morning.");
EXPECT_EQ(get<std::string>(l, "t1('you_have_arrows', {count=1})"), "You have one arrow.");
EXPECT_EQ(get<std::string>(l, "t1('you_have_arrows', {count=5})"), "You have 5 arrows.");
EXPECT_EQ(get<std::string>(l, "t1('Hello %{name}!', {name='World'})"), "Hello World!");
EXPECT_EQ(get<std::string>(l, "t2('good_morning')"), "Morning!");
EXPECT_EQ(get<std::string>(l, "t2('you_have_arrows', {count=3})"), "Arrows count: 3");
}
}

@ -106,7 +106,7 @@ return {
}
EXPECT_EQ(LuaUtil::call(script["useCounter"]).get<int>(), 45);
EXPECT_ERROR(LuaUtil::call(script["incorrectRequire"]), "Resource 'counter.lua' not found");
EXPECT_ERROR(LuaUtil::call(script["incorrectRequire"]), "module not found: counter");
}
TEST_F(LuaStateTest, ReadOnly)
@ -161,7 +161,7 @@ return {
sol::table script2 = lua.runInNewSandbox("bbb/tests.lua", "", {{"test.api", api2}});
EXPECT_ERROR(LuaUtil::call(script1["sqr"], 3), "Resource 'sqrlib.lua' not found");
EXPECT_ERROR(LuaUtil::call(script1["sqr"], 3), "module not found: sqrlib");
EXPECT_EQ(LuaUtil::call(script2["sqr"], 3).get<int>(), 9);
EXPECT_EQ(LuaUtil::call(script1["apiName"]).get<std::string>(), "api1");

@ -10,12 +10,6 @@ namespace
{
using namespace testing;
template <typename T>
T get(sol::state& lua, std::string luaCode)
{
return lua.safe_script("return " + luaCode).get<T>();
}
std::string getAsString(sol::state& lua, std::string luaCode)
{
return LuaUtil::toString(lua.safe_script("return " + luaCode));

@ -2,6 +2,7 @@
#define LUA_TESTING_UTIL_H
#include <sstream>
#include <sol/sol.hpp>
#include <components/vfs/archive.hpp>
#include <components/vfs/manager.hpp>
@ -9,6 +10,12 @@
namespace
{
template <typename T>
T get(sol::state& lua, const std::string& luaCode)
{
return lua.safe_script("return " + luaCode).get<T>();
}
class TestFile : public VFS::File
{
public:

@ -29,7 +29,7 @@ endif (GIT_CHECKOUT)
# source files
add_component_dir (lua
luastate scriptscontainer utilpackage serialization configuration
luastate scriptscontainer utilpackage serialization configuration i18n
)
add_component_dir (settings
@ -160,7 +160,7 @@ add_component_dir (fallback
add_component_dir (queries
query luabindings
)
add_component_dir (lua_ui
widget widgetlist element layers content
text textedit window

@ -0,0 +1,108 @@
#include "i18n.hpp"
#include <components/debug/debuglog.hpp>
namespace sol
{
template <>
struct is_automagical<LuaUtil::I18nManager::Context> : std::false_type {};
}
namespace LuaUtil
{
void I18nManager::init()
{
mPreferredLanguages.push_back("en");
sol::usertype<Context> ctx = mLua->sol().new_usertype<Context>("I18nContext");
ctx[sol::meta_function::call] = &Context::translate;
try
{
mI18nLoader = mLua->loadInternalLib("i18n");
sol::set_environment(mLua->newInternalLibEnvironment(), mI18nLoader);
}
catch (std::exception& e)
{
Log(Debug::Error) << "LuaUtil::I18nManager initialization failed: " << e.what();
}
}
void I18nManager::setPreferredLanguages(const std::vector<std::string>& langs)
{
{
Log msg(Debug::Info);
msg << "I18n preferred languages:";
for (const std::string& l : langs)
msg << " " << l;
}
mPreferredLanguages = langs;
for (auto& [_, context] : mContexts)
context.updateLang(this);
}
void I18nManager::Context::readLangData(I18nManager* manager, const std::string& lang)
{
std::string path = "i18n/";
path.append(mName);
path.append("/");
path.append(lang);
path.append(".lua");
if (!manager->mVFS->exists(path))
return;
try
{
sol::protected_function dataFn = manager->mLua->loadFromVFS(path);
sol::environment emptyEnv(manager->mLua->sol(), sol::create);
sol::set_environment(emptyEnv, dataFn);
sol::table data = manager->mLua->newTable();
data[lang] = call(dataFn);
call(mI18n["load"], data);
mLoadedLangs[lang] = true;
}
catch (std::exception& e)
{
Log(Debug::Error) << "Can not load " << path << ": " << e.what();
}
}
sol::object I18nManager::Context::translate(std::string_view key, const sol::object& data)
{
sol::object res = call(mI18n["translate"], key, data);
if (res != sol::nil)
return res;
// If not found in a language file - register the key itself as a message.
std::string composedKey = call(mI18n["getLocale"]).get<std::string>();
composedKey.push_back('.');
composedKey.append(key);
call(mI18n["set"], composedKey, key);
return call(mI18n["translate"], key, data);
}
void I18nManager::Context::updateLang(I18nManager* manager)
{
for (const std::string& lang : manager->mPreferredLanguages)
{
if (mLoadedLangs[lang] == sol::nil)
readLangData(manager, lang);
if (mLoadedLangs[lang] != sol::nil)
{
Log(Debug::Verbose) << "Language file \"i18n/" << mName << "/" << lang << ".lua\" is enabled";
call(mI18n["setLocale"], lang);
return;
}
}
Log(Debug::Warning) << "No language files for the preferred languages found in \"i18n/" << mName << "\"";
}
sol::object I18nManager::getContext(const std::string& contextName)
{
if (mI18nLoader == sol::nil)
throw std::runtime_error("LuaUtil::I18nManager is not initialized");
Context ctx{contextName, mLua->newTable(), call(mI18nLoader, "i18n.init")};
ctx.updateLang(this);
mContexts.emplace(contextName, ctx);
return sol::make_object(mLua->sol(), ctx);
}
}

@ -0,0 +1,41 @@
#ifndef COMPONENTS_LUA_I18N_H
#define COMPONENTS_LUA_I18N_H
#include "luastate.hpp"
namespace LuaUtil
{
class I18nManager
{
public:
I18nManager(const VFS::Manager* vfs, LuaState* lua) : mVFS(vfs), mLua(lua) {}
void init();
void setPreferredLanguages(const std::vector<std::string>& langs);
const std::vector<std::string>& getPreferredLanguages() const { return mPreferredLanguages; }
sol::object getContext(const std::string& contextName);
private:
struct Context
{
std::string mName;
sol::table mLoadedLangs;
sol::table mI18n;
void updateLang(I18nManager* manager);
void readLangData(I18nManager* manager, const std::string& lang);
sol::object translate(std::string_view key, const sol::object& data);
};
const VFS::Manager* mVFS;
LuaState* mLua;
sol::object mI18nLoader = sol::nil;
std::vector<std::string> mPreferredLanguages;
std::map<std::string, Context> mContexts;
};
}
#endif // COMPONENTS_LUA_I18N_H

@ -4,17 +4,44 @@
#include <luajit.h>
#endif // NO_LUAJIT
#include <filesystem>
#include <components/debug/debuglog.hpp>
namespace LuaUtil
{
static std::string packageNameToPath(std::string_view packageName)
static std::string packageNameToVfsPath(std::string_view packageName, const VFS::Manager* vfs)
{
std::string res(packageName);
std::replace(res.begin(), res.end(), '.', '/');
res.append(".lua");
return res;
std::string path(packageName);
std::replace(path.begin(), path.end(), '.', '/');
std::string pathWithInit = path + "/init.lua";
path.append(".lua");
if (vfs->exists(path))
return path;
else if (vfs->exists(pathWithInit))
return pathWithInit;
else
throw std::runtime_error("module not found: " + std::string(packageName));
}
static std::string packageNameToPath(std::string_view packageName, const std::vector<std::string>& searchDirs)
{
std::string path(packageName);
std::replace(path.begin(), path.end(), '.', '/');
std::string pathWithInit = path + "/init.lua";
path.append(".lua");
for (const std::string& dir : searchDirs)
{
std::filesystem::path base(dir);
std::filesystem::path p1 = base / path;
if (std::filesystem::exists(p1))
return p1.string();
std::filesystem::path p2 = base / pathWithInit;
if (std::filesystem::exists(p2))
return p2.string();
}
throw std::runtime_error("module not found: " + std::string(packageName));
}
static const std::string safeFunctions[] = {
@ -28,7 +55,7 @@ namespace LuaUtil
sol::lib::string, sol::lib::table, sol::lib::debug);
mLua["math"]["randomseed"](static_cast<unsigned>(std::time(nullptr)));
mLua["math"]["randomseed"] = sol::nil;
mLua["math"]["randomseed"] = []{};
mLua["writeToLog"] = [](std::string_view s) { Log(Debug::Level::Info) << s; };
mLua.script(R"(printToLog = function(name, ...)
@ -105,7 +132,7 @@ namespace LuaUtil
const std::string& path, const std::string& namePrefix,
const std::map<std::string, sol::object>& packages, const sol::object& hiddenData)
{
sol::protected_function script = loadScript(path);
sol::protected_function script = loadScriptAndCache(path);
sol::environment env(mLua, sol::create, mSandboxEnv);
std::string envName = namePrefix + "[" + path + "]:";
@ -122,9 +149,9 @@ namespace LuaUtil
sol::object package = packages[packageName];
if (package == sol::nil)
{
sol::protected_function packageLoader = loadScript(packageNameToPath(packageName));
sol::protected_function packageLoader = loadScriptAndCache(packageNameToVfsPath(packageName, mVFS));
sol::set_environment(env, packageLoader);
package = throwIfError(packageLoader());
package = call(packageLoader, packageName);
if (!package.is<sol::table>())
throw std::runtime_error("Lua package must return a table.");
packages[packageName] = package;
@ -138,6 +165,24 @@ namespace LuaUtil
return call(script);
}
sol::environment LuaState::newInternalLibEnvironment()
{
sol::environment env(mLua, sol::create, mSandboxEnv);
sol::table loaded(mLua, sol::create);
for (const std::string& s : safePackages)
loaded[s] = mSandboxEnv[s];
env["require"] = [this, loaded, env](const std::string& module) mutable
{
if (loaded[module] != sol::nil)
return loaded[module];
sol::protected_function initializer = loadInternalLib(module);
sol::set_environment(env, initializer);
loaded[module] = call(initializer, module);
return loaded[module];
};
return env;
}
sol::protected_function_result LuaState::throwIfError(sol::protected_function_result&& res)
{
if (!res.valid() && static_cast<int>(res.get_type()) == LUA_TSTRING)
@ -146,17 +191,31 @@ namespace LuaUtil
return std::move(res);
}
sol::function LuaState::loadScript(const std::string& path)
sol::function LuaState::loadScriptAndCache(const std::string& path)
{
auto iter = mCompiledScripts.find(path);
if (iter != mCompiledScripts.end())
return mLua.load(iter->second.as_string_view(), path, sol::load_mode::binary);
sol::function res = loadFromVFS(path);
mCompiledScripts[path] = res.dump();
return res;
}
sol::function LuaState::loadFromVFS(const std::string& path)
{
std::string fileContent(std::istreambuf_iterator<char>(*mVFS->get(path)), {});
sol::load_result res = mLua.load(fileContent, path, sol::load_mode::text);
if (!res.valid())
throw std::runtime_error("Lua error: " + res.get<std::string>());
mCompiledScripts[path] = res.get<sol::function>().dump();
return res;
}
sol::function LuaState::loadInternalLib(std::string_view libName)
{
std::string path = packageNameToPath(libName, mLibSearchPaths);
sol::load_result res = mLua.load_file(path, sol::load_mode::text);
if (!res.valid())
throw std::runtime_error("Lua error: " + res.get<std::string>());
return res;
}

@ -76,12 +76,18 @@ namespace LuaUtil
const ScriptsConfiguration& getConfiguration() const { return *mConf; }
// Load internal Lua library. All libraries are loaded in one sandbox and shouldn't be exposed to scripts directly.
void addInternalLibSearchPath(const std::string& path) { mLibSearchPaths.push_back(path); }
sol::function loadInternalLib(std::string_view libName);
sol::function loadFromVFS(const std::string& path);
sol::environment newInternalLibEnvironment();
private:
static sol::protected_function_result throwIfError(sol::protected_function_result&&);
template <typename... Args>
friend sol::protected_function_result call(const sol::protected_function& fn, Args&&... args);
sol::function loadScript(const std::string& path);
sol::function loadScriptAndCache(const std::string& path);
sol::state mLua;
const ScriptsConfiguration* mConf;
@ -89,6 +95,7 @@ namespace LuaUtil
std::map<std::string, sol::bytecode> mCompiledScripts;
std::map<std::string, sol::object> mCommonPackages;
const VFS::Manager* mVFS;
std::vector<std::string> mLibSearchPaths;
};
// Should be used for every call of every Lua function.

@ -27,3 +27,14 @@ Values >1 are not yet supported.
This setting can only be configured by editing the settings configuration file.
i18n preferred languages
------------------------
:Type: string
:Default: en
List of the preferred languages separated by comma.
For example "de,en" means German as the first prority and English as a fallback.
This setting can only be configured by editing the settings configuration file.

@ -37,6 +37,39 @@
-- @function [parent=#core] isWorldPaused
-- @return #boolean
-------------------------------------------------------------------------------
-- Return i18n formatting function for the given context.
-- It is based on `i18n.lua` library.
-- Language files should be stored in VFS as `i18n/<ContextName>/<Lang>.lua`.
-- See https://github.com/kikito/i18n.lua for format details.
-- @function [parent=#core] i18n
-- @param #string context I18n context; recommended to use the name of the mod.
-- @return #function
-- @usage
-- -- DataFiles/i18n/MyMod/en.lua
-- return {
-- good_morning = 'Good morning.',
-- you_have_arrows = {
-- one = 'You have one arrow.',
-- other = 'You have %{count} arrows.',
-- },
-- }
-- @usage
-- -- DataFiles/i18n/MyMod/de.lua
-- return {
-- good_morning = "Guten Morgen.",
-- you_have_arrows = {
-- one = "Du hast ein Pfeil.",
-- other = "Du hast %{count} Pfeile.",
-- },
-- ["Hello %{name}!"] = "Hallo %{name}!",
-- }
-- @usage
-- local myMsg = core.i18n('MyMod')
-- print( myMsg('good_morning') )
-- print( myMsg('you_have_arrows', {count=5}) )
-- print( myMsg('Hello %{name}!', {name='World'}) )
-------------------------------------------------------------------------------
-- @type OBJECT_TYPE

@ -1123,3 +1123,7 @@ lua debug = false
# If zero, Lua scripts are processed in the main thread.
lua num threads = 1
# List of the preferred languages separated by comma.
# For example "de,en" means German as the first prority and English as a fallback.
i18n preferred languages = en

Loading…
Cancel
Save