From 55db95d4cfe5cedb1951fa09ea3c9bc2f0458974 Mon Sep 17 00:00:00 2001 From: Petr Mikheev Date: Tue, 29 Nov 2022 18:16:06 +0100 Subject: [PATCH] Update Lua profiler; add ability to run OpenMW with old LuaJit that doesn't allow custom allocator (Lua profiler will be disabled in this case) --- apps/openmw/mwlua/eventqueue.cpp | 5 +- apps/openmw/mwlua/eventqueue.hpp | 2 +- apps/openmw/mwlua/luamanagerimp.cpp | 71 +++++++--- apps/openmw/mwlua/objectbindings.cpp | 2 +- apps/openmw_test_suite/lua/test_l10n.cpp | 5 +- apps/openmw_test_suite/lua/test_storage.cpp | 4 +- components/lua/l10n.cpp | 3 +- components/lua/l10n.hpp | 2 +- components/lua/luastate.cpp | 144 +++++++++++--------- components/lua/luastate.hpp | 69 +++++++--- components/lua/scriptscontainer.cpp | 18 +-- components/lua/scriptscontainer.hpp | 8 +- components/lua/utilpackage.cpp | 3 +- components/lua/utilpackage.hpp | 3 +- files/settings-default.cfg | 12 +- 15 files changed, 221 insertions(+), 130 deletions(-) diff --git a/apps/openmw/mwlua/eventqueue.cpp b/apps/openmw/mwlua/eventqueue.cpp index cb952e4e3c..8c84d234d3 100644 --- a/apps/openmw/mwlua/eventqueue.cpp +++ b/apps/openmw/mwlua/eventqueue.cpp @@ -20,8 +20,9 @@ namespace MWLua saveLuaBinaryData(esm, event.mEventData); } - void loadEvents(sol::state& lua, ESM::ESMReader& esm, GlobalEventQueue& globalEvents, LocalEventQueue& localEvents, - const std::map& contentFileMapping, const LuaUtil::UserdataSerializer* serializer) + void loadEvents(sol::state_view& lua, ESM::ESMReader& esm, GlobalEventQueue& globalEvents, + LocalEventQueue& localEvents, const std::map& contentFileMapping, + const LuaUtil::UserdataSerializer* serializer) { while (esm.isNextSub("LUAE")) { diff --git a/apps/openmw/mwlua/eventqueue.hpp b/apps/openmw/mwlua/eventqueue.hpp index 273a4c2ad7..8854aa12a5 100644 --- a/apps/openmw/mwlua/eventqueue.hpp +++ b/apps/openmw/mwlua/eventqueue.hpp @@ -35,7 +35,7 @@ namespace MWLua using GlobalEventQueue = std::vector; using LocalEventQueue = std::vector; - void loadEvents(sol::state& lua, ESM::ESMReader& esm, GlobalEventQueue&, LocalEventQueue&, + void loadEvents(sol::state_view& lua, ESM::ESMReader& esm, GlobalEventQueue&, LocalEventQueue&, const std::map& contentFileMapping, const LuaUtil::UserdataSerializer* serializer); void saveEvents(ESM::ESMWriter& esm, const GlobalEventQueue&, const LocalEventQueue&); } diff --git a/apps/openmw/mwlua/luamanagerimp.cpp b/apps/openmw/mwlua/luamanagerimp.cpp index d36bc9f92f..220655fbb0 100644 --- a/apps/openmw/mwlua/luamanagerimp.cpp +++ b/apps/openmw/mwlua/luamanagerimp.cpp @@ -39,6 +39,8 @@ namespace MWLua static LuaUtil::LuaStateSettings createLuaStateSettings() { + if (!Settings::Manager::getBool("lua profiler", "Lua")) + LuaUtil::LuaState::disableProfiler(); return { .mInstructionLimit = Settings::Manager::getUInt64("instruction limit per call", "Lua"), .mMemoryLimit = Settings::Manager::getUInt64("memory limit", "Lua"), .mSmallAllocMaxSize = Settings::Manager::getUInt64("small alloc max size", "Lua"), @@ -147,9 +149,9 @@ namespace MWLua mWorldView.update(); - mGlobalScripts.CPUusageNextFrame(); + mGlobalScripts.statsNextFrame(); for (LocalScripts* scripts : mActiveLocalScripts) - scripts->CPUusageNextFrame(); + scripts->statsNextFrame(); std::vector globalEvents = std::move(mGlobalEvents); std::vector localEvents = std::move(mLocalEvents); @@ -620,17 +622,39 @@ namespace MWLua std::string LuaManager::formatResourceUsageStats() const { + if (!LuaUtil::LuaState::isProfilerEnabled()) + return "Lua profiler is disabled"; + std::stringstream out; + constexpr int nameW = 50; + constexpr int valueW = 12; + + auto outMemSize = [&](int64_t bytes) { + constexpr int64_t limit = 10000; + out << std::right << std::setw(valueW - 3); + if (bytes < limit) + out << bytes << " B "; + else if (bytes < limit * 1024) + out << (bytes / 1024) << " KB"; + else if (bytes < limit * 1024 * 1024) + out << (bytes / (1024 * 1024)) << " MB"; + else + out << (bytes / (1024 * 1024 * 1024)) << " GB"; + }; + static const uint64_t smallAllocSize = Settings::Manager::getUInt64("small alloc max size", "Lua"); - out << "Total memory usage: " << mLua.getTotalMemoryUsage() << "\n"; + out << "Total memory usage:"; + outMemSize(mLua.getTotalMemoryUsage()); + out << "\n"; out << "small alloc max size = " << smallAllocSize << " (section [Lua] in settings.cfg)\n"; out << "Smaller values give more information for the profiler, but increase performance overhead.\n"; - out << " Memory allocations <= " << smallAllocSize << " bytes: " << mLua.getSmallAllocMemoryUsage() - << " (not tracked)\n"; - out << " Memory allocations > " << smallAllocSize - << " bytes: " << mLua.getTotalMemoryUsage() - mLua.getSmallAllocMemoryUsage() << " (see the table below)\n"; - out << "\n"; + out << " Memory allocations <= " << smallAllocSize << " bytes:"; + outMemSize(mLua.getSmallAllocMemoryUsage()); + out << " (not tracked)\n"; + out << " Memory allocations > " << smallAllocSize << " bytes:"; + outMemSize(mLua.getTotalMemoryUsage() - mLua.getSmallAllocMemoryUsage()); + out << " (see the table below)\n\n"; using Stats = LuaUtil::ScriptsContainer::ScriptStats; @@ -653,16 +677,22 @@ namespace MWLua out << "No selected object. Use the in-game console to select an object for detailed profile.\n"; out << "\n"; - constexpr int nameW = 50; - constexpr int valueW = 12; + out << "Legend\n"; + out << " ops: Averaged number of Lua instruction per frame;\n"; + out << " memory: Aggregated size of Lua allocations > " << smallAllocSize << " bytes;\n"; + out << " [all]: Sum over all instances of each script;\n"; + out << " [active]: Sum over all active (i.e. currently in scene) instances of each script;\n"; + out << " [inactive]: Sum over all inactive instances of each script;\n"; + out << " [for selected object]: Only for the object that is selected in the console;\n"; + out << "\n"; out << std::left; out << " " << std::setw(nameW + 2) << "*** Resource usage per script"; out << std::right; - out << std::setw(valueW) << "CPU"; + out << std::setw(valueW) << "ops"; out << std::setw(valueW) << "memory"; out << std::setw(valueW) << "memory"; - out << std::setw(valueW) << "CPU"; + out << std::setw(valueW) << "ops"; out << std::setw(valueW) << "memory"; out << "\n"; out << std::left << " " << std::setw(nameW + 2) << "[name]" << std::right; @@ -681,19 +711,24 @@ namespace MWLua if (mConfiguration[i].mScriptPath.size() > nameW) out << "\n " << std::setw(nameW) << ""; // if path is too long, break line out << std::right; - out << std::setw(valueW) << static_cast(activeStats[i].mCPUusage); - out << std::setw(valueW) << activeStats[i].mMemoryUsage; - out << std::setw(valueW) << mLua.getMemoryUsageByScriptIndex(i) - activeStats[i].mMemoryUsage; + out << std::setw(valueW) << static_cast(activeStats[i].mAvgInstructionCount); + outMemSize(activeStats[i].mMemoryUsage); + outMemSize(mLua.getMemoryUsageByScriptIndex(i) - activeStats[i].mMemoryUsage); if (isGlobal) out << std::setw(valueW * 2) << "NA (global script)"; else if (selectedPtr.isEmpty()) out << std::setw(valueW * 2) << "NA (not selected) "; else if (!selectedScripts || !selectedScripts->hasScript(i)) - out << std::setw(valueW) << "-" << std::setw(valueW) << selectedStats[i].mMemoryUsage; + { + out << std::setw(valueW) << "-"; + outMemSize(selectedStats[i].mMemoryUsage); + } else - out << std::setw(valueW) << static_cast(selectedStats[i].mCPUusage) << std::setw(valueW) - << selectedStats[i].mMemoryUsage; + { + out << std::setw(valueW) << static_cast(selectedStats[i].mAvgInstructionCount); + outMemSize(selectedStats[i].mMemoryUsage); + } out << "\n"; } diff --git a/apps/openmw/mwlua/objectbindings.cpp b/apps/openmw/mwlua/objectbindings.cpp index 2b1d572575..c62c1b7b36 100644 --- a/apps/openmw/mwlua/objectbindings.cpp +++ b/apps/openmw/mwlua/objectbindings.cpp @@ -146,7 +146,7 @@ namespace MWLua void registerObjectList(const std::string& prefix, const Context& context) { using ListT = ObjectList; - sol::state& lua = context.mLua->sol(); + sol::state_view& lua = context.mLua->sol(); ObjectRegistry* registry = context.mWorldView->getObjectRegistry(); sol::usertype listT = lua.new_usertype(prefix + "ObjectList"); listT[sol::meta_function::to_string] diff --git a/apps/openmw_test_suite/lua/test_l10n.cpp b/apps/openmw_test_suite/lua/test_l10n.cpp index 57844dfaec..e5558e5010 100644 --- a/apps/openmw_test_suite/lua/test_l10n.cpp +++ b/apps/openmw_test_suite/lua/test_l10n.cpp @@ -15,7 +15,7 @@ namespace using namespace TestingOpenMW; template - T get(sol::state& lua, const std::string& luaCode) + T get(sol::state_view& lua, const std::string& luaCode) { return lua.safe_script("return " + luaCode).get(); } @@ -83,7 +83,7 @@ you_have_arrows: "Arrows count: {count}" TEST_F(LuaL10nTest, L10n) { LuaUtil::LuaState lua{ mVFS.get(), &mCfg }; - sol::state& l = lua.sol(); + sol::state_view& l = lua.sol(); internal::CaptureStdout(); l10n::Manager l10nManager(mVFS.get()); l10nManager.setPreferredLocales({ "de", "en" }); @@ -164,5 +164,4 @@ you_have_arrows: "Arrows count: {count}" l10nManager.setPreferredLocales({ "en" }); EXPECT_EQ(get(l, "t3('Hello {name}!', {name='World'})"), "Hallo World!"); } - } diff --git a/apps/openmw_test_suite/lua/test_storage.cpp b/apps/openmw_test_suite/lua/test_storage.cpp index 8766b1b975..270012c9b6 100644 --- a/apps/openmw_test_suite/lua/test_storage.cpp +++ b/apps/openmw_test_suite/lua/test_storage.cpp @@ -10,7 +10,7 @@ namespace using namespace testing; template - T get(sol::state& lua, std::string luaCode) + T get(sol::state_view& lua, std::string luaCode) { return lua.safe_script("return " + luaCode).get(); } @@ -19,7 +19,7 @@ namespace { // Note: LuaUtil::Callback can be used only if Lua is initialized via LuaUtil::LuaState LuaUtil::LuaState luaState{ nullptr, nullptr }; - sol::state& mLua = luaState.sol(); + sol::state_view& mLua = luaState.sol(); LuaUtil::LuaStorage::initLuaBindings(mLua); LuaUtil::LuaStorage storage(mLua); diff --git a/components/lua/l10n.cpp b/components/lua/l10n.cpp index d402cf9346..9bfad15ad8 100644 --- a/components/lua/l10n.cpp +++ b/components/lua/l10n.cpp @@ -46,8 +46,9 @@ namespace sol namespace LuaUtil { - sol::function initL10nLoader(sol::state& lua, l10n::Manager* manager) + sol::function initL10nLoader(lua_State* L, l10n::Manager* manager) { + sol::state_view lua(L); sol::usertype ctxDef = lua.new_usertype("L10nContext"); ctxDef[sol::meta_function::call] = [](const L10nContext& ctx, std::string_view key, sol::optional args) { diff --git a/components/lua/l10n.hpp b/components/lua/l10n.hpp index 68cb86050c..1fc3e17747 100644 --- a/components/lua/l10n.hpp +++ b/components/lua/l10n.hpp @@ -10,7 +10,7 @@ namespace l10n namespace LuaUtil { - sol::function initL10nLoader(sol::state& lua, l10n::Manager* manager); + sol::function initL10nLoader(lua_State*, l10n::Manager* manager); } #endif // COMPONENTS_LUA_L10N_H diff --git a/components/lua/luastate.cpp b/components/lua/luastate.cpp index d11ac116ce..3b4f5c9e2f 100644 --- a/components/lua/luastate.cpp +++ b/components/lua/luastate.cpp @@ -52,21 +52,24 @@ namespace LuaUtil "tonumber", "tostring", "type", "unpack", "xpcall", "rawequal", "rawget", "rawset", "setmetatable" }; static const std::string safePackages[] = { "coroutine", "math", "string", "table" }; - static constexpr int64_t countHookStep = 2000; + static constexpr int64_t countHookStep = 1000; + + bool LuaState::sProfilerEnabled = true; void LuaState::countHook(lua_State* L, lua_Debug* ar) { - LuaState* THIS; - (void)lua_getallocf(L, reinterpret_cast(&THIS)); - if (!THIS->mActiveScriptId.mContainer) + LuaState* self; + (void)lua_getallocf(L, reinterpret_cast(&self)); + if (self->mActiveScriptIdStack.empty()) return; - THIS->mActiveScriptId.mContainer->addCPUusage(THIS->mActiveScriptId.mIndex, countHookStep); - THIS->mCurrentCallInstructionCounter += countHookStep; - if (THIS->mSettings.mInstructionLimit > 0 - && THIS->mCurrentCallInstructionCounter > THIS->mSettings.mInstructionLimit) + const ScriptId& activeScript = self->mActiveScriptIdStack.back(); + activeScript.mContainer->addInstructionCount(activeScript.mIndex, countHookStep); + self->mWatchdogInstructionCounter += countHookStep; + if (self->mSettings.mInstructionLimit > 0 + && self->mWatchdogInstructionCounter > self->mSettings.mInstructionLimit) { lua_pushstring(L, - "Lua CPU usage exceeded, probably an infinite loop in a script. " + "Lua instruction count exceeded, probably an infinite loop in a script. " "To change the limit set \"[Lua] instruction limit per call\" in settings.cfg"); lua_error(L); } @@ -74,9 +77,9 @@ namespace LuaUtil void* LuaState::trackingAllocator(void* ud, void* ptr, size_t osize, size_t nsize) { - LuaState* THIS = static_cast(ud); - const uint64_t smallAllocSize = THIS->mSettings.mSmallAllocMaxSize; - const uint64_t memoryLimit = THIS->mSettings.mMemoryLimit; + LuaState* self = static_cast(ud); + const uint64_t smallAllocSize = self->mSettings.mSmallAllocMaxSize; + const uint64_t memoryLimit = self->mSettings.mMemoryLimit; if (!ptr) osize = 0; @@ -90,14 +93,14 @@ namespace LuaUtil else bigAllocDelta += nsize; - if (bigAllocDelta > 0 && memoryLimit > 0 && THIS->mTotalMemoryUsage + nsize - osize > memoryLimit) + if (bigAllocDelta > 0 && memoryLimit > 0 && self->mTotalMemoryUsage + nsize - osize > memoryLimit) { Log(Debug::Error) << "Lua realloc " << osize << "->" << nsize << " is blocked because Lua memory limit (configurable in settings.cfg) is exceeded"; return nullptr; } - THIS->mTotalMemoryUsage += smallAllocDelta + bigAllocDelta; - THIS->mSmallAllocMemoryUsage += smallAllocDelta; + self->mTotalMemoryUsage += smallAllocDelta + bigAllocDelta; + self->mSmallAllocMemoryUsage += smallAllocDelta; void* newPtr = nullptr; if (nsize == 0) @@ -107,59 +110,83 @@ namespace LuaUtil if (bigAllocDelta != 0) { - auto it = osize > smallAllocSize ? THIS->mBigAllocOwners.find(ptr) : THIS->mBigAllocOwners.end(); + auto it = osize > smallAllocSize ? self->mBigAllocOwners.find(ptr) : self->mBigAllocOwners.end(); ScriptId id; - if (it != THIS->mBigAllocOwners.end()) + if (it != self->mBigAllocOwners.end()) { if (it->second.mContainer) id = ScriptId{ *it->second.mContainer, it->second.mScriptIndex }; if (ptr != newPtr || nsize <= smallAllocSize) - THIS->mBigAllocOwners.erase(it); + self->mBigAllocOwners.erase(it); } else if (bigAllocDelta > 0) { - id = THIS->mActiveScriptId; + if (!self->mActiveScriptIdStack.empty()) + id = self->mActiveScriptIdStack.back(); bigAllocDelta = nsize; } if (id.mContainer) { - if (static_cast(id.mIndex) >= THIS->mMemoryUsage.size()) - THIS->mMemoryUsage.resize(id.mIndex + 1); - THIS->mMemoryUsage[id.mIndex] += bigAllocDelta; + if (static_cast(id.mIndex) >= self->mMemoryUsage.size()) + self->mMemoryUsage.resize(id.mIndex + 1); + self->mMemoryUsage[id.mIndex] += bigAllocDelta; id.mContainer->addMemoryUsage(id.mIndex, bigAllocDelta); if (newPtr && nsize > smallAllocSize) - THIS->mBigAllocOwners.emplace(newPtr, AllocOwner{ id.mContainer->mThis, id.mIndex }); + self->mBigAllocOwners.emplace(newPtr, AllocOwner{ id.mContainer->mThis, id.mIndex }); } } return newPtr; } + lua_State* LuaState::createLuaRuntime(LuaState* luaState) + { + if (sProfilerEnabled) + { + Log(Debug::Info) << "Initializing LuaUtil::LuaState with profiler"; + lua_State* L = lua_newstate(&trackingAllocator, luaState); + if (L) + return L; + else + { + sProfilerEnabled = false; + Log(Debug::Error) + << "Failed to initialize LuaUtil::LuaState with custom allocator; disabling Lua profiler"; + } + } + Log(Debug::Info) << "Initializing LuaUtil::LuaState without profiler"; + lua_State* L = luaL_newstate(); + if (!L) + throw std::runtime_error("Can't create Lua runtime"); + return L; + } + LuaState::LuaState(const VFS::Manager* vfs, const ScriptsConfiguration* conf, const LuaStateSettings& settings) : mSettings(settings) - , mLua(sol::default_at_panic, &trackingAllocator, this) + , mLuaHolder(createLuaRuntime(this)) + , mSol(mLuaHolder.get()) , mConf(conf) , mVFS(vfs) { - lua_sethook(mLua.lua_state(), &countHook, LUA_MASKCOUNT, countHookStep); - Log(Debug::Verbose) << "Initializing LuaUtil::LuaState"; + if (sProfilerEnabled) + lua_sethook(mLuaHolder.get(), &countHook, LUA_MASKCOUNT, countHookStep); - mLua.open_libraries(sol::lib::base, sol::lib::coroutine, sol::lib::math, sol::lib::bit32, sol::lib::string, + mSol.open_libraries(sol::lib::base, sol::lib::coroutine, sol::lib::math, sol::lib::bit32, sol::lib::string, sol::lib::table, sol::lib::os, sol::lib::debug); - mLua["math"]["randomseed"](static_cast(std::time(nullptr))); - mLua["math"]["randomseed"] = [] {}; + mSol["math"]["randomseed"](static_cast(std::time(nullptr))); + mSol["math"]["randomseed"] = [] {}; - mLua["writeToLog"] = [](std::string_view s) { Log(Debug::Level::Info) << s; }; + mSol["writeToLog"] = [](std::string_view s) { Log(Debug::Level::Info) << s; }; // Some fixes for compatibility between different Lua versions - if (mLua["unpack"] == sol::nil) - mLua["unpack"] = mLua["table"]["unpack"]; - else if (mLua["table"]["unpack"] == sol::nil) - mLua["table"]["unpack"] = mLua["unpack"]; + if (mSol["unpack"] == sol::nil) + mSol["unpack"] = mSol["table"]["unpack"]; + else if (mSol["table"]["unpack"] == sol::nil) + mSol["table"]["unpack"] = mSol["unpack"]; if (LUA_VERSION_NUM <= 501) { - mLua.script(R"( + mSol.script(R"( local _pairs = pairs local _ipairs = ipairs pairs = function(v) return (rawget(getmetatable(v) or {}, '__pairs') or _pairs)(v) end @@ -167,7 +194,7 @@ namespace LuaUtil )"); } - mLua.script(R"( + mSol.script(R"( local printToLog = function(...) local strs = {} for i = 1, select('#', ...) do @@ -212,31 +239,24 @@ namespace LuaUtil end )"); - mSandboxEnv = sol::table(mLua, sol::create); - mSandboxEnv["_VERSION"] = mLua["_VERSION"]; + mSandboxEnv = sol::table(mSol, sol::create); + mSandboxEnv["_VERSION"] = mSol["_VERSION"]; for (const std::string& s : safeFunctions) { - if (mLua[s] == sol::nil) + if (mSol[s] == sol::nil) throw std::logic_error("Lua function not found: " + s); - mSandboxEnv[s] = mLua[s]; + mSandboxEnv[s] = mSol[s]; } for (const std::string& s : safePackages) { - if (mLua[s] == sol::nil) + if (mSol[s] == sol::nil) throw std::logic_error("Lua package not found: " + s); - mCommonPackages[s] = mSandboxEnv[s] = makeReadOnly(mLua[s]); + mCommonPackages[s] = mSandboxEnv[s] = makeReadOnly(mSol[s]); } - mSandboxEnv["getmetatable"] = mLua["getSafeMetatable"]; + mSandboxEnv["getmetatable"] = mSol["getSafeMetatable"]; mCommonPackages["os"] = mSandboxEnv["os"] - = makeReadOnly(tableFromPairs({ { "date", mLua["os"]["date"] }, - { "difftime", mLua["os"]["difftime"] }, { "time", mLua["os"]["time"] } })); - } - - LuaState::~LuaState() - { - // Should be cleaned before destructing mLua. - mCommonPackages.clear(); - mSandboxEnv = sol::nil; + = makeReadOnly(tableFromPairs({ { "date", mSol["os"]["date"] }, + { "difftime", mSol["os"]["difftime"] }, { "time", mSol["os"]["time"] } })); } sol::table makeReadOnly(const sol::table& table, bool strictIndex) @@ -280,9 +300,9 @@ namespace LuaUtil { sol::protected_function script = loadScriptAndCache(path); - sol::environment env(mLua, sol::create, mSandboxEnv); + sol::environment env(mSol, sol::create, mSandboxEnv); std::string envName = namePrefix + "[" + path + "]:"; - env["print"] = mLua["printGen"](envName); + env["print"] = mSol["printGen"](envName); env["_G"] = env; env[sol::metatable_key]["__metatable"] = false; @@ -298,18 +318,18 @@ namespace LuaUtil else return package; }; - sol::table loaded(mLua, sol::create); + sol::table loaded(mSol, sol::create); for (const auto& [key, value] : mCommonPackages) loaded[key] = maybeRunLoader(value); for (const auto& [key, value] : packages) loaded[key] = maybeRunLoader(value); - env["require"] = [this, env, loaded, hiddenData, scriptId](std::string_view packageName) mutable { + env["require"] = [this, env, loaded, hiddenData](std::string_view packageName) mutable { sol::object package = loaded[packageName]; if (package != sol::nil) return package; sol::protected_function packageLoader = loadScriptAndCache(packageNameToVfsPath(packageName, mVFS)); sol::set_environment(env, packageLoader); - package = call(scriptId, packageLoader, packageName); + package = call(packageLoader, packageName); loaded[packageName] = package; return package; }; @@ -320,8 +340,8 @@ namespace LuaUtil sol::environment LuaState::newInternalLibEnvironment() { - sol::environment env(mLua, sol::create, mSandboxEnv); - sol::table loaded(mLua, sol::create); + sol::environment env(mSol, sol::create, mSandboxEnv); + sol::table loaded(mSol, sol::create); for (const std::string& s : safePackages) loaded[s] = static_cast(mSandboxEnv[s]); env["require"] = [this, loaded, env](const std::string& module) mutable { @@ -347,7 +367,7 @@ namespace LuaUtil { auto iter = mCompiledScripts.find(path); if (iter != mCompiledScripts.end()) - return mLua.load(iter->second.as_string_view(), path, sol::load_mode::binary); + return mSol.load(iter->second.as_string_view(), path, sol::load_mode::binary); sol::function res = loadFromVFS(path); mCompiledScripts[path] = res.dump(); return res; @@ -356,7 +376,7 @@ namespace LuaUtil sol::function LuaState::loadFromVFS(const std::string& path) { std::string fileContent(std::istreambuf_iterator(*mVFS->get(path)), {}); - sol::load_result res = mLua.load(fileContent, path, sol::load_mode::text); + sol::load_result res = mSol.load(fileContent, path, sol::load_mode::text); if (!res.valid()) throw std::runtime_error("Lua error: " + res.get()); return res; @@ -365,7 +385,7 @@ namespace LuaUtil sol::function LuaState::loadInternalLib(std::string_view libName) { const auto path = packageNameToPath(libName, mLibSearchPaths); - sol::load_result res = mLua.load_file(Files::pathToUnicodeString(path), sol::load_mode::text); + sol::load_result res = mSol.load_file(Files::pathToUnicodeString(path), sol::load_mode::text); if (!res.valid()) throw std::runtime_error("Lua error: " + res.get()); return res; diff --git a/components/lua/luastate.hpp b/components/lua/luastate.hpp index bda9ad38a4..b0e173d67e 100644 --- a/components/lua/luastate.hpp +++ b/components/lua/luastate.hpp @@ -54,23 +54,21 @@ namespace LuaUtil LuaState(const LuaState&) = delete; LuaState(LuaState&&) = delete; - ~LuaState(); - // Returns underlying sol::state. - sol::state& sol() { return mLua; } + sol::state_view& sol() { return mSol; } // Can be used by a C++ function that is called from Lua to get the Lua traceback. // Makes no sense if called not from Lua code. // Note: It is a slow function, should be used for debug purposes only. - std::string debugTraceback() { return mLua["debug"]["traceback"]().get(); } + std::string debugTraceback() { return mSol["debug"]["traceback"]().get(); } // A shortcut to create a new Lua table. - sol::table newTable() { return sol::table(mLua, sol::create); } + sol::table newTable() { return sol::table(mSol, sol::create); } template sol::table tableFromPairs(std::initializer_list> list) { - sol::table res(mLua, sol::create); + sol::table res(mSol, sol::create); for (const auto& [k, v] : list) res[k] = v; return res; @@ -105,7 +103,7 @@ namespace LuaUtil sol::function loadFromVFS(const std::string& path); sol::environment newInternalLibEnvironment(); - uint64_t getTotalMemoryUsage() const { return mTotalMemoryUsage; } + uint64_t getTotalMemoryUsage() const { return mSol.memory_used(); } uint64_t getSmallAllocMemoryUsage() const { return mSmallAllocMemoryUsage; } uint64_t getMemoryUsageByScriptIndex(unsigned id) const { @@ -114,6 +112,10 @@ namespace LuaUtil const LuaStateSettings& getSettings() const { return mSettings; } + // Note: Lua profiler can not be re-enabled after disabling. + static void disableProfiler() { sProfilerEnabled = false; } + static bool isProfilerEnabled() { return sProfilerEnabled; } + private: static sol::protected_function_result throwIfError(sol::protected_function_result&&); template @@ -126,6 +128,8 @@ namespace LuaUtil static void countHook(lua_State* L, lua_Debug* ar); static void* trackingAllocator(void* ud, void* ptr, size_t osize, size_t nsize); + lua_State* createLuaRuntime(LuaState* luaState); + struct AllocOwner { std::shared_ptr mContainer; @@ -134,21 +138,43 @@ namespace LuaUtil const LuaStateSettings mSettings; - // Needed to track resource usage per script, must be initialized before mLua. - ScriptId mActiveScriptId; - uint64_t mCurrentCallInstructionCounter = 0; + // Needed to track resource usage per script, must be initialized before mLuaHolder. + std::vector mActiveScriptIdStack; + uint64_t mWatchdogInstructionCounter = 0; std::map mBigAllocOwners; uint64_t mTotalMemoryUsage = 0; uint64_t mSmallAllocMemoryUsage = 0; std::vector mMemoryUsage; - sol::state mLua; + class LuaStateHolder + { + public: + LuaStateHolder(lua_State* L) + : L(L) + { + sol::set_default_state(L); + } + ~LuaStateHolder() { lua_close(L); } + LuaStateHolder(const LuaStateHolder&) = delete; + LuaStateHolder(LuaStateHolder&&) = delete; + lua_State* get() { return L; } + + private: + lua_State* L; + }; + + // Must be declared before mSol and all sol-related objects. Then on exit it will be destructed the last. + LuaStateHolder mLuaHolder; + + sol::state_view mSol; const ScriptsConfiguration* mConf; sol::table mSandboxEnv; std::map mCompiledScripts; std::map mCommonPackages; const VFS::Manager* mVFS; std::vector mLibSearchPaths; + + static bool sProfilerEnabled; }; // LuaUtil::call should be used for every call of every Lua function. @@ -178,25 +204,30 @@ namespace LuaUtil template sol::protected_function_result call(ScriptId scriptId, const sol::protected_function& fn, Args&&... args) { - LuaState* luaState; - (void)lua_getallocf(fn.lua_state(), reinterpret_cast(&luaState)); - assert(luaState->mActiveScriptId.mContainer == nullptr && "recursive call of LuaUtil::call(scriptId, ...) ?"); - luaState->mActiveScriptId = scriptId; - luaState->mCurrentCallInstructionCounter = 0; + LuaState* luaState = nullptr; + if (LuaState::sProfilerEnabled && scriptId.mContainer) + { + (void)lua_getallocf(fn.lua_state(), reinterpret_cast(&luaState)); + luaState->mActiveScriptIdStack.push_back(scriptId); + luaState->mWatchdogInstructionCounter = 0; + } try { auto res = LuaState::throwIfError(fn(std::forward(args)...)); - luaState->mActiveScriptId = {}; + if (luaState) + luaState->mActiveScriptIdStack.pop_back(); return res; } catch (std::exception&) { - luaState->mActiveScriptId = {}; + if (luaState) + luaState->mActiveScriptIdStack.pop_back(); throw; } catch (...) { - luaState->mActiveScriptId = {}; + if (luaState) + luaState->mActiveScriptIdStack.pop_back(); throw std::runtime_error("Unknown error"); } } diff --git a/components/lua/scriptscontainer.cpp b/components/lua/scriptscontainer.cpp index 40a1919fbe..6b6151b293 100644 --- a/components/lua/scriptscontainer.cpp +++ b/components/lua/scriptscontainer.cpp @@ -75,7 +75,7 @@ namespace LuaUtil script.mHiddenData[sScriptIdKey] = ScriptId{ this, scriptId }; script.mHiddenData[sScriptDebugNameKey] = debugName; script.mPath = path; - script.mStats.mCPUusage = 0; + script.mStats.mAvgInstructionCount = 0; const auto oldMemoryUsageIt = mRemovedScriptsMemoryUsage.find(scriptId); if (oldMemoryUsageIt != mRemovedScriptsMemoryUsage.end()) @@ -590,24 +590,24 @@ namespace LuaUtil updateTimerQueue(mGameTimersQueue, gameTime); } - static constexpr float CPUusageAvgCoef = 1.0 / 30; // averaging over approximately 30 frames + static constexpr float instructionCountAvgCoef = 1.0 / 30; // averaging over approximately 30 frames - void ScriptsContainer::CPUusageNextFrame() + void ScriptsContainer::statsNextFrame() { for (auto& [scriptId, script] : mScripts) { // The averaging formula is: averageValue = averageValue * (1-c) + newValue * c - script.mStats.mCPUusage *= 1 - CPUusageAvgCoef; - if (script.mStats.mCPUusage < 5) - script.mStats.mCPUusage = 0; // speeding up converge to zero if newValue is zero + script.mStats.mAvgInstructionCount *= 1 - instructionCountAvgCoef; + if (script.mStats.mAvgInstructionCount < 5) + script.mStats.mAvgInstructionCount = 0; // speeding up converge to zero if newValue is zero } } - void ScriptsContainer::addCPUusage(int scriptId, int64_t CPUusage) + void ScriptsContainer::addInstructionCount(int scriptId, int64_t instructionCount) { auto it = mScripts.find(scriptId); if (it != mScripts.end()) - it->second.mStats.mCPUusage += CPUusage * CPUusageAvgCoef; + it->second.mStats.mAvgInstructionCount += instructionCount * instructionCountAvgCoef; } void ScriptsContainer::addMemoryUsage(int scriptId, int64_t memoryDelta) @@ -640,7 +640,7 @@ namespace LuaUtil stats.resize(mLua.getConfiguration().size()); for (auto& [id, script] : mScripts) { - stats[id].mCPUusage += script.mStats.mCPUusage; + stats[id].mAvgInstructionCount += script.mStats.mAvgInstructionCount; stats[id].mMemoryUsage += script.mStats.mMemoryUsage; } for (auto& [id, mem] : mRemovedScriptsMemoryUsage) diff --git a/components/lua/scriptscontainer.hpp b/components/lua/scriptscontainer.hpp index 6b9df70ac3..ebc7fd6d55 100644 --- a/components/lua/scriptscontainer.hpp +++ b/components/lua/scriptscontainer.hpp @@ -146,12 +146,12 @@ namespace LuaUtil // because they can not be stored in saves. I.e. loading a saved game will not fully restore the state. void setupUnsavableTimer(TimerType type, double time, int scriptId, sol::main_protected_function callback); - // Informs that new frame is started. Needed to track CPU usage per frame. - void CPUusageNextFrame(); + // Informs that new frame is started. Needed to track Lua instruction count per frame. + void statsNextFrame(); struct ScriptStats { - float mCPUusage = 0; // averaged number of Lua instructions per frame + float mAvgInstructionCount = 0; // averaged number of Lua instructions per frame int64_t mMemoryUsage = 0; // bytes }; void collectStats(std::vector& stats) const; @@ -227,7 +227,7 @@ namespace LuaUtil using EventHandlerList = std::vector; friend class LuaState; - void addCPUusage(int scriptId, int64_t CPUusage); + void addInstructionCount(int scriptId, int64_t instructionCount); void addMemoryUsage(int scriptId, int64_t memoryDelta); // Add to container without calling onInit/onLoad. diff --git a/components/lua/utilpackage.cpp b/components/lua/utilpackage.cpp index 91f189337b..65431f3f6e 100644 --- a/components/lua/utilpackage.cpp +++ b/components/lua/utilpackage.cpp @@ -89,8 +89,9 @@ namespace LuaUtil } } - sol::table initUtilPackage(sol::state& lua) + sol::table initUtilPackage(lua_State* L) { + sol::state_view lua(L); sol::table util(lua, sol::create); // Lua bindings for Vec2 diff --git a/components/lua/utilpackage.hpp b/components/lua/utilpackage.hpp index de47710334..2b5da99346 100644 --- a/components/lua/utilpackage.hpp +++ b/components/lua/utilpackage.hpp @@ -34,8 +34,7 @@ namespace LuaUtil return { q }; } - sol::table initUtilPackage(sol::state&); - + sol::table initUtilPackage(lua_State*); } #endif // COMPONENTS_LUA_UTILPACKAGE_H diff --git a/files/settings-default.cfg b/files/settings-default.cfg index 0cefc3047f..6f672a65b4 100644 --- a/files/settings-default.cfg +++ b/files/settings-default.cfg @@ -1141,17 +1141,21 @@ lua debug = false # If zero, Lua scripts are processed in the main thread. lua num threads = 1 +# Enable Lua profiler +lua profiler = true + # No ownership tracking for allocations below or equal this size. small alloc max size = 1024 -# Memory limit for Lua runtime. If exceeded then only small allocations are allowed. Small allocations are always allowed, so e.g. Lua console can function. -# Default value is 2GB. +# Memory limit for Lua runtime (only if lua profiler = true). If exceeded then only small allocations are allowed. +# Small allocations are always allowed, so e.g. Lua console can function. Default value is 2GB. memory limit = 2147483648 -# Print debug info about memory usage. +# Print debug info about memory usage (only if lua profiler = true). log memory usage = false -# The maximal number of Lua instructions per function call. If exceeded (e.g. because of an infinite loop) the function will be terminated. +# The maximal number of Lua instructions per function call (only if lua profiler = true). +# If exceeded (e.g. because of an infinite loop) the function will be terminated. instruction limit per call = 100000000 [Stereo]