Merge branch 'unloadedcontainers' into 'master'

Fix Lua memory usage

See merge request OpenMW/openmw!4363

apps/components_tests/lua/test_scriptscontainer.cpp

@@ -6,6 +6,7 @@
 #include <components/lua/asyncpackage.hpp>
 #include <components/lua/luastate.hpp>
 #include <components/lua/scriptscontainer.hpp>
+#include <components/lua/scripttracker.hpp>
 
 #include <components/testing/util.hpp>
 
@@ -135,6 +136,31 @@ return {
         end,
     },
 }
+)X");
+
+    constexpr VFS::Path::NormalizedView unloadPath("unload.lua");
+
+    VFSTestFile unloadScript(R"X(
+x = 0
+y = 0
+z = 0
+return {
+    engineHandlers = {
+        onSave = function(state)
+            print('saving', x, y, z)
+            return {x = x, y = y}
+        end,
+        onLoad = function(state)
+            x, y = state.x, state.y
+            print('loaded', x, y, z)
+        end
+    },
+    eventHandlers = {
+        Set = function(eventData)
+            x, y, z = eventData.x, eventData.y, eventData.z
+        end
+    }
+}
 )X");
 
     struct LuaScriptsContainerTest : Test
@@ -151,6 +177,7 @@ return {
             { testInterfacePath, &interfaceScript },
             { overrideInterfacePath, &overrideInterfaceScript },
             { useInterfacePath, &useInterfaceScript },
+            { unloadPath, &unloadScript },
         });
 
         LuaUtil::ScriptsConfiguration mCfg;
@@ -171,6 +198,7 @@ CUSTOM, NPC: loadSave2.lua
 CUSTOM, PLAYER: testInterface.lua
 CUSTOM, PLAYER: overrideInterface.lua
 CUSTOM, PLAYER: useInterface.lua
+CUSTOM: unload.lua
 )X");
             mCfg.init(std::move(cfg));
         }
@@ -511,4 +539,35 @@ CUSTOM, PLAYER: useInterface.lua
         Log::sMinDebugLevel = level;
     }
 
+    TEST_F(LuaScriptsContainerTest, Unload)
+    {
+        LuaUtil::ScriptTracker tracker;
+        LuaUtil::ScriptsContainer scripts1(&mLua, "Test", &tracker, false);
+
+        EXPECT_TRUE(scripts1.addCustomScript(*mCfg.findId(unloadPath)));
+        EXPECT_EQ(tracker.size(), 1);
+
+        mLua.protectedCall([&](LuaUtil::LuaView& lua) {
+            scripts1.receiveEvent("Set", LuaUtil::serialize(lua.sol().create_table_with("x", 3, "y", 2, "z", 1)));
+            testing::internal::CaptureStdout();
+            for (int i = 0; i < 600; ++i)
+                tracker.unloadInactiveScripts(lua);
+            EXPECT_EQ(tracker.size(), 0);
+            scripts1.receiveEvent("Set", LuaUtil::serialize(lua.sol().create_table_with("x", 10, "y", 20, "z", 30)));
+            EXPECT_EQ(internal::GetCapturedStdout(),
+                "Test[unload.lua]:\tsaving\t3\t2\t1\n"
+                "Test[unload.lua]:\tloaded\t3\t2\t0\n");
+        });
+        EXPECT_EQ(tracker.size(), 1);
+        ESM::LuaScripts data;
+        scripts1.save(data);
+        EXPECT_EQ(tracker.size(), 1);
+        mLua.protectedCall([&](LuaUtil::LuaView& lua) {
+            for (int i = 0; i < 600; ++i)
+                tracker.unloadInactiveScripts(lua);
+        });
+        EXPECT_EQ(tracker.size(), 0);
+        scripts1.load(data);
+        EXPECT_EQ(tracker.size(), 0);
+    }
 }

apps/openmw/mwlua/localscripts.cpp

@@ -224,8 +224,8 @@ namespace MWLua
         };
     }
 
-    LocalScripts::LocalScripts(LuaUtil::LuaState* lua, const LObject& obj)
-        : LuaUtil::ScriptsContainer(lua, "L" + obj.id().toString())
+    LocalScripts::LocalScripts(LuaUtil::LuaState* lua, const LObject& obj, LuaUtil::ScriptTracker* tracker)
+        : LuaUtil::ScriptsContainer(lua, "L" + obj.id().toString(), tracker, false)
         , mData(obj)
     {
         lua->protectedCall(

apps/openmw/mwlua/localscripts.hpp

@@ -62,12 +62,13 @@ namespace MWLua
     {
     public:
         static void initializeSelfPackage(const Context&);
-        LocalScripts(LuaUtil::LuaState* lua, const LObject& obj);
+        LocalScripts(LuaUtil::LuaState* lua, const LObject& obj, LuaUtil::ScriptTracker* tracker = nullptr);
 
         MWBase::LuaManager::ActorControls* getActorControls() { return &mData.mControls; }
         const MWWorld::Ptr& getPtrOrEmpty() const { return mData.ptrOrEmpty(); }
 
         void setActive(bool active);
+        bool isActive() const override { return mData.mIsActive; }
         void onConsume(const LObject& consumable) { callEngineHandlers(mOnConsumeHandlers, consumable); }
         void onActivated(const LObject& actor) { callEngineHandlers(mOnActivatedHandlers, actor); }
         void onTeleported() { callEngineHandlers(mOnTeleportedHandlers); }

apps/openmw/mwlua/luamanagerimp.cpp

@@ -210,6 +210,8 @@ namespace MWLua
                 scripts->update(frameDuration);
             mGlobalScripts.update(frameDuration);
         }
+
+        mLua.protectedCall([&](LuaUtil::LuaView& lua) { mScriptTracker.unloadInactiveScripts(lua); });
     }
 
     void LuaManager::objectTeleported(const MWWorld::Ptr& ptr)
@@ -560,7 +562,7 @@ namespace MWLua
         }
         else
         {
-            scripts = std::make_shared<LocalScripts>(&mLua, LObject(getId(ptr)));
+            scripts = std::make_shared<LocalScripts>(&mLua, LObject(getId(ptr)), &mScriptTracker);
             if (!autoStartConf.has_value())
                 autoStartConf = mConfiguration.getLocalConf(type, ptr.getCellRef().getRefId(), getId(ptr));
             scripts->setAutoStartConf(std::move(*autoStartConf));

apps/openmw/mwlua/luamanagerimp.hpp

@@ -8,6 +8,7 @@
 
 #include <components/lua/inputactions.hpp>
 #include <components/lua/luastate.hpp>
+#include <components/lua/scripttracker.hpp>
 #include <components/lua/storage.hpp>
 #include <components/lua_ui/resources.hpp>
 #include <components/misc/color.hpp>
@@ -234,6 +235,8 @@ namespace MWLua
 
         LuaUtil::InputAction::Registry mInputActions;
         LuaUtil::InputTrigger::Registry mInputTriggers;
+
+        LuaUtil::ScriptTracker mScriptTracker;
     };
 
 }

components/CMakeLists.txt

@@ -59,7 +59,7 @@ list(APPEND COMPONENT_FILES "${OpenMW_BINARY_DIR}/${OSG_PLUGIN_CHECKER_CPP_FILE}
 
 add_component_dir (lua
     luastate scriptscontainer asyncpackage utilpackage serialization configuration l10n storage utf8
-    shapes/box inputactions yamlloader
+    shapes/box inputactions yamlloader scripttracker
     )
 
 add_component_dir (l10n

components/lua/scriptscontainer.cpp

@@ -1,7 +1,18 @@
 #include "scriptscontainer.hpp"
 
+#include "scripttracker.hpp"
+
 #include <components/esm/luascripts.hpp>
 
+namespace
+{
+    struct ScriptInfo
+    {
+        std::string_view mInitData;
+        const ESM::LuaScript* mSavedData;
+    };
+}
+
 namespace LuaUtil
 {
     static constexpr std::string_view ENGINE_HANDLERS = "engineHandlers";
@@ -17,17 +28,23 @@ namespace LuaUtil
 
     int64_t ScriptsContainer::sInstanceCount = 0;
 
-    ScriptsContainer::ScriptsContainer(LuaUtil::LuaState* lua, std::string_view namePrefix)
+    ScriptsContainer::ScriptsContainer(
+        LuaUtil::LuaState* lua, std::string_view namePrefix, ScriptTracker* tracker, bool load)
         : mNamePrefix(namePrefix)
         , mLua(*lua)
         , mThis(std::make_shared<ScriptsContainer*>(this))
+        , mTracker(tracker)
     {
         sInstanceCount++;
         registerEngineHandlers({ &mUpdateHandlers });
-        lua->protectedCall([&](LuaView& view) {
-            mPublicInterfaces = sol::table(view.sol(), sol::create);
-            addPackage("openmw.interfaces", mPublicInterfaces);
-        });
+        if (load)
+        {
+            LoadedData& data = mData.emplace<LoadedData>();
+            mLua.protectedCall([&](LuaView& view) {
+                data.mPublicInterfaces = sol::table(view.sol(), sol::create);
+                addPackage("openmw.interfaces", makeReadOnly(data.mPublicInterfaces));
+            });
+        }
     }
 
     void ScriptsContainer::printError(int scriptId, std::string_view msg, const std::exception& e)
@@ -37,7 +54,9 @@ namespace LuaUtil
 
     void ScriptsContainer::addPackage(std::string packageName, sol::object package)
     {
-        mAPI.insert_or_assign(std::move(packageName), makeReadOnly(std::move(package)));
+        if (!package.is<sol::userdata>())
+            throw std::logic_error("Expected package to be read-only: " + packageName);
+        mAPI.insert_or_assign(std::move(packageName), std::move(package));
     }
 
     bool ScriptsContainer::addCustomScript(int scriptId, std::string_view initData)
@@ -70,7 +89,7 @@ namespace LuaUtil
         LuaView& view, int scriptId, std::optional<sol::function>& onInit, std::optional<sol::function>& onLoad)
     {
         assert(scriptId >= 0 && scriptId < static_cast<int>(mLua.getConfiguration().size()));
-        if (mScripts.count(scriptId) != 0)
+        if (hasScript(scriptId))
             return false; // already present
 
         const VFS::Path::Normalized& path = scriptPath(scriptId);
@@ -79,7 +98,8 @@ namespace LuaUtil
         debugName.append(path);
         debugName.push_back(']');
 
-        Script& script = mScripts[scriptId];
+        LoadedData& data = ensureLoaded();
+        Script& script = data.mScripts[scriptId];
         script.mHiddenData = view.newTable();
         script.mHiddenData[sScriptIdKey] = ScriptId{ this, scriptId };
         script.mHiddenData[sScriptDebugNameKey] = debugName;
@@ -144,9 +164,9 @@ namespace LuaUtil
                 for (const auto& [key, fn] : cast<sol::table>(eventHandlers))
                 {
                     std::string_view eventName = cast<std::string_view>(key);
-                    auto it = mEventHandlers.find(eventName);
-                    if (it == mEventHandlers.end())
-                        it = mEventHandlers.emplace(std::string(eventName), EventHandlerList()).first;
+                    auto it = data.mEventHandlers.find(eventName);
+                    if (it == data.mEventHandlers.end())
+                        it = data.mEventHandlers.emplace(std::string(eventName), EventHandlerList()).first;
                     insertHandler(it->second, scriptId, cast<sol::function>(fn));
                 }
             }
@@ -167,29 +187,55 @@ namespace LuaUtil
         }
         catch (std::exception& e)
         {
-            auto iter = mScripts.find(scriptId);
-            iter->second.mHiddenData[sScriptIdKey] = sol::nil;
+            auto iter = data.mScripts.find(scriptId);
             mRemovedScriptsMemoryUsage[scriptId] = iter->second.mStats.mMemoryUsage;
-            mScripts.erase(iter);
+            data.mScripts.erase(iter);
             Log(Debug::Error) << "Can't start " << debugName << "; " << e.what();
             return false;
         }
     }
 
+    bool ScriptsContainer::hasScript(int scriptId) const
+    {
+        return std::visit(
+            [&](auto&& variant) {
+                using T = std::decay_t<decltype(variant)>;
+                if constexpr (std::is_same_v<T, UnloadedData>)
+                {
+                    const auto& conf = mLua.getConfiguration();
+                    if (scriptId >= 0 && static_cast<size_t>(scriptId) < conf.size())
+                    {
+                        const auto& path = conf[scriptId].mScriptPath;
+                        for (const ESM::LuaScript& script : variant.mScripts)
+                        {
+                            if (script.mScriptPath == path)
+                                return true;
+                        }
+                    }
+                }
+                else if constexpr (std::is_same_v<T, LoadedData>)
+                {
+                    return variant.mScripts.count(scriptId) != 0;
+                }
+                return false;
+            },
+            mData);
+    }
+
     void ScriptsContainer::removeScript(int scriptId)
     {
-        auto scriptIter = mScripts.find(scriptId);
-        if (scriptIter == mScripts.end())
+        LoadedData& data = ensureLoaded();
+        auto scriptIter = data.mScripts.find(scriptId);
+        if (scriptIter == data.mScripts.end())
             return; // no such script
         Script& script = scriptIter->second;
         if (script.mInterface)
             removeInterface(scriptId, script);
-        script.mHiddenData[sScriptIdKey] = sol::nil;
         mRemovedScriptsMemoryUsage[scriptId] = script.mStats.mMemoryUsage;
-        mScripts.erase(scriptIter);
+        data.mScripts.erase(scriptIter);
         for (auto& [_, handlers] : mEngineHandlers)
             removeHandler(handlers->mList, scriptId);
-        for (auto& [_, handlers] : mEventHandlers)
+        for (auto& [_, handlers] : data.mEventHandlers)
             removeHandler(handlers, scriptId);
     }
 
@@ -199,7 +245,8 @@ namespace LuaUtil
         const Script* prev = nullptr;
         const Script* next = nullptr;
         int nextId = 0;
-        for (const auto& [otherId, otherScript] : mScripts)
+        LoadedData& data = ensureLoaded();
+        for (const auto& [otherId, otherScript] : data.mScripts)
         {
             if (scriptId == otherId || script.mInterfaceName != otherScript.mInterfaceName)
                 continue;
@@ -235,7 +282,7 @@ namespace LuaUtil
             }
         }
         if (next == nullptr)
-            mPublicInterfaces[script.mInterfaceName] = *script.mInterface;
+            data.mPublicInterfaces[script.mInterfaceName] = *script.mInterface;
     }
 
     void ScriptsContainer::removeInterface(int scriptId, const Script& script)
@@ -244,7 +291,8 @@ namespace LuaUtil
         const Script* prev = nullptr;
         const Script* next = nullptr;
         int nextId = 0;
-        for (const auto& [otherId, otherScript] : mScripts)
+        LoadedData& data = ensureLoaded();
+        for (const auto& [otherId, otherScript] : data.mScripts)
         {
             if (scriptId == otherId || script.mInterfaceName != otherScript.mInterfaceName)
                 continue;
@@ -275,9 +323,9 @@ namespace LuaUtil
             }
         }
         else if (prev)
-            mPublicInterfaces[script.mInterfaceName] = *prev->mInterface;
+            data.mPublicInterfaces[script.mInterfaceName] = *prev->mInterface;
         else
-            mPublicInterfaces[script.mInterfaceName] = sol::nil;
+            data.mPublicInterfaces[script.mInterfaceName] = sol::nil;
     }
 
     void ScriptsContainer::insertHandler(std::vector<Handler>& list, int scriptId, sol::function fn)
@@ -302,8 +350,9 @@ namespace LuaUtil
 
     void ScriptsContainer::receiveEvent(std::string_view eventName, std::string_view eventData)
     {
-        auto it = mEventHandlers.find(eventName);
-        if (it == mEventHandlers.end())
+        LoadedData& data = ensureLoaded();
+        auto it = data.mEventHandlers.find(eventName);
+        if (it == data.mEventHandlers.end())
             return;
         mLua.protectedCall([&](LuaView& view) {
             sol::object data;
@@ -355,6 +404,12 @@ namespace LuaUtil
 
     void ScriptsContainer::save(ESM::LuaScripts& data)
     {
+        if (UnloadedData* unloadedData = std::get_if<UnloadedData>(&mData))
+        {
+            data.mScripts = unloadedData->mScripts;
+            return;
+        }
+        const auto& loadedData = std::get<LoadedData>(mData);
         std::map<int, std::vector<ESM::LuaTimer>> timers;
         auto saveTimerFn = [&](const Timer& timer, TimerType timerType) {
             if (!timer.mSerializable)
@@ -366,12 +421,12 @@ namespace LuaUtil
             savedTimer.mCallbackArgument = timer.mSerializedArg;
             timers[timer.mScriptId].push_back(std::move(savedTimer));
         };
-        for (const Timer& timer : mSimulationTimersQueue)
+        for (const Timer& timer : loadedData.mSimulationTimersQueue)
             saveTimerFn(timer, TimerType::SIMULATION_TIME);
-        for (const Timer& timer : mGameTimersQueue)
+        for (const Timer& timer : loadedData.mGameTimersQueue)
             saveTimerFn(timer, TimerType::GAME_TIME);
         data.mScripts.clear();
-        for (auto& [scriptId, script] : mScripts)
+        for (auto& [scriptId, script] : loadedData.mScripts)
         {
             ESM::LuaScript savedScript;
             // Note: We can not use `scriptPath(scriptId)` here because `save` can be called during
@@ -401,11 +456,6 @@ namespace LuaUtil
         removeAllScripts();
         const ScriptsConfiguration& cfg = mLua.getConfiguration();
 
-        struct ScriptInfo
-        {
-            std::string_view mInitData;
-            const ESM::LuaScript* mSavedData;
-        };
         std::map<int, ScriptInfo> scripts;
         for (const auto& [scriptId, initData] : mAutoStartScripts)
             scripts[scriptId] = { initData, nullptr };
@@ -428,6 +478,60 @@ namespace LuaUtil
         }
 
         mLua.protectedCall([&](LuaView& view) {
+            UnloadedData& container = ensureUnloaded(view);
+
+            for (const auto& [scriptId, scriptInfo] : scripts)
+            {
+                if (scriptInfo.mSavedData == nullptr)
+                    continue;
+                ESM::LuaScript& script = container.mScripts.emplace_back(*scriptInfo.mSavedData);
+                for (ESM::LuaTimer& savedTimer : script.mTimers)
+                {
+                    try
+                    {
+                        sol::object arg = deserialize(view.sol(), savedTimer.mCallbackArgument, mSavedDataDeserializer);
+                        // It is important if the order of content files was changed. The deserialize-serialize
+                        // procedure updates refnums, so timer.mSerializedArg may be not equal to
+                        // savedTimer.mCallbackArgument.
+                        savedTimer.mCallbackArgument = serialize(arg, mSerializer);
+                    }
+                    catch (std::exception& e)
+                    {
+                        printError(scriptId, "can not load timer", e);
+                    }
+                }
+            }
+        });
+    }
+
+    ScriptsContainer::LoadedData& ScriptsContainer::ensureLoaded()
+    {
+        mRequiredLoading = true;
+        if (LoadedData* data = std::get_if<LoadedData>(&mData))
+            return *data;
+        UnloadedData& unloadedData = std::get<UnloadedData>(mData);
+        std::vector<ESM::LuaScript> savedScripts = std::move(unloadedData.mScripts);
+        LoadedData& data = mData.emplace<LoadedData>();
+
+        const ScriptsConfiguration& cfg = mLua.getConfiguration();
+
+        std::map<int, ScriptInfo> scripts;
+        for (const auto& [scriptId, initData] : mAutoStartScripts)
+            scripts[scriptId] = { initData, nullptr };
+        for (const ESM::LuaScript& s : savedScripts)
+        {
+            std::optional<int> scriptId = cfg.findId(s.mScriptPath);
+            auto it = scripts.find(*scriptId);
+            if (it != scripts.end())
+                it->second.mSavedData = &s;
+            else if (cfg.isCustomScript(*scriptId))
+                scripts[*scriptId] = { cfg[*scriptId].mInitializationData, &s };
+        }
+
+        mLua.protectedCall([&](LuaView& view) {
+            data.mPublicInterfaces = sol::table(view.sol(), sol::create);
+            addPackage("openmw.interfaces", makeReadOnly(data.mPublicInterfaces));
+
             for (const auto& [scriptId, scriptInfo] : scripts)
             {
                 std::optional<sol::function> onInit, onLoad;
@@ -471,9 +575,9 @@ namespace LuaUtil
                         timer.mSerializedArg = serialize(timer.mArg, mSerializer);
 
                         if (savedTimer.mType == TimerType::GAME_TIME)
-                            mGameTimersQueue.push_back(std::move(timer));
+                            data.mGameTimersQueue.push_back(std::move(timer));
                         else
-                            mSimulationTimersQueue.push_back(std::move(timer));
+                            data.mSimulationTimersQueue.push_back(std::move(timer));
                     }
                     catch (std::exception& e)
                     {
@@ -483,15 +587,32 @@ namespace LuaUtil
             }
         });
 
-        std::make_heap(mSimulationTimersQueue.begin(), mSimulationTimersQueue.end());
-        std::make_heap(mGameTimersQueue.begin(), mGameTimersQueue.end());
+        std::make_heap(data.mSimulationTimersQueue.begin(), data.mSimulationTimersQueue.end());
+        std::make_heap(data.mGameTimersQueue.begin(), data.mGameTimersQueue.end());
+
+        if (mTracker)
+            mTracker->onLoad(*this);
+
+        return data;
+    }
+
+    ScriptsContainer::UnloadedData& ScriptsContainer::ensureUnloaded(LuaView&)
+    {
+        if (UnloadedData* data = std::get_if<UnloadedData>(&mData))
+            return *data;
+        UnloadedData data;
+        save(data);
+        mAPI.erase("openmw.interfaces");
+        UnloadedData& out = mData.emplace<UnloadedData>(std::move(data));
+        for (auto& [_, handlers] : mEngineHandlers)
+            handlers->mList.clear();
+        mRequiredLoading = false;
+        return out;
     }
 
     ScriptsContainer::~ScriptsContainer()
     {
         sInstanceCount--;
-        for (auto& [_, script] : mScripts)
-            script.mHiddenData[sScriptIdKey] = sol::nil;
         *mThis = nullptr;
     }
 
@@ -499,24 +620,36 @@ namespace LuaUtil
     // external objects that are already removed during child class destruction.
     void ScriptsContainer::removeAllScripts()
     {
-        for (auto& [id, script] : mScripts)
-        {
-            script.mHiddenData[sScriptIdKey] = sol::nil;
-            mRemovedScriptsMemoryUsage[id] = script.mStats.mMemoryUsage;
-        }
-        mScripts.clear();
-        for (auto& [_, handlers] : mEngineHandlers)
-            handlers->mList.clear();
-        mEventHandlers.clear();
-        mSimulationTimersQueue.clear();
-        mGameTimersQueue.clear();
-        mPublicInterfaces.clear();
+        std::visit(
+            [&](auto&& variant) {
+                using T = std::decay_t<decltype(variant)>;
+                if constexpr (std::is_same_v<T, UnloadedData>)
+                {
+                    variant.mScripts.clear();
+                }
+                else if constexpr (std::is_same_v<T, LoadedData>)
+                {
+                    for (auto& [id, script] : variant.mScripts)
+                    {
+                        mRemovedScriptsMemoryUsage[id] = script.mStats.mMemoryUsage;
+                    }
+                    variant.mScripts.clear();
+                    for (auto& [_, handlers] : mEngineHandlers)
+                        handlers->mList.clear();
+                    variant.mEventHandlers.clear();
+                    variant.mSimulationTimersQueue.clear();
+                    variant.mGameTimersQueue.clear();
+                    variant.mPublicInterfaces.clear();
+                }
+            },
+            mData);
     }
 
     ScriptsContainer::Script& ScriptsContainer::getScript(int scriptId)
     {
-        auto it = mScripts.find(scriptId);
-        if (it == mScripts.end())
+        LoadedData& data = ensureLoaded();
+        auto it = data.mScripts.find(scriptId);
+        if (it == data.mScripts.end())
             throw std::logic_error("Script doesn't exist");
         return it->second;
     }
@@ -543,7 +676,8 @@ namespace LuaUtil
         t.mTime = time;
         t.mArg = std::move(callbackArg);
         t.mSerializedArg = serialize(t.mArg, mSerializer);
-        insertTimer(type == TimerType::GAME_TIME ? mGameTimersQueue : mSimulationTimersQueue, std::move(t));
+        LoadedData& data = ensureLoaded();
+        insertTimer(type == TimerType::GAME_TIME ? data.mGameTimersQueue : data.mSimulationTimersQueue, std::move(t));
     }
 
     void ScriptsContainer::setupUnsavableTimer(
@@ -557,8 +691,8 @@ namespace LuaUtil
         t.mCallback = mTemporaryCallbackCounter;
         getScript(t.mScriptId).mTemporaryCallbacks.emplace(mTemporaryCallbackCounter, std::move(callback));
         mTemporaryCallbackCounter++;
-
-        insertTimer(type == TimerType::GAME_TIME ? mGameTimersQueue : mSimulationTimersQueue, std::move(t));
+        LoadedData& data = ensureLoaded();
+        insertTimer(type == TimerType::GAME_TIME ? data.mGameTimersQueue : data.mSimulationTimersQueue, std::move(t));
     }
 
     void ScriptsContainer::callTimer(const Timer& t)
@@ -599,41 +733,52 @@ namespace LuaUtil
 
     void ScriptsContainer::processTimers(double simulationTime, double gameTime)
     {
-        updateTimerQueue(mSimulationTimersQueue, simulationTime);
-        updateTimerQueue(mGameTimersQueue, gameTime);
+        LoadedData& data = ensureLoaded();
+        updateTimerQueue(data.mSimulationTimersQueue, simulationTime);
+        updateTimerQueue(data.mGameTimersQueue, gameTime);
     }
 
     static constexpr float instructionCountAvgCoef = 1.0f / 30; // averaging over approximately 30 frames
 
     void ScriptsContainer::statsNextFrame()
     {
-        for (auto& [scriptId, script] : mScripts)
+        if (LoadedData* data = std::get_if<LoadedData>(&mData))
         {
-            // The averaging formula is: averageValue = averageValue * (1-c) + newValue * c
-            script.mStats.mAvgInstructionCount *= 1 - instructionCountAvgCoef;
-            if (script.mStats.mAvgInstructionCount < 5)
-                script.mStats.mAvgInstructionCount = 0; // speeding up converge to zero if newValue is zero
+            for (auto& [scriptId, script] : data->mScripts)
+            {
+                // The averaging formula is: averageValue = averageValue * (1-c) + newValue * c
+                script.mStats.mAvgInstructionCount *= 1 - instructionCountAvgCoef;
+                if (script.mStats.mAvgInstructionCount < 5)
+                    script.mStats.mAvgInstructionCount = 0; // speeding up converge to zero if newValue is zero
+            }
         }
     }
 
     void ScriptsContainer::addInstructionCount(int scriptId, int64_t instructionCount)
     {
-        auto it = mScripts.find(scriptId);
-        if (it != mScripts.end())
-            it->second.mStats.mAvgInstructionCount += instructionCount * instructionCountAvgCoef;
+        if (LoadedData* data = std::get_if<LoadedData>(&mData))
+        {
+            auto it = data->mScripts.find(scriptId);
+            if (it != data->mScripts.end())
+                it->second.mStats.mAvgInstructionCount += instructionCount * instructionCountAvgCoef;
+        }
     }
 
     void ScriptsContainer::addMemoryUsage(int scriptId, int64_t memoryDelta)
     {
-        int64_t* usage;
-        auto it = mScripts.find(scriptId);
-        if (it != mScripts.end())
-            usage = &it->second.mStats.mMemoryUsage;
-        else
-        {
-            auto [rIt, _] = mRemovedScriptsMemoryUsage.emplace(scriptId, 0);
-            usage = &rIt->second;
-        }
+        int64_t* usage = std::visit(
+            [&](auto&& variant) {
+                using T = std::decay_t<decltype(variant)>;
+                if constexpr (std::is_same_v<T, LoadedData>)
+                {
+                    auto it = variant.mScripts.find(scriptId);
+                    if (it != variant.mScripts.end())
+                        return &it->second.mStats.mMemoryUsage;
+                }
+                auto [rIt, _] = mRemovedScriptsMemoryUsage.emplace(scriptId, 0);
+                return &rIt->second;
+            },
+            mData);
         *usage += memoryDelta;
 
         if (mLua.getSettings().mLogMemoryUsage)
@@ -651,12 +796,21 @@ namespace LuaUtil
     void ScriptsContainer::collectStats(std::vector<ScriptStats>& stats) const
     {
         stats.resize(mLua.getConfiguration().size());
-        for (auto& [id, script] : mScripts)
+        if (const LoadedData* data = std::get_if<LoadedData>(&mData))
         {
-            stats[id].mAvgInstructionCount += script.mStats.mAvgInstructionCount;
-            stats[id].mMemoryUsage += script.mStats.mMemoryUsage;
+            for (auto& [id, script] : data->mScripts)
+            {
+                stats[id].mAvgInstructionCount += script.mStats.mAvgInstructionCount;
+                stats[id].mMemoryUsage += script.mStats.mMemoryUsage;
+            }
         }
         for (auto& [id, mem] : mRemovedScriptsMemoryUsage)
             stats[id].mMemoryUsage += mem;
     }
+
+    ScriptsContainer::Script::~Script()
+    {
+        if (mHiddenData != sol::nil)
+            mHiddenData[sScriptIdKey] = sol::nil;
+    }
 }

components/lua/scriptscontainer.hpp

@@ -4,6 +4,7 @@
 #include <map>
 #include <set>
 #include <string>
+#include <variant>
 
 #include <components/debug/debuglog.hpp>
 #include <components/esm/luascripts.hpp>
@@ -13,6 +14,7 @@
 
 namespace LuaUtil
 {
+    class ScriptTracker;
 
     // ScriptsContainer is a base class for all scripts containers (LocalScripts,
     // GlobalScripts, PlayerScripts, etc). Each script runs in a separate sandbox.
@@ -72,14 +74,17 @@ namespace LuaUtil
         using TimerType = ESM::LuaTimer::Type;
 
         // `namePrefix` is a common prefix for all scripts in the container. Used in logs for error messages and `print`
-        // output. `autoStartScripts` specifies the list of scripts that should be autostarted in this container;
-        //     the script names themselves are stored in ScriptsConfiguration.
-        ScriptsContainer(LuaState* lua, std::string_view namePrefix);
+        // output. `tracker` is a tracker for managing the container's state. `load` specifies whether the container
+        // should be constructed in a loaded state.
+        ScriptsContainer(
+            LuaState* lua, std::string_view namePrefix, ScriptTracker* tracker = nullptr, bool load = true);
 
         ScriptsContainer(const ScriptsContainer&) = delete;
         ScriptsContainer(ScriptsContainer&&) = delete;
         virtual ~ScriptsContainer();
 
+        // `conf` specifies the list of scripts that should be autostarted in this container; the script
+        // names themselves are stored in ScriptsConfiguration.
         void setAutoStartConf(ScriptIdsWithInitializationData conf) { mAutoStartScripts = std::move(conf); }
         const ScriptIdsWithInitializationData& getAutoStartConf() const { return mAutoStartScripts; }
 
@@ -91,9 +96,9 @@ namespace LuaUtil
         // new script, adds it to the container, and calls onInit for this script. Returns `true` if the script was
         // successfully added. The script should have CUSTOM flag. If the flag is not set, or file not found, or has
         // syntax errors, returns false. If such script already exists in the container, then also returns false.
-        bool addCustomScript(int scriptId, std::string_view initData = "");
+        bool addCustomScript(int scriptId, std::string_view initData = {});
 
-        bool hasScript(int scriptId) const { return mScripts.count(scriptId) != 0; }
+        bool hasScript(int scriptId) const;
         void removeScript(int scriptId);
 
         void processTimers(double simulationTime, double gameTime);
@@ -157,6 +162,8 @@ namespace LuaUtil
         void collectStats(std::vector<ScriptStats>& stats) const;
         static int64_t getInstanceCount() { return sInstanceCount; }
 
+        virtual bool isActive() const { return false; }
+
     protected:
         struct Handler
         {
@@ -180,6 +187,7 @@ namespace LuaUtil
         template <typename... Args>
         void callEngineHandlers(EngineHandlerList& handlers, const Args&... args)
         {
+            ensureLoaded();
             for (Handler& handler : handlers.mList)
             {
                 try
@@ -213,6 +221,8 @@ namespace LuaUtil
             std::map<int64_t, sol::main_protected_function> mTemporaryCallbacks;
             VFS::Path::Normalized mPath;
             ScriptStats mStats;
+
+            ~Script();
         };
         struct Timer
         {
@@ -257,21 +267,39 @@ namespace LuaUtil
         ScriptIdsWithInitializationData mAutoStartScripts;
         const UserdataSerializer* mSerializer = nullptr;
         const UserdataSerializer* mSavedDataDeserializer = nullptr;
+
         std::map<std::string, sol::object> mAPI;
+        struct LoadedData
+        {
+            std::map<int, Script> mScripts;
+            sol::table mPublicInterfaces;
 
-        std::map<int, Script> mScripts;
-        sol::table mPublicInterfaces;
+            std::map<std::string, EventHandlerList, std::less<>> mEventHandlers;
+
+            std::vector<Timer> mSimulationTimersQueue;
+            std::vector<Timer> mGameTimersQueue;
+        };
+        using UnloadedData = ESM::LuaScripts;
+
+        // Unloads the container to free resources held by the shared Lua state. This method serializes the container's
+        // state. The serialized data is automatically restored to the Lua state as required. Unloading and reloading
+        // the container is functionally equivalent to saving and loading the game, meaning the appropriate engine
+        // handlers are invoked.
+        UnloadedData& ensureUnloaded(LuaView& lua);
+        LoadedData& ensureLoaded();
 
         EngineHandlerList mUpdateHandlers{ "onUpdate" };
         std::map<std::string_view, EngineHandlerList*> mEngineHandlers;
-        std::map<std::string, EventHandlerList, std::less<>> mEventHandlers;
-
-        std::vector<Timer> mSimulationTimersQueue;
-        std::vector<Timer> mGameTimersQueue;
+        std::variant<UnloadedData, LoadedData> mData;
         int64_t mTemporaryCallbackCounter = 0;
 
         std::map<int, int64_t> mRemovedScriptsMemoryUsage;
-        std::shared_ptr<ScriptsContainer*> mThis; // used by LuaState to track ownership of memory allocations
+        using WeakPtr = std::shared_ptr<ScriptsContainer*>;
+        WeakPtr mThis; // used by LuaState to track ownership of memory allocations
+
+        ScriptTracker* mTracker;
+        bool mRequiredLoading = false;
+        friend class ScriptTracker;
 
         static int64_t sInstanceCount; // debug information, shown in Lua profiler
     };

components/lua/scripttracker.cpp

@@ -0,0 +1,53 @@
+#include "scripttracker.hpp"
+
+namespace LuaUtil
+{
+    namespace
+    {
+        constexpr unsigned sMinLoadedFrames = 50;
+        constexpr unsigned sMaxLoadedFrames = 600;
+        constexpr unsigned sUsageFrameGrowth = 10;
+        constexpr std::size_t sMinToProcess = 1;
+        constexpr std::size_t sToProcessDiv = 20; // 5%
+    }
+
+    void ScriptTracker::onLoad(ScriptsContainer& container)
+    {
+        mLoadedScripts.emplace(container.mThis, sMinLoadedFrames + mFrame);
+    }
+
+    void ScriptTracker::unloadInactiveScripts(LuaView& lua)
+    {
+        // This code is technically incorrect if mFrame overflows... but at 300fps that takes about half a year
+        std::size_t toProcess = std::max(mLoadedScripts.size() / sToProcessDiv, sMinToProcess);
+        while (toProcess && !mLoadedScripts.empty())
+        {
+            --toProcess;
+            auto [ptr, ttl] = std::move(mLoadedScripts.front());
+            mLoadedScripts.pop();
+            ScriptsContainer* container = *ptr.get();
+            // Object no longer exists, cease tracking
+            if (!container)
+                continue;
+            // Ignore activity of local scripts in the active grid
+            if (container->isActive())
+                ttl = std::max(ttl, mFrame + sMinLoadedFrames);
+            else
+            {
+                bool activeSinceLastPop = container->mRequiredLoading;
+                if (activeSinceLastPop)
+                {
+                    container->mRequiredLoading = false;
+                    ttl = std::min(ttl + sUsageFrameGrowth, mFrame + sMaxLoadedFrames);
+                }
+                else if (ttl < mFrame)
+                {
+                    container->ensureUnloaded(lua);
+                    continue;
+                }
+            }
+            mLoadedScripts.emplace(std::move(ptr), ttl);
+        }
+        ++mFrame;
+    }
+}

components/lua/scripttracker.hpp

@@ -0,0 +1,28 @@
+#ifndef COMPONENTS_LUA_SCRIPTTRACKER_H
+#define COMPONENTS_LUA_SCRIPTTRACKER_H
+
+#include <memory>
+#include <queue>
+#include <utility>
+
+#include "scriptscontainer.hpp"
+
+namespace LuaUtil
+{
+    class ScriptTracker
+    {
+        using Frame = unsigned int;
+        using TrackedScriptContainer = std::pair<ScriptsContainer::WeakPtr, Frame>;
+        std::queue<TrackedScriptContainer> mLoadedScripts;
+        Frame mFrame = 0;
+
+    public:
+        void unloadInactiveScripts(LuaView& lua);
+
+        void onLoad(ScriptsContainer& container);
+
+        std::size_t size() const { return mLoadedScripts.size(); }
+    };
+}
+
+#endif // COMPONENTS_LUA_SCRIPTTRACKER_H