Saving/loading for Lua scripts (saves format is changed)

apps/openmw/CMakeLists.txt

@@ -56,7 +56,7 @@ add_openmw_dir (mwscript
     )
 
 add_openmw_dir (mwlua
-    luamanagerimp localscripts object worldview luabindings userdataserializer
+    luamanagerimp localscripts object worldview luabindings userdataserializer eventqueue
     objectbindings asyncbindings
     )
 

apps/openmw/mwbase/luamanager.hpp

@@ -8,6 +8,18 @@ namespace MWWorld
     class Ptr;
 }
 
+namespace Loading
+{
+    class Listener;
+}
+
+namespace ESM
+{
+    class ESMReader;
+    class ESMWriter;
+    class LuaScripts;
+}
+
 namespace MWBase
 {
 
@@ -35,6 +47,18 @@ namespace MWBase
 
         virtual void clear() = 0;
         virtual void setupPlayer(const MWWorld::Ptr&) = 0;
+
+        // Saving
+        int countSavedGameRecords() const { return 1; };
+        virtual void write(ESM::ESMWriter& writer, Loading::Listener& progress) = 0;
+        virtual void saveLocalScripts(const MWWorld::Ptr& ptr, ESM::LuaScripts& data) = 0;
+
+        // Loading from a save
+        virtual void readRecord(ESM::ESMReader& reader, uint32_t type) = 0;
+        virtual void loadLocalScripts(const MWWorld::Ptr& ptr, const ESM::LuaScripts& data) = 0;
+
+        // Should be called before loading. The map is used to fix refnums if the order of content files was changed.
+        virtual void setContentFileMapping(const std::map<int, int>&) = 0;
     };
 
 }

apps/openmw/mwlua/eventqueue.cpp

@@ -0,0 +1,63 @@
+#include "eventqueue.hpp"
+
+#include <components/debug/debuglog.hpp>
+
+#include <components/esm/esmreader.hpp>
+#include <components/esm/esmwriter.hpp>
+#include <components/esm/luascripts.hpp>
+
+#include <components/lua/serialization.hpp>
+
+namespace MWLua
+{
+
+    template <typename Event>
+    void saveEvent(ESM::ESMWriter& esm, const ObjectId& dest, const Event& event)
+    {
+        esm.writeHNString("LUAE", event.eventName);
+        dest.save(esm, true);
+        if (!event.eventData.empty())
+            saveLuaBinaryData(esm, event.eventData);
+    }
+
+    void loadEvents(sol::state& lua, ESM::ESMReader& esm, GlobalEventQueue& globalEvents, LocalEventQueue& localEvents,
+                    const std::map<int, int>& contentFileMapping, const LuaUtil::UserdataSerializer* serializer)
+    {
+        while (esm.isNextSub("LUAE"))
+        {
+            std::string name = esm.getHString();
+            ObjectId dest;
+            dest.load(esm, true);
+            std::string data = loadLuaBinaryData(esm);
+            try
+            {
+                data = LuaUtil::serialize(LuaUtil::deserialize(lua, data, serializer), serializer);
+            }
+            catch (std::exception& e)
+            {
+                Log(Debug::Error) << "loadEvent: invalid event data: " << e.what();
+            }
+            if (dest.isSet())
+            {
+                auto it = contentFileMapping.find(dest.mContentFile);
+                if (it != contentFileMapping.end())
+                    dest.mContentFile = it->second;
+                localEvents.push_back({dest, std::move(name), std::move(data)});
+            }
+            else
+                globalEvents.push_back({std::move(name), std::move(data)});
+        }
+    }
+
+    void saveEvents(ESM::ESMWriter& esm, const GlobalEventQueue& globalEvents, const LocalEventQueue& localEvents)
+    {
+        ObjectId globalId;
+        globalId.unset();  // Used as a marker of a global event.
+
+        for (const GlobalEvent& e : globalEvents)
+            saveEvent(esm, globalId, e);
+        for (const LocalEvent& e : localEvents)
+            saveEvent(esm, e.dest, e);
+    }
+
+}

apps/openmw/mwlua/eventqueue.hpp

@@ -3,6 +3,22 @@
 
 #include "object.hpp"
 
+namespace ESM
+{
+    class ESMReader;
+    class ESMWriter;
+}
+
+namespace LuaUtil
+{
+    class UserdataSerializer;
+}
+
+namespace sol
+{
+    class state;
+}
+
 namespace MWLua
 {
     struct GlobalEvent
@@ -18,6 +34,10 @@ namespace MWLua
     };
     using GlobalEventQueue = std::vector<GlobalEvent>;
     using LocalEventQueue = std::vector<LocalEvent>;
+
+    void loadEvents(sol::state& lua, ESM::ESMReader& esm, GlobalEventQueue&, LocalEventQueue&,
+                    const std::map<int, int>& contentFileMapping, const LuaUtil::UserdataSerializer* serializer);
+    void saveEvents(ESM::ESMWriter& esm, const GlobalEventQueue&, const LocalEventQueue&);
 }
 
 #endif // MWLUA_EVENTQUEUE_H

apps/openmw/mwlua/luamanagerimp.cpp

@@ -1,6 +1,11 @@
 #include "luamanagerimp.hpp"
 
 #include <components/debug/debuglog.hpp>
+
+#include <components/esm/esmreader.hpp>
+#include <components/esm/esmwriter.hpp>
+#include <components/esm/luascripts.hpp>
+
 #include <components/lua/utilpackage.hpp>
 
 #include "../mwworld/class.hpp"
@@ -15,8 +20,12 @@ namespace MWLua
     LuaManager::LuaManager(const VFS::Manager* vfs) : mLua(vfs)
     {
         Log(Debug::Info) << "Lua version: " << LuaUtil::getLuaVersion();
+
         mGlobalSerializer = createUserdataSerializer(false, mWorldView.getObjectRegistry());
         mLocalSerializer = createUserdataSerializer(true, mWorldView.getObjectRegistry());
+        mGlobalLoader = createUserdataSerializer(false, mWorldView.getObjectRegistry(), &mContentFileMapping);
+        mLocalLoader = createUserdataSerializer(true, mWorldView.getObjectRegistry(), &mContentFileMapping);
+
         mGlobalScripts.setSerializer(mGlobalSerializer.get());
 
         Context context;
@@ -210,4 +219,56 @@ namespace MWLua
         return refData.getLuaScripts();
     }
 
+    void LuaManager::write(ESM::ESMWriter& writer, Loading::Listener& progress)
+    {
+        writer.startRecord(ESM::REC_LUAM);
+
+        mWorldView.save(writer);
+        ESM::LuaScripts globalScripts;
+        mGlobalScripts.save(globalScripts);
+        globalScripts.save(writer);
+        saveEvents(writer, mGlobalEvents, mLocalEvents);
+
+        writer.endRecord(ESM::REC_LUAM);
+    }
+
+    void LuaManager::readRecord(ESM::ESMReader& reader, uint32_t type)
+    {
+        if (type != ESM::REC_LUAM)
+            throw std::runtime_error("ESM::REC_LUAM is expected");
+
+        mWorldView.load(reader);
+        ESM::LuaScripts globalScripts;
+        globalScripts.load(reader);
+        loadEvents(mLua.sol(), reader, mGlobalEvents, mLocalEvents, mContentFileMapping, mGlobalLoader.get());
+
+        mGlobalScripts.setSerializer(mGlobalLoader.get());
+        mGlobalScripts.load(globalScripts, false);
+        mGlobalScripts.setSerializer(mGlobalSerializer.get());
+    }
+
+    void LuaManager::saveLocalScripts(const MWWorld::Ptr& ptr, ESM::LuaScripts& data)
+    {
+        if (ptr.getRefData().getLuaScripts())
+            ptr.getRefData().getLuaScripts()->save(data);
+        else
+            data.mScripts.clear();
+    }
+
+    void LuaManager::loadLocalScripts(const MWWorld::Ptr& ptr, const ESM::LuaScripts& data)
+    {
+        if (data.mScripts.empty())
+        {
+            if (ptr.getRefData().getLuaScripts())
+                ptr.getRefData().setLuaScripts(nullptr);
+            return;
+        }
+
+        mWorldView.getObjectRegistry()->registerPtr(ptr);
+        LocalScripts* scripts = createLocalScripts(ptr);
+
+        scripts->setSerializer(mLocalLoader.get());
+        scripts->load(data, true);
+        scripts->setSerializer(mLocalSerializer.get());
+    }
 }

apps/openmw/mwlua/luamanagerimp.hpp

@@ -49,6 +49,15 @@ namespace MWLua
         // Used only in luabindings.cpp
         void addLocalScript(const MWWorld::Ptr&, const std::string& scriptPath);
 
+        // Saving
+        void write(ESM::ESMWriter& writer, Loading::Listener& progress) override;
+        void saveLocalScripts(const MWWorld::Ptr& ptr, ESM::LuaScripts& data) override;
+
+        // Loading from a save
+        void readRecord(ESM::ESMReader& reader, uint32_t type) override;
+        void loadLocalScripts(const MWWorld::Ptr& ptr, const ESM::LuaScripts& data) override;
+        void setContentFileMapping(const std::map<int, int>& mapping) override { mContentFileMapping = mapping; }
+
     private:
         LocalScripts* createLocalScripts(const MWWorld::Ptr& ptr);
 
@@ -69,6 +78,10 @@ namespace MWLua
         std::unique_ptr<LuaUtil::UserdataSerializer> mGlobalSerializer;
         std::unique_ptr<LuaUtil::UserdataSerializer> mLocalSerializer;
 
+        std::map<int, int> mContentFileMapping;
+        std::unique_ptr<LuaUtil::UserdataSerializer> mGlobalLoader;
+        std::unique_ptr<LuaUtil::UserdataSerializer> mLocalLoader;
+
         std::vector<SDL_Keysym> mKeyPressEvents;
         std::vector<ObjectId> mActorAddedEvents;
     };

apps/openmw/mwlua/userdataserializer.cpp

@@ -11,8 +11,8 @@ namespace MWLua
     class Serializer final : public LuaUtil::UserdataSerializer
     {
     public:
-        explicit Serializer(bool localSerializer, ObjectRegistry* registry)
+        explicit Serializer(bool localSerializer, ObjectRegistry* registry, std::map<int, int>* contentFileMapping)
-            : mLocalSerializer(localSerializer), mObjectRegistry(registry) {}
+            : mLocalSerializer(localSerializer), mObjectRegistry(registry), mContentFileMapping(contentFileMapping) {}
 
     private:
         // Appends serialized sol::userdata to the end of BinaryData.
@@ -43,6 +43,12 @@ namespace MWLua
                 std::memcpy(&id, binaryData.data(), sizeof(ObjectId));
                 id.mIndex = Misc::fromLittleEndian(id.mIndex);
                 id.mContentFile = Misc::fromLittleEndian(id.mContentFile);
+                if (id.hasContentFile() && mContentFileMapping)
+                {
+                    auto iter = mContentFileMapping->find(id.mContentFile);
+                    if (iter != mContentFileMapping->end())
+                        id.mContentFile = iter->second;
+                }
                 if (mLocalSerializer)
                     sol::stack::push<LObject>(lua, LObject(id, mObjectRegistry));
                 else
@@ -54,11 +60,13 @@ namespace MWLua
 
         bool mLocalSerializer;
         ObjectRegistry* mObjectRegistry;
+        std::map<int, int>* mContentFileMapping;
     };
 
-    std::unique_ptr<LuaUtil::UserdataSerializer> createUserdataSerializer(bool local, ObjectRegistry* registry)
+    std::unique_ptr<LuaUtil::UserdataSerializer> createUserdataSerializer(
+        bool local, ObjectRegistry* registry, std::map<int, int>* contentFileMapping)
     {
-        return std::make_unique<Serializer>(local, registry);
+        return std::make_unique<Serializer>(local, registry, contentFileMapping);
     }
 
 }

apps/openmw/mwlua/userdataserializer.hpp

@@ -13,7 +13,10 @@ namespace MWLua
     // UserdataSerializer is an extension for components/lua/serialization.hpp
     // Needed to serialize references to objects.
     // If local=true, then during deserialization creates LObject, otherwise creates GObject.
-    std::unique_ptr<LuaUtil::UserdataSerializer> createUserdataSerializer(bool local, ObjectRegistry* registry);
+    // contentFileMapping is used only for deserialization. Needed to fix references if the order
+    // of content files was changed.
+    std::unique_ptr<LuaUtil::UserdataSerializer> createUserdataSerializer(
+        bool local, ObjectRegistry* registry, std::map<int, int>* contentFileMapping = nullptr);
 }
 
 #endif // MWLUA_USERDATASERIALIZER_H

apps/openmw/mwlua/worldview.cpp

@@ -1,5 +1,8 @@
 #include "worldview.hpp"
 
+#include <components/esm/esmreader.hpp>
+#include <components/esm/esmwriter.hpp>
+
 #include "../mwworld/class.hpp"
 #include "../mwworld/timestamp.hpp"
 
@@ -43,6 +46,20 @@ namespace MWLua
         return static_cast<double>(timeStamp.getDay()) * 24 + timeStamp.getHour();
     }
 
+    void WorldView::load(ESM::ESMReader& esm)
+    {
+        esm.getHNT(mGameSeconds, "LUAW");
+        ObjectId lastAssignedId;
+        lastAssignedId.load(esm, true);
+        mObjectRegistry.setLastAssignedId(lastAssignedId);
+    }
+
+    void WorldView::save(ESM::ESMWriter& esm) const
+    {
+        esm.writeHNT("LUAW", mGameSeconds);
+        mObjectRegistry.getLastAssignedId().save(esm, true);
+    }
+
     void WorldView::ObjectGroup::updateList()
     {
         if (mChanged)

apps/openmw/mwlua/worldview.hpp

@@ -3,6 +3,12 @@
 
 #include "object.hpp"
 
+namespace ESM
+{
+    class ESMWriter;
+    class ESMReader;
+}
+
 namespace MWLua
 {
 
@@ -31,6 +37,9 @@ namespace MWLua
         void objectAddedToScene(const MWWorld::Ptr& ptr);
         void objectRemovedFromScene(const MWWorld::Ptr& ptr);
 
+        void load(ESM::ESMReader& esm);
+        void save(ESM::ESMWriter& esm) const;
+
     private:
         struct ObjectGroup
         {

apps/openmw/mwstate/statemanagerimp.cpp

@@ -251,6 +251,7 @@ void MWState::StateManager::saveGame (const std::string& description, const Slot
 
         int recordCount =         1 // saved game header
                 +MWBase::Environment::get().getJournal()->countSavedGameRecords()
+                +MWBase::Environment::get().getLuaManager()->countSavedGameRecords()
                 +MWBase::Environment::get().getWorld()->countSavedGameRecords()
                 +MWBase::Environment::get().getScriptManager()->getGlobalScripts().countSavedGameRecords()
                 +MWBase::Environment::get().getDialogueManager()->countSavedGameRecords()
@@ -274,6 +275,9 @@ void MWState::StateManager::saveGame (const std::string& description, const Slot
 
         MWBase::Environment::get().getJournal()->write (writer, listener);
         MWBase::Environment::get().getDialogueManager()->write (writer, listener);
+        // LuaManager::write should be called before World::write because world also saves
+        // local scripts that depend on LuaManager.
+        MWBase::Environment::get().getLuaManager()->write(writer, listener);
         MWBase::Environment::get().getWorld()->write (writer, listener);
         MWBase::Environment::get().getScriptManager()->getGlobalScripts().write (writer, listener);
         MWBase::Environment::get().getWindowManager()->write(writer, listener);
@@ -384,6 +388,7 @@ void MWState::StateManager::loadGame (const Character *character, const std::str
             throw std::runtime_error("This save file was created using a newer version of OpenMW and is thus not supported. Please upgrade to the newest OpenMW version to load this file.");
 
         std::map<int, int> contentFileMap = buildContentFileIndexMap (reader);
+        MWBase::Environment::get().getLuaManager()->setContentFileMapping(contentFileMap);
 
         Loading::Listener& listener = *MWBase::Environment::get().getWindowManager()->getLoadingScreen();
 
@@ -482,6 +487,10 @@ void MWState::StateManager::loadGame (const Character *character, const std::str
                     MWBase::Environment::get().getInputManager()->readRecord(reader, n.intval);
                     break;
 
+                case ESM::REC_LUAM:
+                    MWBase::Environment::get().getLuaManager()->readRecord(reader, n.intval);
+                    break;
+
                 default:
 
                     // ignore invalid records

apps/openmw/mwworld/livecellref.cpp

@@ -5,6 +5,7 @@
 
 #include "../mwbase/environment.hpp"
 #include "../mwbase/world.hpp"
+#include "../mwbase/luamanager.hpp"
 
 #include "ptr.hpp"
 #include "class.hpp"
@@ -52,6 +53,8 @@ void MWWorld::LiveCellRefBase::loadImp (const ESM::ObjectState& state)
         Log(Debug::Warning) << "Soul '" << mRef.getSoul() << "' not found, removing the soul from soul gem";
         mRef.setSoul(std::string());
     }
+
+    MWBase::Environment::get().getLuaManager()->loadLocalScripts(ptr, state.mLuaScripts);
 }
 
 void MWWorld::LiveCellRefBase::saveImp (ESM::ObjectState& state) const
@@ -61,6 +64,7 @@ void MWWorld::LiveCellRefBase::saveImp (ESM::ObjectState& state) const
     ConstPtr ptr (this);
 
     mData.write (state, mClass->getScript (ptr));
+    MWBase::Environment::get().getLuaManager()->saveLocalScripts(Ptr(const_cast<LiveCellRefBase*>(this)), state.mLuaScripts);
 
     mClass->writeAdditionalState (ptr, state);
 }

components/esm/defs.hpp

@@ -164,7 +164,10 @@ enum RecNameInts
 
     // format 1
     REC_FILT = FourCC<'F','I','L','T'>::value,
-    REC_DBGP = FourCC<'D','B','G','P'>::value ///< only used in project files
+    REC_DBGP = FourCC<'D','B','G','P'>::value, ///< only used in project files
+
+    // format 16 - Lua scripts in saved games
+    REC_LUAM = FourCC<'L','U','A','M'>::value,  // LuaManager data
 };
 
 /// Common subrecords

components/esm/objectstate.cpp

@@ -20,6 +20,8 @@ void ESM::ObjectState::load (ESMReader &esm)
     if (mHasLocals)
         mLocals.load (esm);
 
+    mLuaScripts.load(esm);
+
     mEnabled = 1;
     esm.getHNOT (mEnabled, "ENAB");
 
@@ -56,6 +58,8 @@ void ESM::ObjectState::save (ESMWriter &esm, bool inInventory) const
         mLocals.save (esm);
     }
 
+    mLuaScripts.save(esm);
+
     if (!mEnabled && !inInventory)
         esm.writeHNT ("ENAB", mEnabled);
 

components/esm/objectstate.hpp

@@ -6,6 +6,7 @@
 
 #include "cellref.hpp"
 #include "locals.hpp"
+#include "luascripts.hpp"
 #include "animationstate.hpp"
 
 namespace ESM
@@ -27,6 +28,7 @@ namespace ESM
 
         unsigned char mHasLocals;
         Locals mLocals;
+        LuaScripts mLuaScripts;
         unsigned char mEnabled;
         int mCount;
         ESM::Position mPosition;

components/esm/savedgame.cpp

@@ -4,7 +4,7 @@
 #include "esmwriter.hpp"
 
 unsigned int ESM::SavedGame::sRecordId = ESM::REC_SAVE;
-int ESM::SavedGame::sCurrentFormat = 15;
+int ESM::SavedGame::sCurrentFormat = 16;
 
 void ESM::SavedGame::load (ESMReader &esm)
 {