ESSImport: add some subrecords to CellRef and others, most files should load now, importacdt/cellref class structure need some refactoring

apps/essimporter/CMakeLists.txt

@@ -6,6 +6,7 @@ set(ESSIMPORTER_FILES
     importcrec.cpp
     importcellref.cpp
     importacdt.cpp
+    importinventory.cpp
     importercontext.cpp
     converter.cpp
     convertacdt.cpp

apps/essimporter/converter.cpp

@@ -60,12 +60,17 @@ namespace ESSImport
         // TODO: add bleeding of FOW into neighbouring cells (openmw handles this by writing to the textures,
         // MW handles it when rendering only)
         unsigned char nam8[32];
-        if (esm.isNextSub("NAM8"))
+        // exterior has 1 NAM8, interior can have multiple ones, and have an extra 4 byte flag at the start
+        // (probably offset of that specific fog texture?)
+        while (esm.isNextSub("NAM8"))
         {
             esm.getSubHeader();
-            // FIXME: different size in interior cells for some reason
+
             if (esm.getSubSize() == 36)
+            {
+                // flag on interiors
                 esm.skip(4);
+            }
 
             esm.getExact(nam8, 32);
 
@@ -82,11 +87,25 @@ namespace ESSImport
                 }
             }
 
+            if (cell.isExterior())
+            {
                 std::ostringstream filename;
                 filename << "fog_" << cell.mData.mX << "_" << cell.mData.mY << ".tga";
 
                 convertImage((char*)&newcell.mFogOfWar[0], newcell.mFogOfWar.size()*4, 16, 16, Ogre::PF_BYTE_RGBA, filename.str());
             }
+        }
+
+        // moved reference, not handled yet
+        // NOTE: MVRF can also occur in within normal references (importcellref.cpp)?
+        // this does not match the ESM file implementation,
+        // verify if that can happen with ESM files too
+        while (esm.isNextSub("MVRF"))
+        {
+            esm.skipHSub(); // skip MVRF
+            esm.getSubName();
+            esm.skipHSub(); // skip CNDT
+        }
 
         std::vector<CellRef> cellrefs;
         while (esm.hasMoreSubs())
@@ -94,7 +113,11 @@ namespace ESSImport
             CellRef ref;
             ref.load (esm);
             if (esm.isNextSub("DELE"))
+            {
+                // strangely this can be e.g. 52 instead of just 1,
                 std::cout << "deleted ref " << ref.mIndexedRefId << std::endl;
+                esm.skipHSub();
+            }
             cellrefs.push_back(ref);
         }
 
@@ -148,6 +171,7 @@ namespace ESSImport
                     objstate.mPosition = cellref.mPos;
                     objstate.mRef = out;
                     objstate.mRef.mRefNum = cellref.mRefNum;
+                    // FIXME: change save format to not require object type, instead look up it up using the RefId
                     esm.writeHNT ("OBJE", ESM::REC_CREA);
                     objstate.save(esm);
                     continue;

apps/essimporter/converter.hpp

@@ -83,14 +83,18 @@ public:
         // this is always the player
         ESM::NPC npc;
         std::string id = esm.getHNString("NAME");
-        assert (id == "player");
         npc.load(esm);
+        if (id != "player") // seems to occur sometimes, with "chargen X" names
+            std::cerr << "non-player NPC record: " << id << std::endl;
+        else
+        {
             mContext->mPlayer.mObject.mCreatureStats.mLevel = npc.mNpdt52.mLevel;
             mContext->mPlayerBase = npc;
             std::map<const int, float> empty;
             for (std::vector<std::string>::const_iterator it = npc.mSpells.mList.begin(); it != npc.mSpells.mList.end(); ++it)
                 mContext->mPlayer.mObject.mCreatureStats.mSpells.mSpells[*it] = empty;
         }
+    }
 };
 
 class ConvertGlobal : public DefaultConverter<ESM::Global>

apps/essimporter/convertnpcc.cpp

@@ -6,17 +6,5 @@ namespace ESSImport
     void convertNPCC(const NPCC &npcc, ESM::NpcState &npcState)
     {
         npcState.mNpcStats.mReputation = npcc.mNPDT.mReputation;
-
-        for (std::vector<NPCC::InventoryItem>::const_iterator it = npcc.mInventory.begin();
-             it != npcc.mInventory.end(); ++it)
-        {
-            ESM::ObjectState obj;
-            obj.blank();
-            obj.mRef.mRefID = it->mId;
-            obj.mRef.mCharge = it->mCondition;
-
-            // Don't know type of object :( change save format?
-            // npcState.mInventory.mItems.push_back(std::make_pair(obj, std::make_pair(0,0)));
-        }
     }
 }

apps/essimporter/importacdt.cpp

@@ -2,17 +2,73 @@
 
 #include <components/esm/esmreader.hpp>
 
+#include <components/esm/cellref.hpp>
+
 namespace ESSImport
 {
 
     void ActorData::load(ESM::ESMReader &esm)
     {
-        esm.getHNT(mACDT, "ACDT");
+         // unsure at which point between NAME and ESM::CellRef
+        if (esm.isNextSub("MNAM"))
+            esm.skipHSub();
+
+        if (esm.isNextSub("ACTN"))
+            esm.skipHSub();
+
+        if (esm.isNextSub("STPR"))
+            esm.skipHSub();
+
+        ESM::CellRef bla;
+        bla.ESM::CellRef::loadData(esm);
+
+        // FIXME: actually should be required for all actors?, but ActorData is currently in base CellRef
+        esm.getHNOT(mACDT, "ACDT");
 
         ACSC acsc;
         esm.getHNOT(acsc, "ACSC");
         esm.getHNOT(acsc, "ACSL");
 
+        if (esm.isNextSub("CSTN"))
+            esm.skipHSub(); // "PlayerSaveGame", link to some object?
+
+        if (esm.isNextSub("LSTN"))
+            esm.skipHSub(); // "PlayerSaveGame", link to some object?
+
+        // unsure at which point between LSTN and TGTN
+        if (esm.isNextSub("CSHN"))
+            esm.skipHSub(); // "PlayerSaveGame", link to some object?
+
+        // unsure if before or after CSTN/LSTN
+        if (esm.isNextSub("LSHN"))
+            esm.skipHSub(); // "PlayerSaveGame", link to some object?
+
+        while (esm.isNextSub("TGTN"))
+            esm.skipHSub(); // "PlayerSaveGame", link to some object?
+
+        while (esm.isNextSub("FGTN"))
+            esm.getHString(); // fight target?
+
+        // unsure at which point between FGTN and CHRD
+        if (esm.isNextSub("PWPC"))
+            esm.skipHSub();
+        if (esm.isNextSub("PWPS"))
+            esm.skipHSub();
+
+        if (esm.isNextSub("WNAM"))
+        {
+            esm.skipHSub(); // seen values: "ancestor guardian", "bound dagger_en". Summoned creature / bound weapons?
+
+            if (esm.isNextSub("XNAM"))
+            {
+                // "demon tanto", probably the ID of spell/item that created the bound weapon/crature?
+                esm.skipHSub();
+            }
+
+            if (esm.isNextSub("YNAM"))
+                esm.skipHSub(); // 4 byte, 0
+        }
+
         if (esm.isNextSub("CHRD")) // npc only
                 esm.getHExact(mSkills, 27*2*sizeof(int));
 
@@ -21,8 +77,21 @@ namespace ESSImport
 
         mScript = esm.getHNOString("SCRI");
 
+        // script variables?
+        if (!mScript.empty())
+        {
+            if (esm.isNextSub("SLCS"))
+                esm.skipHSub();
+            if (esm.isNextSub("SLSD")) // Short Data?
+                esm.skipHSub();
+            if (esm.isNextSub("SLFD")) // Float Data?
+                esm.skipHSub();
+        }
+
         if (esm.isNextSub("ND3D"))
             esm.skipHSub();
+        if (esm.isNextSub("ANIS"))
+            esm.skipHSub();
     }
 
 }

apps/essimporter/importcellref.cpp

@@ -7,7 +7,7 @@ namespace ESSImport
 
     void CellRef::load(ESM::ESMReader &esm)
     {
-        esm.getHNT(mRefNum.mIndex, "FRMR"); // TODO: adjust RefNum
+        esm.getHNT(mRefNum.mIndex, "FRMR");
 
         // this is required since openmw supports more than 255 content files
         int pluginIndex = (mRefNum.mIndex & 0xff000000) >> 24;
@@ -16,19 +16,27 @@ namespace ESSImport
 
         mIndexedRefId = esm.getHNString("NAME");
 
-        // the following two occur in ESM::CellRef too (Charge and Gold),
-        // but may have entirely different meanings here
-        int intv;
-        esm.getHNOT(intv, "INTV");
-        int nam9;
-        esm.getHNOT(nam9, "NAM9");
+        if (esm.isNextSub("LVCR"))
+            esm.skipHSub();
 
         mActorData.load(esm);
 
         mEnabled = true;
         esm.getHNOT(mEnabled, "ZNAM");
 
-        esm.getHNT(mPos, "DATA", 24);
+        // should occur for all references but not levelled creature spawners
+        esm.getHNOT(mPos, "DATA", 24);
+
+        // i've seen DATA record TWICE on a creature record - and with the exact same content too! weird
+        // alarmvoi0000.ess
+        esm.getHNOT(mPos, "DATA", 24);
+
+        if (esm.isNextSub("MVRF"))
+        {
+            esm.skipHSub();
+            esm.getSubName();
+            esm.skipHSub();
+        }
     }
 
 }

apps/essimporter/importcrec.cpp

@@ -8,6 +8,13 @@ namespace ESSImport
     void CREC::load(ESM::ESMReader &esm)
     {
         esm.getHNT(mIndex, "INDX");
+
+        // equivalent of ESM::Creature XSCL? probably don't have to convert this,
+        // since the value can't be changed
+        float scale;
+        esm.getHNOT(scale, "XSCL");
+
+        mInventory.load(esm);
     }
 
 }

apps/essimporter/importcrec.hpp

@@ -1,6 +1,8 @@
 #ifndef OPENMW_ESSIMPORT_CREC_H
 #define OPENMW_ESSIMPORT_CREC_H
 
+#include "importinventory.hpp"
+
 namespace ESM
 {
     class ESMReader;
@@ -14,6 +16,8 @@ namespace ESSImport
     {
         int mIndex;
 
+        Inventory mInventory;
+
         void load(ESM::ESMReader& esm);
     };
 

apps/essimporter/importinventory.cpp

@@ -0,0 +1,46 @@
+#include "importinventory.hpp"
+
+#include <components/esm/esmreader.hpp>
+
+namespace ESSImport
+{
+
+    void Inventory::load(ESM::ESMReader &esm)
+    {
+        while (esm.isNextSub("NPCO"))
+        {
+            InventoryItem item;
+            item.mId = esm.getHString();
+
+            if (esm.isNextSub("XIDX"))
+                esm.skipHSub();
+
+            std::string script = esm.getHNOString("SCRI");
+            // script variables?
+            // unsure if before or after ESM::CellRef
+            if (!script.empty())
+            {
+                if (esm.isNextSub("SLCS"))
+                    esm.skipHSub();
+                if (esm.isNextSub("SLSD")) // Short Data?
+                    esm.skipHSub();
+                if (esm.isNextSub("SLFD")) // Float Data?
+                    esm.skipHSub();
+            }
+
+            // for XSOL and XCHG seen so far, but probably others too
+            item.ESM::CellRef::loadData(esm);
+
+            item.mCondition = -1;
+            esm.getHNOT(item.mCondition, "XHLT");
+            mItems.push_back(item);
+        }
+
+        while (esm.isNextSub("WIDX"))
+        {
+            // equipping?
+            esm.skipHSub();
+        }
+    }
+
+}

apps/essimporter/importinventory.hpp

@@ -0,0 +1,31 @@
+#ifndef OPENMW_ESSIMPORT_IMPORTINVENTORY_H
+#define OPENMW_ESSIMPORT_IMPORTINVENTORY_H
+
+#include <vector>
+#include <string>
+
+#include <components/esm/cellref.hpp>
+
+namespace ESM
+{
+    class ESMReader;
+}
+
+namespace ESSImport
+{
+
+    struct Inventory
+    {
+        struct InventoryItem : public ESM::CellRef
+        {
+            std::string mId;
+            int mCondition;
+        };
+        std::vector<InventoryItem> mItems;
+
+        void load(ESM::ESMReader& esm);
+    };
+
+}
+
+#endif

apps/essimporter/importnpcc.cpp

@@ -12,24 +12,14 @@ namespace ESSImport
 
         esm.getHNT(mNPDT, "NPDT");
 
-        while (esm.isNextSub("NPCO"))
-        {
-            InventoryItem item;
-            item.mId = esm.getHString();
-
-            if (esm.isNextSub("XIDX"))
+        if (esm.isNextSub("AI_E"))
             esm.skipHSub();
-
-            item.mCondition = -1;
-            esm.getHNOT(item.mCondition, "XHLT");
-            mInventory.push_back(item);
-        }
-
-        while (esm.isNextSub("WIDX"))
-        {
-            // equipping?
+        if (esm.isNextSub("AI_T"))
             esm.skipHSub();
-        }
+        if (esm.isNextSub("AI_F"))
+            esm.skipHSub();
+
+        mInventory.load(esm);
     }
 
 }

apps/essimporter/importnpcc.hpp

@@ -3,6 +3,10 @@
 
 #include <components/esm/loadcont.hpp>
 
+#include <components/esm/aipackage.hpp>
+
+#include "importinventory.hpp"
+
 namespace ESM
 {
     class ESMReader;
@@ -20,12 +24,7 @@ namespace ESSImport
             unsigned char unknown2[5];
         } mNPDT;
 
-        struct InventoryItem
-        {
-            std::string mId;
-            int mCondition;
-        };
-        std::vector<InventoryItem> mInventory;
+        Inventory mInventory;
 
         int mIndex;
 

apps/essimporter/importplayer.cpp

@@ -11,9 +11,6 @@ namespace ESSImport
 
         mRefID = esm.getHNString("NAME");
 
-        if (esm.isNextSub("STPR"))
-            esm.skipHSub(); // ESS TODO
-
         mActorData.load(esm);
 
         esm.getHNOT(mPos, "DATA", 24);
@@ -21,6 +18,12 @@ namespace ESSImport
 
     void PCDT::load(ESM::ESMReader &esm)
     {
+        while (esm.isNextSub("DNAM"))
+        {
+            // TODO: deal with encoding?
+            mKnownDialogueTopics.push_back(esm.getHString());
+        }
+
         if (esm.isNextSub("PNAM"))
             esm.skipHSub();
         if (esm.isNextSub("SNAM"))
@@ -33,6 +36,17 @@ namespace ESSImport
 
         mBirthsign = esm.getHNOString("BNAM");
 
+        // Holds the names of the last used Alchemy apparatus. Don't need to import this ATM,
+        // because our GUI auto-selects the best apparatus.
+        if (esm.isNextSub("NAM0"))
+            esm.skipHSub();
+        if (esm.isNextSub("NAM1"))
+            esm.skipHSub();
+        if (esm.isNextSub("NAM2"))
+            esm.skipHSub();
+        if (esm.isNextSub("NAM3"))
+            esm.skipHSub();
+
         if (esm.isNextSub("ENAM"))
             esm.skipHSub();
 
@@ -48,6 +62,14 @@ namespace ESSImport
 
         if (esm.isNextSub("KNAM"))
             esm.skipHSub();
+
+        if (esm.isNextSub("WERE"))
+        {
+            // some werewolf data, 152 bytes
+            // maybe current skills and attributes for werewolf form
+            esm.getSubHeader();
+            esm.skip(152);
+        }
     }
 
 }

apps/essimporter/importplayer.hpp

@@ -36,6 +36,8 @@ struct PCDT
     int mBounty;
     std::string mBirthsign;
 
+    std::vector<std::string> mKnownDialogueTopics;
+
     struct FNAM
     {
         unsigned char mRank;

components/esm/cellref.cpp

@@ -20,6 +20,11 @@ void ESM::CellRef::load (ESMReader& esm, bool wideRefNum)
 
     mRefID = esm.getHNString ("NAME");
 
+    loadData(esm);
+}
+
+void ESM::CellRef::loadData(ESMReader &esm)
+{
     // Again, UNAM sometimes appears after NAME and sometimes later.
     // Or perhaps this UNAM means something different?
     mReferenceBlocked = -1;

components/esm/cellref.hpp

@@ -91,6 +91,9 @@ namespace ESM
 
             void load (ESMReader& esm, bool wideRefNum = false);
 
+            /// Implicitly called by load
+            void loadData (ESMReader& esm);
+
             void save (ESMWriter &esm, bool wideRefNum = false, bool inInventory = false) const;
 
             void blank();