OpenMW Integration.

Read the ESM/ESP records but do nothing with them for the moment.

CMakeLists.txt

@@ -580,6 +580,7 @@ add_subdirectory (extern/shiny)
 add_subdirectory (extern/ogre-ffmpeg-videoplayer)
 add_subdirectory (extern/oics)
 add_subdirectory (extern/sdl4ogre)
+add_subdirectory (extern/esm4)
 add_subdirectory (extern/murmurhash)
 add_subdirectory (extern/BSAOpt)
 

apps/openmw/CMakeLists.txt

@@ -117,6 +117,7 @@ target_link_libraries(openmw
     ${OGRE_LIBRARIES}
     ${OGRE_STATIC_PLUGINS}
     ${SHINY_LIBRARIES}
+    ${ESM4_LIBRARIES}
     ${BSAOPTHASH_LIBRARIES}
     ${ZLIB_LIBRARY}
     ${OPENAL_LIBRARY}

apps/openmw/mwworld/esmloader.cpp

@@ -2,6 +2,7 @@
 #include "esmstore.hpp"
 
 #include <components/esm/esmreader.hpp>
+#include <components/esm/esm4reader.hpp>
 
 namespace MWWorld
 {
@@ -42,8 +43,22 @@ void EsmLoader::load(const boost::filesystem::path& filepath, std::vector<std::v
         else
             tesVerIndex = 3;
 
-        // do nothing for now
+        lEsm->close();
-        return;
+        delete lEsm;
+        ESM::ESM4Reader *esm = new ESM::ESM4Reader(isTes4); // NOTE: TES4 headers are 4 bytes shorter
+        esm->setEncoder(mEncoder);
+
+        index = contentFiles[tesVerIndex].size();
+        contentFiles[tesVerIndex].push_back(filepath.filename().string());
+        esm->setIndex(index);
+
+        esm->reader().setModIndex(index);
+        esm->openTes4File(filepath.string());
+        esm->reader().updateModIndicies(contentFiles[tesVerIndex]);
+        // FIXME: this does not work well (copies the base class pointer)
+        //i.e. have to check TES4/TES5 versions each time before use within EsmStore::load,
+        //static casting as required
+        mEsm[tesVerIndex].push_back(esm);
     }
     else
     {

apps/openmw/mwworld/esmstore.cpp

@@ -8,6 +8,8 @@
 #include <components/loadinglistener/loadinglistener.hpp>
 
 #include <components/esm/esmreader.hpp>
+#include <components/esm/esmwriter.hpp>
+#include <components/esm/esm4reader.hpp>
 
 namespace MWWorld
 {
@@ -82,6 +84,16 @@ void ESMStore::load(ESM::ESMReader &esm, Loading::Listener* listener)
     // Loop through all records
     while(esm.hasMoreRecs())
     {
+        if (isTes4 || isTes5 || isFONV)
+        {
+            ESM4::Reader& reader = static_cast<ESM::ESM4Reader*>(&esm)->reader();
+            reader.checkGroupStatus();
+
+            loadTes4Group(esm);
+            listener->setProgress(static_cast<size_t>(esm.getFileOffset() / (float)esm.getFileSize() * 1000));
+            continue;
+        }
+
         ESM::NAME n = esm.getRecName();
         esm.getRecHeader();
 
@@ -136,6 +148,138 @@ void ESMStore::load(ESM::ESMReader &esm, Loading::Listener* listener)
     }
 }
 
+// Can't use ESM4::Reader& as the parameter here because we need esm.hasMoreRecs() for
+// checking an empty group followed by EOF
+void ESMStore::loadTes4Group (ESM::ESMReader &esm)
+{
+    ESM4::Reader& reader = static_cast<ESM::ESM4Reader*>(&esm)->reader();
+
+    reader.getRecordHeader();
+    const ESM4::RecordHeader& hdr = reader.hdr();
+
+    if (hdr.record.typeId != ESM4::REC_GRUP)
+        return loadTes4Record(esm);
+
+    switch (hdr.group.type)
+    {
+        case ESM4::Grp_RecordType:
+        {
+            // FIXME: rewrite to workaround reliability issue
+            if (hdr.group.label.value == ESM4::REC_NAVI || hdr.group.label.value == ESM4::REC_WRLD ||
+                hdr.group.label.value == ESM4::REC_REGN || hdr.group.label.value == ESM4::REC_STAT ||
+                hdr.group.label.value == ESM4::REC_ANIO || hdr.group.label.value == ESM4::REC_CONT ||
+                hdr.group.label.value == ESM4::REC_MISC || hdr.group.label.value == ESM4::REC_ACTI ||
+                hdr.group.label.value == ESM4::REC_ARMO || hdr.group.label.value == ESM4::REC_NPC_ ||
+                hdr.group.label.value == ESM4::REC_FLOR || hdr.group.label.value == ESM4::REC_GRAS ||
+                hdr.group.label.value == ESM4::REC_TREE || hdr.group.label.value == ESM4::REC_LIGH ||
+                hdr.group.label.value == ESM4::REC_BOOK || hdr.group.label.value == ESM4::REC_FURN ||
+                hdr.group.label.value == ESM4::REC_SOUN || hdr.group.label.value == ESM4::REC_WEAP ||
+                hdr.group.label.value == ESM4::REC_DOOR || hdr.group.label.value == ESM4::REC_AMMO ||
+                hdr.group.label.value == ESM4::REC_CLOT || hdr.group.label.value == ESM4::REC_ALCH ||
+                hdr.group.label.value == ESM4::REC_APPA || hdr.group.label.value == ESM4::REC_INGR ||
+                hdr.group.label.value == ESM4::REC_SGST || hdr.group.label.value == ESM4::REC_SLGM ||
+                hdr.group.label.value == ESM4::REC_KEYM || hdr.group.label.value == ESM4::REC_HAIR ||
+                hdr.group.label.value == ESM4::REC_EYES || hdr.group.label.value == ESM4::REC_CELL ||
+                hdr.group.label.value == ESM4::REC_CREA || hdr.group.label.value == ESM4::REC_LVLC ||
+                hdr.group.label.value == ESM4::REC_LVLI || hdr.group.label.value == ESM4::REC_MATO ||
+                hdr.group.label.value == ESM4::REC_IDLE || hdr.group.label.value == ESM4::REC_LTEX ||
+                hdr.group.label.value == ESM4::REC_RACE || hdr.group.label.value == ESM4::REC_SBSP
+                )
+            {
+                reader.saveGroupStatus();
+                loadTes4Group(esm);
+            }
+            else
+            {
+                // Skip groups that are of no interest (for now).
+                //  GMST GLOB CLAS FACT SKIL MGEF SCPT ENCH SPEL BSGN WTHR CLMT DIAL
+                //  QUST PACK CSTY LSCR LVSP WATR EFSH
+
+                // FIXME: The label field of a group is not reliable, so we will need to check here as well
+                //std::cout << "skipping group... " << ESM4::printLabel(hdr.group.label, hdr.group.type) << std::endl;
+                reader.skipGroup();
+                return;
+            }
+
+            break;
+        }
+        case ESM4::Grp_CellChild:
+        case ESM4::Grp_WorldChild:
+        case ESM4::Grp_TopicChild:
+        case ESM4::Grp_CellPersistentChild:
+        {
+            reader.adjustGRUPFormId();  // not needed or even shouldn't be done? (only labels anyway)
+            reader.saveGroupStatus();
+//#if 0
+            // Below test shows that Oblivion.esm does not have any persistent cell child
+            // groups under exterior world sub-block group.  Haven't checked other files yet.
+             if (reader.grp(0).type == ESM4::Grp_CellPersistentChild &&
+                 reader.grp(1).type == ESM4::Grp_CellChild &&
+                 !(reader.grp(2).type == ESM4::Grp_WorldChild || reader.grp(2).type == ESM4::Grp_InteriorSubCell))
+                 std::cout << "Unexpected persistent child group in exterior subcell" << std::endl;
+//#endif
+            if (!esm.hasMoreRecs())
+                return; // may have been an empty group followed by EOF
+
+            loadTes4Group(esm);
+
+            break;
+        }
+        case ESM4::Grp_CellTemporaryChild:
+        case ESM4::Grp_CellVisibleDistChild:
+        {
+            // NOTE: preload strategy and persistent records
+            //
+            // Current strategy defers loading of "temporary" or "visible when distant"
+            // references and other records (land and pathgrid) until they are needed.
+            //
+            // The "persistent" records need to be loaded up front, however.  This is to allow,
+            // for example, doors to work.  A door reference will have a FormId of the
+            // destination door FormId.  But we have no way of knowing to which cell the
+            // destination FormId belongs until that cell and that reference is loaded.
+            //
+            // For worldspaces the persistent records are usully (always?) stored in a dummy
+            // cell under a "world child" group.  It may be possible to skip the whole "cell
+            // child" group without scanning for persistent records.  See above short test.
+            reader.skipGroup();
+            break;
+        }
+        case ESM4::Grp_ExteriorCell:
+        case ESM4::Grp_ExteriorSubCell:
+        case ESM4::Grp_InteriorCell:
+        case ESM4::Grp_InteriorSubCell:
+        {
+            reader.saveGroupStatus();
+            loadTes4Group(esm);
+
+            break;
+        }
+        default:
+            reader.skipGroup();
+            break;
+    }
+
+    return;
+}
+
+void ESMStore::loadTes4Record (ESM::ESMReader& esm)
+{
+    // Assumes that the reader has just read the record header only.
+    ESM4::Reader& reader = static_cast<ESM::ESM4Reader*>(&esm)->reader();
+    const ESM4::RecordHeader& hdr = reader.hdr();
+
+    switch (hdr.record.typeId)
+    {
+
+        // FIXME: removed for now
+
+        default:
+            reader.skipRecordData();
+    }
+
+    return;
+}
+
 void ESMStore::setUp()
 {
     std::map<int, StoreBase *>::iterator it = mStores.begin();
@@ -213,7 +357,7 @@ void ESMStore::setUp()
                 if (type==ESM::REC_NPC_)
                 {
                     // NPC record will always be last and we know that there can be only one
-                    // dynamic NPC record (player) -> We are done here with dynamic record laoding
+                    // dynamic NPC record (player) -> We are done here with dynamic record loading
                     setUp();
 
                     const ESM::NPC *player = mNpcs.find ("player");

apps/openmw/mwworld/esmstore.hpp

@@ -6,6 +6,12 @@
 #include <components/esm/records.hpp>
 #include "store.hpp"
 
+namespace ESM4
+{
+    class Reader;
+    union RecordHeader;
+}
+
 namespace Loading
 {
     class Listener;
@@ -73,6 +79,9 @@ namespace MWWorld
 
         unsigned int mDynamicCount;
 
+        void loadTes4Group (ESM::ESMReader& esm);
+        void loadTes4Record (ESM::ESMReader& esm);
+
     public:
         /// \todo replace with SharedIterator<StoreBase>
         typedef std::map<int, StoreBase *>::const_iterator iterator;

components/CMakeLists.txt

@@ -6,16 +6,16 @@ set (VERSION_HPP ${CMAKE_CURRENT_SOURCE_DIR}/version/version.hpp)
 if (GIT_CHECKOUT)
     add_custom_target (git-version
         COMMAND ${CMAKE_COMMAND}
-	    -DGIT_EXECUTABLE=${GIT_EXECUTABLE}
+        -DGIT_EXECUTABLE=${GIT_EXECUTABLE}
-            -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}
+        -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}
-	    -DVERSION_HPP_IN=${VERSION_HPP_IN}
+        -DVERSION_HPP_IN=${VERSION_HPP_IN}
-	    -DVERSION_HPP=${VERSION_HPP}
+        -DVERSION_HPP=${VERSION_HPP}
-	    -DOPENMW_VERSION_MAJOR=${OPENMW_VERSION_MAJOR}
+        -DOPENMW_VERSION_MAJOR=${OPENMW_VERSION_MAJOR}
-	    -DOPENMW_VERSION_MINOR=${OPENMW_VERSION_MINOR}
+        -DOPENMW_VERSION_MINOR=${OPENMW_VERSION_MINOR}
-	    -DOPENMW_VERSION_RELEASE=${OPENMW_VERSION_RELEASE}
+        -DOPENMW_VERSION_RELEASE=${OPENMW_VERSION_RELEASE}
-	    -DOPENMW_VERSION=${OPENMW_VERSION}
+        -DOPENMW_VERSION=${OPENMW_VERSION}
-            -P ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/GitVersion.cmake
+        -P ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/GitVersion.cmake
-	    VERBATIM)
+        VERBATIM)
 else (GIT_CHECKOUT)
     configure_file(${VERSION_HPP_IN} ${VERSION_HPP})
 endif (GIT_CHECKOUT)
@@ -62,7 +62,7 @@ add_component_dir (esm
     loadweap records aipackage effectlist spelllist variant variantimp loadtes3 cellref filter
     savedgame journalentry queststate locals globalscript player objectstate cellid cellstate globalmap inventorystate containerstate npcstate creaturestate dialoguestate statstate
     npcstats creaturestats weatherstate quickkeys fogstate spellstate activespells creaturelevliststate doorstate projectilestate debugprofile
-    aisequence magiceffects util custommarkerstate stolenitems transport
+    aisequence magiceffects util custommarkerstate stolenitems transport esm4reader
     )
 
 add_component_dir (esmterrain
@@ -165,10 +165,10 @@ include_directories(${BULLET_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR})
 
 add_library(components STATIC ${COMPONENT_FILES} ${MOC_SRCS} ${ESM_UI_HDR})
 
-target_link_libraries(components 
+target_link_libraries(components
-    ${Boost_LIBRARIES} 
+    ${Boost_LIBRARIES}
     ${OGRE_LIBRARIES}
-    ${OENGINE_LIBRARY}    
+    ${OENGINE_LIBRARY}
 )
 
 if (WIN32)

components/esm/esm4reader.cpp

@@ -0,0 +1,94 @@
+#include "esm4reader.hpp"
+
+ESM::ESM4Reader::ESM4Reader(bool oldHeader)
+{
+    // TES4 header size is 4 bytes smaller than TES5 header
+    mReader.setRecHeaderSize(oldHeader ? sizeof(ESM4::RecordHeader)-4 : sizeof(ESM4::RecordHeader));
+}
+
+ESM::ESM4Reader::~ESM4Reader()
+{
+}
+
+void ESM::ESM4Reader::openTes4File(const std::string &name)
+{
+    mCtx.filename = name;
+    // WARNING: may throw
+    mCtx.leftFile = mReader.openTes4File(name);
+    mReader.registerForUpdates(this); // for updating mCtx.leftFile
+
+    mReader.getRecordHeader();
+    if (mReader.hdr().record.typeId == ESM4::REC_TES4)
+    {
+        mReader.loadHeader();
+        mCtx.leftFile -= mReader.hdr().record.dataSize;
+
+        // Hack: copy over values to TES3 header for getVer() and getRecordCount() to work
+        mHeader.mData.version = mReader.esmVersion();
+        mHeader.mData.records = mReader.numRecords();
+
+        mReader.buildLStringIndex(); // for localised strings in Skyrim
+    }
+    else
+        fail("Unknown file format");
+}
+
+ESM4::ReaderContext ESM::ESM4Reader::getESM4Context()
+{
+    return mReader.getContext();
+}
+
+void ESM::ESM4Reader::restoreESM4Context(const ESM4::ReaderContext& ctx)
+{
+    // Reopen the file if necessary
+    if (mCtx.filename != ctx.filename)
+        openTes4File(ctx.filename);
+
+    // mCtx.leftFile is the only thing used in the old context.  Strictly speaking, updating it
+    // with the correct value is not really necessary since we're not going to load the rest of
+    // the file (most likely to load a CELL or LAND then be done with it).
+    mCtx.leftFile = mReader.getFileSize() - mReader.getFileOffset();
+
+    // restore group stack, load the header, etc.
+    mReader.restoreContext(ctx);
+}
+
+void ESM::ESM4Reader::restoreCellChildrenContext(const ESM4::ReaderContext& ctx)
+{
+    // Reopen the file if necessary
+    if (mCtx.filename != ctx.filename)
+        openTes4File(ctx.filename);
+
+    mReader.restoreContext(ctx); // restore group stack, load the CELL header, etc.
+    if (mReader.hdr().record.typeId != ESM4::REC_CELL) // FIXME: testing only
+        fail("Restore Cell Children failed");
+    mReader.skipRecordData();    // skip the CELL record
+
+    mReader.getRecordHeader();   // load the header for cell child group (hopefully)
+    // this is a hack to load only the cell child group...
+    if (mReader.hdr().group.typeId == ESM4::REC_GRUP && mReader.hdr().group.type == ESM4::Grp_CellChild)
+    {
+        mCtx.leftFile = mReader.hdr().group.groupSize - ctx.recHeaderSize;
+        return;
+    }
+
+    // But some cells may have no child groups...
+    // Suspect "ICMarketDistrict" 7 18 is one, followed by cell record 00165F2C "ICMarketDistrict" 6 17
+    if (mReader.hdr().group.typeId != ESM4::REC_GRUP && mReader.hdr().record.typeId == ESM4::REC_CELL)
+    {
+        mCtx.leftFile = 0;
+        return;
+    }
+
+    // Maybe the group is completed
+    // See "ICMarketDistrict" 9 15 which is followed by a exterior sub-cell block
+    ESM4::ReaderContext tempCtx = mReader.getContext();
+    if (!tempCtx.groupStack.empty() && tempCtx.groupStack.back().second == 0)
+    {
+        mCtx.leftFile = 0;
+        return;
+    }
+    else
+        fail("Restore Cell Children failed");
+
+}

components/esm/esm4reader.hpp

@@ -0,0 +1,38 @@
+#ifndef COMPONENT_ESM_4READER_H
+#define COMPONENT_ESM_4READER_H
+
+#include <extern/esm4/tes4.hpp>
+#include <extern/esm4/reader.hpp>
+
+#include "esmreader.hpp"
+
+namespace ESM
+{
+    // Wrapper class for integrating into OpenCS
+    class ESM4Reader : public ESMReader, public ESM4::ReaderObserver
+    {
+        ESM4::Reader mReader;
+
+    public:
+
+        ESM4Reader(bool oldHeader = true);
+        virtual ~ESM4Reader();
+
+        ESM4::Reader& reader() { return mReader; }
+
+        // Added for use with OpenMW (loading progress bar)
+        inline size_t getFileSize() { return mReader.getFileSize(); }
+        inline size_t getFileOffset() { return mReader.getFileOffset(); }
+
+        // Added for loading Cell/Land
+        ESM4::ReaderContext getESM4Context();
+        void restoreESM4Context(const ESM4::ReaderContext& ctx);
+        void restoreCellChildrenContext(const ESM4::ReaderContext& ctx);
+
+        void openTes4File(const std::string &name);
+
+        // callback from mReader to ensure hasMoreRecs() can reliably track to EOF
+        inline void update(std::size_t size) { mCtx.leftFile -= size; }
+    };
+}
+#endif // COMPONENT_ESM_4READER_H