Add ESM data loader

A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
3 years ago · 046b5f83ee
parent 62dfbb33d9
commit 046b5f83ee
12 changed files with 726 additions and 0 deletions
--- a/apps/openmw_test_suite/CMakeLists.txt
+++ b/apps/openmw_test_suite/CMakeLists.txt
@ -48,6 +48,8 @@ if (GTEST_FOUND AND GMOCK_FOUND)
        sqlite3/request.cpp
        sqlite3/statement.cpp
        sqlite3/transaction.cpp
+
+        esmloader/load.cpp
    )

    source_group(apps\\openmw_test_suite FILES openmw_test_suite.cpp ${UNITTEST_SRC_FILES})
@ -73,4 +75,17 @@ if (GTEST_FOUND AND GMOCK_FOUND)
        endif (CMAKE_CL_64)
    endif (MSVC)

+    include(FetchContent)
+    FetchContent_Declare(example_suite_template_game
+        URL https://gitlab.com/OpenMW/example-suite/-/raw/8966dab24692555eec720c854fb0f73d108070cd/data/template.omwgame
+        URL_HASH MD5=bf3691034a38611534c74c3b89a7d2c3
+        SOURCE_DIR fetched/example_suite_template_game
+        DOWNLOAD_NO_EXTRACT ON
+    )
+    FetchContent_MakeAvailableExcludeFromAll(example_suite_template_game)
+
+    add_custom_target(example_suite_template_game SOURCES fetched/example_suite_template_game/template.omwgame)
+
+    add_dependencies(openmw_test_suite example_suite_template_game)
+
 endif()
--- a/apps/openmw_test_suite/esmloader/load.cpp
+++ b/apps/openmw_test_suite/esmloader/load.cpp
@ -0,0 +1,104 @@
+#include "settings.hpp"
+
+#include <components/esmloader/load.cpp>
+#include <components/files/collections.hpp>
+#include <components/files/multidircollection.hpp>
+#include <components/to_utf8/to_utf8.hpp>
+
+#include <gtest/gtest.h>
+
+namespace
+{
+    using namespace testing;
+    using namespace EsmLoader;
+    using EsmLoaderTests::Settings;
+
+    struct EsmLoaderTest : Test
+    {
+        const boost::filesystem::path mDataDir {Settings::impl().mBasePath / "apps/openmw_test_suite/fetched/example_suite_template_game"};
+        const Files::PathContainer mDataDirs {{mDataDir.string()}};
+        const Files::Collections mFileCollections {mDataDirs, true};
+        const std::vector<std::string> mContentFiles {{"template.omwgame"}};
+    };
+
+    TEST_F(EsmLoaderTest, loadEsmDataShouldSupportOmwgame)
+    {
+        Query query;
+        query.mLoadActivators = true;
+        query.mLoadCells = true;
+        query.mLoadContainers = true;
+        query.mLoadDoors = true;
+        query.mLoadGameSettings = true;
+        query.mLoadLands = true;
+        query.mLoadStatics = true;
+        std::vector<ESM::ESMReader> readers(mContentFiles.size());
+        ToUTF8::Utf8Encoder* const encoder = nullptr;
+        const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
+        EXPECT_EQ(esmData.mActivators.size(), 0);
+        EXPECT_EQ(esmData.mCells.size(), 1);
+        EXPECT_EQ(esmData.mContainers.size(), 0);
+        EXPECT_EQ(esmData.mDoors.size(), 0);
+        EXPECT_EQ(esmData.mGameSettings.size(), 1521);
+        EXPECT_EQ(esmData.mLands.size(), 1);
+        EXPECT_EQ(esmData.mStatics.size(), 2);
+    }
+
+    TEST_F(EsmLoaderTest, shouldIgnoreCellsWhenQueryLoadCellsIsFalse)
+    {
+        Query query;
+        query.mLoadActivators = true;
+        query.mLoadCells = false;
+        query.mLoadContainers = true;
+        query.mLoadDoors = true;
+        query.mLoadGameSettings = true;
+        query.mLoadLands = true;
+        query.mLoadStatics = true;
+        std::vector<ESM::ESMReader> readers(mContentFiles.size());
+        ToUTF8::Utf8Encoder* const encoder = nullptr;
+        const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
+        EXPECT_EQ(esmData.mActivators.size(), 0);
+        EXPECT_EQ(esmData.mCells.size(), 0);
+        EXPECT_EQ(esmData.mContainers.size(), 0);
+        EXPECT_EQ(esmData.mDoors.size(), 0);
+        EXPECT_EQ(esmData.mGameSettings.size(), 1521);
+        EXPECT_EQ(esmData.mLands.size(), 1);
+        EXPECT_EQ(esmData.mStatics.size(), 2);
+    }
+
+    TEST_F(EsmLoaderTest, shouldIgnoreCellsGameSettingsWhenQueryLoadGameSettingsIsFalse)
+    {
+        Query query;
+        query.mLoadActivators = true;
+        query.mLoadCells = true;
+        query.mLoadContainers = true;
+        query.mLoadDoors = true;
+        query.mLoadGameSettings = false;
+        query.mLoadLands = true;
+        query.mLoadStatics = true;
+        std::vector<ESM::ESMReader> readers(mContentFiles.size());
+        ToUTF8::Utf8Encoder* const encoder = nullptr;
+        const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
+        EXPECT_EQ(esmData.mActivators.size(), 0);
+        EXPECT_EQ(esmData.mCells.size(), 1);
+        EXPECT_EQ(esmData.mContainers.size(), 0);
+        EXPECT_EQ(esmData.mDoors.size(), 0);
+        EXPECT_EQ(esmData.mGameSettings.size(), 0);
+        EXPECT_EQ(esmData.mLands.size(), 1);
+        EXPECT_EQ(esmData.mStatics.size(), 2);
+    }
+
+    TEST_F(EsmLoaderTest, shouldIgnoreAllWithDefaultQuery)
+    {
+        const Query query;
+        std::vector<ESM::ESMReader> readers(mContentFiles.size());
+        ToUTF8::Utf8Encoder* const encoder = nullptr;
+        const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
+        EXPECT_EQ(esmData.mActivators.size(), 0);
+        EXPECT_EQ(esmData.mCells.size(), 0);
+        EXPECT_EQ(esmData.mContainers.size(), 0);
+        EXPECT_EQ(esmData.mDoors.size(), 0);
+        EXPECT_EQ(esmData.mGameSettings.size(), 0);
+        EXPECT_EQ(esmData.mLands.size(), 0);
+        EXPECT_EQ(esmData.mStatics.size(), 0);
+    }
+}
--- a/apps/openmw_test_suite/esmloader/settings.hpp
+++ b/apps/openmw_test_suite/esmloader/settings.hpp
@ -0,0 +1,20 @@
+#ifndef OPENMW_TEST_SUITE_LOAD_SETTINGS_H
+#define OPENMW_TEST_SUITE_LOAD_SETTINGS_H
+
+#include <boost/filesystem/path.hpp>
+
+namespace EsmLoaderTests
+{
+    struct Settings
+    {
+        boost::filesystem::path mBasePath;
+
+        static Settings& impl()
+        {
+            static Settings value;
+            return value;
+        }
+    };
+}
+
+#endif
--- a/apps/openmw_test_suite/openmw_test_suite.cpp
+++ b/apps/openmw_test_suite/openmw_test_suite.cpp
@ -1,5 +1,9 @@
+#include "esmloader/settings.hpp"
+
 #include <gtest/gtest.h>

+#include <boost/filesystem/path.hpp>
+
 #ifdef WIN32
 //we cannot use GTEST_API_ before main if we're building standalone exe application,
 //and we're linking GoogleTest / GoogleMock as DLLs and not linking gtest_main / gmock_main
@ -7,6 +11,7 @@ int main(int argc, char **argv) {
 #else
 GTEST_API_ int main(int argc, char **argv) {
 #endif
+    EsmLoaderTests::Settings::impl().mBasePath = boost::filesystem::path(argv[0]).parent_path();
    testing::InitGoogleTest(&argc, argv);
    return RUN_ALL_TESTS();
 }
--- a/components/CMakeLists.txt
+++ b/components/CMakeLists.txt
@ -207,6 +207,11 @@ add_component_dir(sqlite3
    transaction
 )

+add_component_dir(esmloader
+    load
+    esmdata
+)
+
 set (ESM_UI ${CMAKE_SOURCE_DIR}/files/ui/contentselector.ui
    )

--- a/components/esm/esmcommon.hpp
+++ b/components/esm/esmcommon.hpp
@ -121,6 +121,13 @@ struct FIXED_STRING<4> : public FIXED_STRING_BASE<FIXED_STRING, 4>

    char const* ro_data() const { return data; }
    char*       rw_data() { return data; }
+
+    std::uint32_t toInt() const
+    {
+        std::uint32_t value;
+        std::memcpy(&value, data, sizeof(data));
+        return value;
+    }
 };

 typedef FIXED_STRING<4> NAME;
--- a/components/esmloader/esmdata.cpp
+++ b/components/esmloader/esmdata.cpp
@ -0,0 +1,77 @@
+#include "esmdata.hpp"
+#include "lessbyid.hpp"
+#include "record.hpp"
+
+#include <components/esm/defs.hpp>
+#include <components/esm/loadacti.hpp>
+#include <components/esm/loadcont.hpp>
+#include <components/esm/loaddoor.hpp>
+#include <components/esm/loadgmst.hpp>
+#include <components/esm/loadland.hpp>
+#include <components/esm/loadstat.hpp>
+#include <components/esm/variant.hpp>
+#include <components/misc/stringops.hpp>
+
+#include <algorithm>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace EsmLoader
+{
+    namespace
+    {
+        template <class F>
+        auto returnAs(F&& f)
+        {
+            using Result = decltype(std::forward<F>(f)(ESM::Static {}));
+            if constexpr (!std::is_same_v<Result, void>)
+                return Result {};
+        }
+
+        template <class T, class F>
+        auto withStatic(std::string_view refId, const std::vector<T>& values, F&& f)
+        {
+            const auto it = std::lower_bound(values.begin(), values.end(), refId, LessById {});
+
+            if (it == values.end() && it->mId != refId)
+                return returnAs(std::forward<F>(f));
+
+            return std::forward<F>(f)(*it);
+        }
+
+        template <class F>
+        auto withStatic(std::string_view refId, ESM::RecNameInts type, const EsmData& content, F&& f)
+        {
+            switch (type)
+            {
+                case ESM::REC_ACTI: return withStatic(refId, content.mActivators, std::forward<F>(f));
+                case ESM::REC_CONT: return withStatic(refId, content.mContainers, std::forward<F>(f));
+                case ESM::REC_DOOR: return withStatic(refId, content.mDoors, std::forward<F>(f));
+                case ESM::REC_STAT: return withStatic(refId, content.mStatics, std::forward<F>(f));
+                default: break;
+            }
+
+            return returnAs(std::forward<F>(f));
+        }
+    }
+
+    EsmData::~EsmData() {}
+
+    std::string_view getModel(const EsmData& content, std::string_view refId, ESM::RecNameInts type)
+    {
+        return withStatic(refId, type, content, [] (const auto& v) { return std::string_view(v.mModel); });
+    }
+
+    ESM::Variant getGameSetting(const std::vector<ESM::GameSetting>& records, std::string_view id)
+    {
+        const std::string lower = Misc::StringUtils::lowerCase(id);
+        auto it = std::lower_bound(records.begin(), records.end(), lower, LessById {});
+        if (it == records.end() || it->mId != lower)
+            throw std::runtime_error("Game settings \"" + std::string(id) + "\" is not found");
+        return it->mValue;
+    }
+}
--- a/components/esmloader/esmdata.hpp
+++ b/components/esmloader/esmdata.hpp
@ -0,0 +1,52 @@
+#ifndef OPENMW_COMPONENTS_ESMLOADER_ESMDATA_H
+#define OPENMW_COMPONENTS_ESMLOADER_ESMDATA_H
+
+#include <components/esm/defs.hpp>
+
+#include <string_view>
+#include <vector>
+
+namespace ESM
+{
+    struct Activator;
+    struct Cell;
+    struct Container;
+    struct Door;
+    struct GameSetting;
+    struct Land;
+    struct Static;
+    struct Variant;
+}
+
+namespace EsmLoader
+{
+    struct RefIdWithType
+    {
+        std::string_view mId;
+        ESM::RecNameInts mType;
+    };
+
+    struct EsmData
+    {
+        std::vector<ESM::Activator> mActivators;
+        std::vector<ESM::Cell> mCells;
+        std::vector<ESM::Container> mContainers;
+        std::vector<ESM::Door> mDoors;
+        std::vector<ESM::GameSetting> mGameSettings;
+        std::vector<ESM::Land> mLands;
+        std::vector<ESM::Static> mStatics;
+        std::vector<RefIdWithType> mRefIdTypes;
+
+        EsmData() = default;
+        EsmData(const EsmData&) = delete;
+        EsmData(EsmData&&) = default;
+
+        ~EsmData();
+    };
+
+    std::string_view getModel(const EsmData& content, std::string_view refId, ESM::RecNameInts type);
+
+    ESM::Variant getGameSetting(const std::vector<ESM::GameSetting>& records, std::string_view id);
+}
+
+#endif
--- a/components/esmloader/lessbyid.hpp
+++ b/components/esmloader/lessbyid.hpp
@ -0,0 +1,24 @@
+#ifndef OPENMW_COMPONENTS_CONTENT_LESSBYID_H
+#define OPENMW_COMPONENTS_CONTENT_LESSBYID_H
+
+#include <string_view>
+
+namespace EsmLoader
+{
+    struct LessById
+    {
+        template <class T>
+        bool operator()(const T& lhs, const T& rhs) const
+        {
+            return lhs.mId < rhs.mId;
+        }
+
+        template <class T>
+        bool operator()(const T& lhs, std::string_view rhs) const
+        {
+            return lhs.mId < rhs;
+        }
+    };
+}
+
+#endif
--- a/components/esmloader/load.cpp
+++ b/components/esmloader/load.cpp
@ -0,0 +1,334 @@
+#include "load.hpp"
+#include "esmdata.hpp"
+#include "lessbyid.hpp"
+#include "record.hpp"
+
+#include <components/debug/debuglog.hpp>
+#include <components/esm/cellref.hpp>
+#include <components/esm/defs.hpp>
+#include <components/esm/esmreader.hpp>
+#include <components/esm/loadacti.hpp>
+#include <components/esm/loadcell.hpp>
+#include <components/esm/loadcont.hpp>
+#include <components/esm/loaddoor.hpp>
+#include <components/esm/loadgmst.hpp>
+#include <components/esm/loadland.hpp>
+#include <components/esm/loadstat.hpp>
+#include <components/files/collections.hpp>
+#include <components/files/multidircollection.hpp>
+#include <components/misc/resourcehelpers.hpp>
+#include <components/misc/stringops.hpp>
+
+#include <algorithm>
+#include <cstddef>
+#include <map>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace EsmLoader
+{
+    namespace
+    {
+        struct GetKey
+        {
+            template <class T>
+            decltype(auto) operator()(const T& v) const
+            {
+                return (v.mId);
+            }
+
+            const ESM::CellId& operator()(const ESM::Cell& v) const
+            {
+                return v.mCellId;
+            }
+
+            std::pair<int, int> operator()(const ESM::Land& v) const
+            {
+                return std::pair(v.mX, v.mY);
+            }
+
+            template <class T>
+            decltype(auto) operator()(const Record<T>& v) const
+            {
+                return (*this)(v.mValue);
+            }
+        };
+
+        struct CellRecords
+        {
+            Records<ESM::Cell> mValues;
+            std::map<std::string, std::size_t> mByName;
+            std::map<std::pair<int, int>, std::size_t> mByPosition;
+        };
+
+        template <class T, class = std::void_t<>>
+        struct HasId : std::false_type {};
+
+        template <class T>
+        struct HasId<T, std::void_t<decltype(T::mId)>> : std::true_type {};
+
+        template <class T>
+        constexpr bool hasId = HasId<T>::value;
+
+        template <class T>
+        auto loadRecord(ESM::ESMReader& reader, Records<T>& records)
+            -> std::enable_if_t<hasId<T>>
+        {
+            T record;
+            bool deleted = false;
+            record.load(reader, deleted);
+            Misc::StringUtils::lowerCaseInPlace(record.mId);
+            if (Misc::ResourceHelpers::isHiddenMarker(record.mId))
+                return;
+            records.emplace_back(deleted, std::move(record));
+        }
+
+        template <class T>
+        auto loadRecord(ESM::ESMReader& reader, Records<T>& records)
+            -> std::enable_if_t<!hasId<T>>
+        {
+            T record;
+            bool deleted = false;
+            record.load(reader, deleted);
+            records.emplace_back(deleted, std::move(record));
+        }
+
+        void loadRecord(ESM::ESMReader& reader, CellRecords& records)
+        {
+            ESM::Cell record;
+            bool deleted = false;
+            record.loadNameAndData(reader, deleted);
+            Misc::StringUtils::lowerCaseInPlace(record.mName);
+
+            if ((record.mData.mFlags & ESM::Cell::Interior) != 0)
+            {
+                const auto it = records.mByName.find(record.mName);
+                if (it == records.mByName.end())
+                {
+                    record.loadCell(reader, true);
+                    records.mByName.emplace_hint(it, record.mName, records.mValues.size());
+                    records.mValues.emplace_back(deleted, std::move(record));
+                }
+                else
+                {
+                    Record<ESM::Cell>& old = records.mValues[it->second];
+                    old.mValue.mData = record.mData;
+                    old.mValue.loadCell(reader, true);
+                }
+            }
+            else
+            {
+                const std::pair<int, int> position(record.mData.mX, record.mData.mY);
+                const auto it = records.mByPosition.find(position);
+                if (it == records.mByPosition.end())
+                {
+                    record.loadCell(reader, true);
+                    records.mByPosition.emplace_hint(it, position, records.mValues.size());
+                    records.mValues.emplace_back(deleted, std::move(record));
+                }
+                else
+                {
+                    Record<ESM::Cell>& old = records.mValues[it->second];
+                    old.mValue.mData = record.mData;
+                    old.mValue.loadCell(reader, true);
+                }
+            }
+        }
+
+        struct ShallowContent
+        {
+            Records<ESM::Activator> mActivators;
+            CellRecords mCells;
+            Records<ESM::Container> mContainers;
+            Records<ESM::Door> mDoors;
+            Records<ESM::GameSetting> mGameSettings;
+            Records<ESM::Land> mLands;
+            Records<ESM::Static> mStatics;
+        };
+
+        void loadRecord(const Query& query, const ESM::NAME& name, ESM::ESMReader& reader, ShallowContent& content)
+        {
+            switch (name.toInt())
+            {
+                case ESM::REC_ACTI:
+                    if (query.mLoadActivators)
+                        return loadRecord(reader, content.mActivators);
+                    break;
+                case ESM::REC_CELL:
+                    if (query.mLoadCells)
+                        return loadRecord(reader, content.mCells);
+                    break;
+                case ESM::REC_CONT:
+                    if (query.mLoadContainers)
+                        return loadRecord(reader, content.mContainers);
+                    break;
+                case ESM::REC_DOOR:
+                    if (query.mLoadDoors)
+                        return loadRecord(reader, content.mDoors);
+                    break;
+                case ESM::REC_GMST:
+                    if (query.mLoadGameSettings)
+                        return loadRecord(reader, content.mGameSettings);
+                    break;
+                case ESM::REC_LAND:
+                    if (query.mLoadLands)
+                        return loadRecord(reader, content.mLands);
+                    break;
+                case ESM::REC_STAT:
+                    if (query.mLoadStatics)
+                        return loadRecord(reader, content.mStatics);
+                    break;
+            }
+
+            reader.skipRecord();
+        }
+
+        ESM::ESMReader loadEsm(const Query& query, ESM::ESMReader& reader, ShallowContent& content)
+        {
+            Log(Debug::Info) << "Loading ESM file " << reader.getName();
+
+            while (reader.hasMoreRecs())
+            {
+                const ESM::NAME recName = reader.getRecName();
+                reader.getRecHeader();
+                loadRecord(query, recName, reader, content);
+            }
+
+            return reader;
+        }
+
+        ShallowContent shallowLoad(const Query& query, const std::vector<std::string>& contentFiles,
+            const Files::Collections& fileCollections, std::vector<ESM::ESMReader>& readers,
+            ToUTF8::Utf8Encoder* encoder)
+        {
+            ShallowContent result;
+
+            for (std::size_t i = 0; i < contentFiles.size(); ++i)
+            {
+                const std::string &file = contentFiles[i];
+                const Files::MultiDirCollection& collection = fileCollections.getCollection(boost::filesystem::path(file).extension().string());
+
+                ESM::ESMReader& reader = readers[i];
+                reader.setEncoder(encoder);
+                reader.setIndex(static_cast<int>(i));
+                reader.setGlobalReaderList(&readers);
+                reader.open(collection.getPath(file).string());
+
+                loadEsm(query, readers[i], result);
+            }
+
+            return result;
+        }
+
+        struct WithType
+        {
+            ESM::RecNameInts mType;
+
+            template <class T>
+            RefIdWithType operator()(const T& v) const { return {v.mId, mType}; }
+        };
+
+        template <class T>
+        void addRefIdsTypes(const std::vector<T>& values, std::vector<RefIdWithType>& refIdsTypes)
+        {
+            std::transform(values.begin(), values.end(), std::back_inserter(refIdsTypes),
+                           WithType {static_cast<ESM::RecNameInts>(T::sRecordId)});
+        }
+
+        void addRefIdsTypes(EsmData& content)
+        {
+            content.mRefIdTypes.reserve(
+                content.mActivators.size()
+                + content.mContainers.size()
+                + content.mDoors.size()
+                + content.mStatics.size()
+            );
+
+            addRefIdsTypes(content.mActivators, content.mRefIdTypes);
+            addRefIdsTypes(content.mContainers, content.mRefIdTypes);
+            addRefIdsTypes(content.mDoors, content.mRefIdTypes);
+            addRefIdsTypes(content.mStatics, content.mRefIdTypes);
+
+            std::sort(content.mRefIdTypes.begin(), content.mRefIdTypes.end(), LessById {});
+        }
+
+        std::vector<ESM::Cell> prepareCellRecords(Records<ESM::Cell>& records)
+        {
+            std::vector<ESM::Cell> result;
+            for (Record<ESM::Cell>& v : records)
+                if (!v.mDeleted)
+                    result.emplace_back(std::move(v.mValue));
+            return result;
+        }
+    }
+
+    EsmData loadEsmData(const Query& query, const std::vector<std::string>& contentFiles,
+        const Files::Collections& fileCollections, std::vector<ESM::ESMReader>& readers, ToUTF8::Utf8Encoder* encoder)
+    {
+        Log(Debug::Info) << "Loading ESM data...";
+
+        ShallowContent content = shallowLoad(query, contentFiles, fileCollections, readers, encoder);
+
+        std::ostringstream loaded;
+
+        if (query.mLoadActivators)
+            loaded << ' ' << content.mActivators.size() << " activators,";
+        if (query.mLoadCells)
+            loaded << ' ' << content.mCells.mValues.size() << " cells,";
+        if (query.mLoadContainers)
+            loaded << ' ' << content.mContainers.size() << " containers,";
+        if (query.mLoadDoors)
+            loaded << ' ' << content.mDoors.size() << " doors,";
+        if (query.mLoadGameSettings)
+            loaded << ' ' << content.mGameSettings.size() << " game settings,";
+        if (query.mLoadLands)
+            loaded << ' ' << content.mLands.size() << " lands,";
+        if (query.mLoadStatics)
+            loaded << ' ' << content.mStatics.size() << " statics,";
+
+        Log(Debug::Info) << "Loaded" << loaded.str();
+
+        EsmData result;
+
+        if (query.mLoadActivators)
+            result.mActivators = prepareRecords(content.mActivators, GetKey {});
+        if (query.mLoadCells)
+            result.mCells = prepareCellRecords(content.mCells.mValues);
+        if (query.mLoadContainers)
+            result.mContainers = prepareRecords(content.mContainers, GetKey {});
+        if (query.mLoadDoors)
+            result.mDoors = prepareRecords(content.mDoors, GetKey {});
+        if (query.mLoadGameSettings)
+            result.mGameSettings = prepareRecords(content.mGameSettings, GetKey {});
+        if (query.mLoadLands)
+            result.mLands = prepareRecords(content.mLands, GetKey {});
+        if (query.mLoadStatics)
+            result.mStatics = prepareRecords(content.mStatics, GetKey {});
+
+        addRefIdsTypes(result);
+
+        std::ostringstream prepared;
+
+        if (query.mLoadActivators)
+            prepared << ' ' << result.mActivators.size() << " unique activators,";
+        if (query.mLoadCells)
+            prepared << ' ' << result.mCells.size() << " unique cells,";
+        if (query.mLoadContainers)
+            prepared << ' ' << result.mContainers.size() << " unique containers,";
+        if (query.mLoadDoors)
+            prepared << ' ' << result.mDoors.size() << " unique doors,";
+        if (query.mLoadGameSettings)
+            prepared << ' ' << result.mGameSettings.size() << " unique game settings,";
+        if (query.mLoadLands)
+            prepared << ' ' << result.mLands.size() << " unique lands,";
+        if (query.mLoadStatics)
+            prepared << ' ' << result.mStatics.size() << " unique statics,";
+
+        Log(Debug::Info) << "Prepared" << prepared.str();
+
+        return result;
+    }
+}
--- a/components/esmloader/load.hpp
+++ b/components/esmloader/load.hpp
@ -0,0 +1,39 @@
+#ifndef OPENMW_COMPONENTS_ESMLOADER_LOAD_H
+#define OPENMW_COMPONENTS_ESMLOADER_LOAD_H
+
+#include <components/esm/esmreader.hpp>
+
+#include <string>
+#include <vector>
+
+namespace ToUTF8
+{
+    class Utf8Encoder;
+}
+
+namespace Files
+{
+    class Collections;
+}
+
+namespace EsmLoader
+{
+    struct EsmData;
+
+    struct Query
+    {
+        bool mLoadActivators = false;
+        bool mLoadCells = false;
+        bool mLoadContainers = false;
+        bool mLoadDoors = false;
+        bool mLoadGameSettings = false;
+        bool mLoadLands = false;
+        bool mLoadStatics = false;
+    };
+
+    EsmData loadEsmData(const Query& query, const std::vector<std::string>& contentFiles,
+        const Files::Collections& fileCollections, std::vector<ESM::ESMReader>& readers,
+        ToUTF8::Utf8Encoder* encoder);
+}
+
+#endif
--- a/components/esmloader/record.hpp
+++ b/components/esmloader/record.hpp
@ -0,0 +1,44 @@
+#ifndef OPENMW_COMPONENTS_ESMLOADER_RECORD_H
+#define OPENMW_COMPONENTS_ESMLOADER_RECORD_H
+
+#include <components/esm/loadcell.hpp>
+
+#include <algorithm>
+#include <utility>
+#include <vector>
+
+namespace EsmLoader
+{
+    template <class T>
+    struct Record
+    {
+        bool mDeleted;
+        T mValue;
+
+        template <class ... Args>
+        explicit Record(bool deleted, Args&& ... args)
+            : mDeleted(deleted)
+            , mValue(std::forward<Args>(args) ...)
+        {}
+    };
+
+    template <class T>
+    using Records = std::vector<Record<T>>;
+
+    template <class T, class GetKey>
+    inline std::vector<T> prepareRecords(Records<T>& records, const GetKey& getKey)
+    {
+        const auto greaterByKey = [&] (const auto& l, const auto& r) { return getKey(r) < getKey(l); };
+        const auto equalByKey = [&] (const auto& l, const auto& r) { return getKey(l) == getKey(r); };
+        std::stable_sort(records.begin(), records.end(), greaterByKey);
+        records.erase(std::unique(records.begin(), records.end(), equalByKey), records.end());
+        std::reverse(records.begin(), records.end());
+        std::vector<T> result;
+        for (Record<T>& v : records)
+            if (!v.mDeleted)
+                result.emplace_back(std::move(v.mValue));
+        return result;
+    }
+}
+
+#endif