diff --git a/apps/opencs/model/world/refcollection.cpp b/apps/opencs/model/world/refcollection.cpp
index 51588cbc2..dfdb8e73b 100644
--- a/apps/opencs/model/world/refcollection.cpp
+++ b/apps/opencs/model/world/refcollection.cpp
@@ -108,11 +108,13 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool
 
             Record<CellRef> record;
             record.mState = base ? RecordBase::State_BaseOnly : RecordBase::State_ModifiedOnly;
-            (base ? record.mBase : record.mModified) = ref;
+            const ESM::RefNum refNum = ref.mRefNum;
+            std::string refId = ref.mId;
+            (base ? record.mBase : record.mModified) = std::move(ref);
 
             appendRecord (record);
 
-            cache.insert (std::make_pair (ref.mRefNum, ref.mId));
+            cache.emplace(refNum, std::move(refId));
         }
         else
         {
@@ -123,7 +125,7 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool
 
             Record<CellRef> record = getRecord (index);
             record.mState = base ? RecordBase::State_BaseOnly : RecordBase::State_Modified;
-            (base ? record.mBase : record.mModified) = ref;
+            (base ? record.mBase : record.mModified) = std::move(ref);
 
             setRecord (index, record);
         }
diff --git a/apps/openmw/mwrender/groundcover.cpp b/apps/openmw/mwrender/groundcover.cpp
index fd2246253..63df3e249 100644
--- a/apps/openmw/mwrender/groundcover.cpp
+++ b/apps/openmw/mwrender/groundcover.cpp
@@ -241,7 +241,7 @@ namespace MWRender
                         if (model.empty()) continue;
                         model = "meshes/" + model;
 
-                        instances[model].emplace_back(ref, model);
+                        instances[model].emplace_back(std::move(ref), std::move(model));
                     }
                 }
             }
diff --git a/apps/openmw/mwrender/objectpaging.cpp b/apps/openmw/mwrender/objectpaging.cpp
index 174c25959..78e05675a 100644
--- a/apps/openmw/mwrender/objectpaging.cpp
+++ b/apps/openmw/mwrender/objectpaging.cpp
@@ -426,13 +426,13 @@ namespace MWRender
                         bool deleted = false;
                         while(cell->getNextRef(esm[index], ref, deleted))
                         {
-                            Misc::StringUtils::lowerCaseInPlace(ref.mRefID);
                             if (std::find(cell->mMovedRefs.begin(), cell->mMovedRefs.end(), ref.mRefNum) != cell->mMovedRefs.end()) continue;
+                            Misc::StringUtils::lowerCaseInPlace(ref.mRefID);
                             int type = store.findStatic(ref.mRefID);
                             if (!typeFilter(type,size>=2)) continue;
                             if (deleted) { refs.erase(ref.mRefNum); continue; }
                             if (ref.mRefNum.fromGroundcoverFile()) continue;
-                            refs[ref.mRefNum] = ref;
+                            refs[ref.mRefNum] = std::move(ref);
                         }
                     }
                     catch (std::exception&)
@@ -443,12 +443,12 @@ namespace MWRender
                 for (ESM::CellRefTracker::const_iterator it = cell->mLeasedRefs.begin(); it != cell->mLeasedRefs.end(); ++it)
                 {
                     ESM::CellRef ref = it->first;
-                    Misc::StringUtils::lowerCaseInPlace(ref.mRefID);
                     bool deleted = it->second;
                     if (deleted) { refs.erase(ref.mRefNum); continue; }
+                    Misc::StringUtils::lowerCaseInPlace(ref.mRefID);
                     int type = store.findStatic(ref.mRefID);
                     if (!typeFilter(type,size>=2)) continue;
-                    refs[ref.mRefNum] = ref;
+                    refs[ref.mRefNum] = std::move(ref);
                 }
             }
         }
diff --git a/apps/openmw/mwworld/cellstore.cpp b/apps/openmw/mwworld/cellstore.cpp
index 73f69e220..813042f15 100644
--- a/apps/openmw/mwworld/cellstore.cpp
+++ b/apps/openmw/mwworld/cellstore.cpp
@@ -566,7 +566,8 @@ namespace MWWorld
                         continue;
                     }
 
-                    mIds.push_back (Misc::StringUtils::lowerCase (ref.mRefID));
+                    Misc::StringUtils::lowerCaseInPlace(ref.mRefID);
+                    mIds.push_back(std::move(ref.mRefID));
                 }
             }
             catch (std::exception& e)
diff --git a/apps/openmw/mwworld/esmstore.cpp b/apps/openmw/mwworld/esmstore.cpp
index 0ec6d8d73..b1885edaf 100644
--- a/apps/openmw/mwworld/esmstore.cpp
+++ b/apps/openmw/mwworld/esmstore.cpp
@@ -1,5 +1,6 @@
 #include "esmstore.hpp"
 
+#include <algorithm>
 #include <set>
 
 #include <boost/filesystem/operations.hpp>
@@ -8,12 +9,23 @@
 #include <components/loadinglistener/loadinglistener.hpp>
 #include <components/esm/esmreader.hpp>
 #include <components/esm/esmwriter.hpp>
+#include <components/misc/algorithm.hpp>
 
 #include "../mwmechanics/spelllist.hpp"
 
 namespace
 {
-    void readRefs(const ESM::Cell& cell, std::map<ESM::RefNum, std::string>& refs, std::vector<ESM::ESMReader>& readers)
+    struct Ref
+    {
+        ESM::RefNum mRefNum;
+        std::size_t mRefID;
+
+        Ref(ESM::RefNum refNum, std::size_t refID) : mRefNum(refNum), mRefID(refID) {}
+    };
+
+    constexpr std::size_t deletedRefID = std::numeric_limits<std::size_t>::max();
+
+    void readRefs(const ESM::Cell& cell, std::vector<Ref>& refs, std::vector<std::string>& refIDs, std::vector<ESM::ESMReader>& readers)
     {
         for (size_t i = 0; i < cell.mContextList.size(); i++)
         {
@@ -27,24 +39,22 @@ namespace
             while(cell.getNextRef(readers[index], ref, deleted))
             {
                 if(deleted)
-                    refs.erase(ref.mRefNum);
+                    refs.emplace_back(ref.mRefNum, deletedRefID);
                 else if (std::find(cell.mMovedRefs.begin(), cell.mMovedRefs.end(), ref.mRefNum) == cell.mMovedRefs.end())
                 {
-                    Misc::StringUtils::lowerCaseInPlace(ref.mRefID);
-                    refs[ref.mRefNum] = ref.mRefID;
+                    refs.emplace_back(ref.mRefNum, refIDs.size());
+                    refIDs.push_back(std::move(ref.mRefID));
                 }
             }
         }
-        for(const auto& it : cell.mLeasedRefs)
+        for(const auto& [value, deleted] : cell.mLeasedRefs)
         {
-            bool deleted = it.second;
             if(deleted)
-                refs.erase(it.first.mRefNum);
+                refs.emplace_back(value.mRefNum, deletedRefID);
             else
             {
-                ESM::CellRef ref = it.first;
-                Misc::StringUtils::lowerCaseInPlace(ref.mRefID);
-                refs[ref.mRefNum] = ref.mRefID;
+                refs.emplace_back(value.mRefNum, refIDs.size());
+                refIDs.push_back(value.mRefID);
             }
         }
     }
@@ -248,14 +258,26 @@ void ESMStore::countRecords()
 {
     if(!mRefCount.empty())
         return;
-    std::map<ESM::RefNum, std::string> refs;
+    std::vector<Ref> refs;
+    std::vector<std::string> refIDs;
     std::vector<ESM::ESMReader> readers;
     for(auto it = mCells.intBegin(); it != mCells.intEnd(); it++)
-        readRefs(*it, refs, readers);
+        readRefs(*it, refs, refIDs, readers);
     for(auto it = mCells.extBegin(); it != mCells.extEnd(); it++)
-        readRefs(*it, refs, readers);
-    for(const auto& pair : refs)
-        mRefCount[pair.second]++;
+        readRefs(*it, refs, refIDs, readers);
+    const auto lessByRefNum = [] (const Ref& l, const Ref& r) { return l.mRefNum < r.mRefNum; };
+    std::stable_sort(refs.begin(), refs.end(), lessByRefNum);
+    const auto equalByRefNum = [] (const Ref& l, const Ref& r) { return l.mRefNum == r.mRefNum; };
+    const auto incrementRefCount = [&] (const Ref& value)
+    {
+        if (value.mRefID != deletedRefID)
+        {
+            std::string& refId = refIDs[value.mRefID];
+            Misc::StringUtils::lowerCaseInPlace(refId);
+            ++mRefCount[std::move(refId)];
+        }
+    };
+    Misc::forEachUnique(refs.rbegin(), refs.rend(), equalByRefNum, incrementRefCount);
 }
 
 int ESMStore::getRefCount(const std::string& id) const
diff --git a/apps/openmw/mwworld/esmstore.hpp b/apps/openmw/mwworld/esmstore.hpp
index 26f497a52..608b5489e 100644
--- a/apps/openmw/mwworld/esmstore.hpp
+++ b/apps/openmw/mwworld/esmstore.hpp
@@ -4,6 +4,7 @@
 #include <memory>
 #include <sstream>
 #include <stdexcept>
+#include <unordered_map>
 
 #include <components/esm/records.hpp>
 #include "store.hpp"
@@ -76,7 +77,7 @@ namespace MWWorld
         std::map<std::string, int> mIds;
         std::map<std::string, int> mStaticIds;
 
-        std::map<std::string, int> mRefCount;
+        std::unordered_map<std::string, int> mRefCount;
 
         std::map<int, StoreBase *> mStores;
 
diff --git a/apps/openmw/mwworld/store.cpp b/apps/openmw/mwworld/store.cpp
index 53ba3cfc2..8e6596cd3 100644
--- a/apps/openmw/mwworld/store.cpp
+++ b/apps/openmw/mwworld/store.cpp
@@ -518,9 +518,9 @@ namespace MWWorld
             // But there may be duplicates here!
             ESM::CellRefTracker::iterator iter = std::find_if(cellAlt->mLeasedRefs.begin(), cellAlt->mLeasedRefs.end(), ESM::CellRefTrackerPredicate(ref.mRefNum));
             if (iter == cellAlt->mLeasedRefs.end())
-                cellAlt->mLeasedRefs.push_back(std::make_pair(ref, deleted));
+                cellAlt->mLeasedRefs.emplace_back(std::move(ref), deleted);
             else
-                *iter = std::make_pair(ref, deleted);
+                *iter = std::make_pair(std::move(ref), deleted);
         }
     }
     const ESM::Cell *Store<ESM::Cell>::search(const std::string &id) const
diff --git a/components/misc/algorithm.hpp b/components/misc/algorithm.hpp
new file mode 100644
index 000000000..4d70afa86
--- /dev/null
+++ b/components/misc/algorithm.hpp
@@ -0,0 +1,36 @@
+#ifndef OPENMW_COMPONENTS_MISC_ALGORITHM_H
+#define OPENMW_COMPONENTS_MISC_ALGORITHM_H
+
+#include <iterator>
+#include <type_traits>
+
+namespace Misc
+{
+    template <typename Iterator, typename BinaryPredicate, typename Function>
+    inline Iterator forEachUnique(Iterator begin, Iterator end, BinaryPredicate predicate, Function function)
+    {
+        static_assert(
+            std::is_base_of_v<
+                std::forward_iterator_tag,
+                typename std::iterator_traits<Iterator>::iterator_category
+            >
+        );
+        if (begin == end)
+            return begin;
+        function(*begin);
+        auto last = begin;
+        ++begin;
+        while (begin != end)
+        {
+            if (!predicate(*begin, *last))
+            {
+                function(*begin);
+                last = begin;
+            }
+            ++begin;
+        }
+        return begin;
+    }
+}
+
+#endif