Use MurmurHash3_x64_128 for file hash

apps/openmw_test_suite/CMakeLists.txt

@@ -62,6 +62,8 @@ if (GTEST_FOUND AND GMOCK_FOUND)
 
         esmloader/load.cpp
         esmloader/esmdata.cpp
+
+        files/hash.cpp
     )
 
     source_group(apps\\openmw_test_suite FILES openmw_test_suite.cpp ${UNITTEST_SRC_FILES})

apps/openmw_test_suite/files/hash.cpp

@@ -0,0 +1,55 @@
+#include <components/files/hash.hpp>
+#include <components/files/constrainedfilestream.hpp>
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include <algorithm>
+#include <fstream>
+#include <sstream>
+#include <string>
+
+namespace
+{
+    using namespace testing;
+    using namespace Files;
+
+    struct Params
+    {
+        std::size_t mSize;
+        std::array<std::uint64_t, 2> mHash;
+    };
+
+    struct FilesGetHash : TestWithParam<Params> {};
+
+    TEST_P(FilesGetHash, shouldReturnHashForStringStream)
+    {
+        const std::string fileName = "fileName";
+        std::string content;
+        std::fill_n(std::back_inserter(content), GetParam().mSize, 'a');
+        std::istringstream stream(content);
+        EXPECT_EQ(getHash(fileName, stream), GetParam().mHash);
+    }
+
+    TEST_P(FilesGetHash, shouldReturnHashForConstrainedFileStream)
+    {
+        std::string fileName(UnitTest::GetInstance()->current_test_info()->name());
+        std::replace(fileName.begin(), fileName.end(), '/', '_');
+        std::string content;
+        std::fill_n(std::back_inserter(content), GetParam().mSize, 'a');
+        std::fstream(fileName, std::ios_base::out | std::ios_base::binary)
+            .write(content.data(), static_cast<std::streamsize>(content.size()));
+        const auto stream = Files::openConstrainedFileStream(fileName.data(), 0, content.size());
+        EXPECT_EQ(getHash(fileName, *stream), GetParam().mHash);
+    }
+
+    INSTANTIATE_TEST_SUITE_P(Params, FilesGetHash, Values(
+        Params {0, {0, 0}},
+        Params {1, {9607679276477937801ull, 16624257681780017498ull}},
+        Params {128, {15287858148353394424ull, 16818615825966581310ull}},
+        Params {1000, {11018119256083894017ull, 6631144854802791578ull}},
+        Params {4096, {11972283295181039100ull, 16027670129106775155ull}},
+        Params {4097, {16717956291025443060ull, 12856404199748778153ull}},
+        Params {5000, {15775925571142117787ull, 10322955217889622896ull}}
+    ));
+}

apps/openmw_test_suite/nifloader/testbulletnifloader.cpp

@@ -335,7 +335,7 @@ namespace
         MOCK_METHOD(void, setUseSkinning, (bool), (override));
         MOCK_METHOD(bool, getUseSkinning, (), (const, override));
         MOCK_METHOD(std::string, getFilename, (), (const, override));
-        MOCK_METHOD(std::uint64_t, getHash, (), (const, override));
+        MOCK_METHOD(std::string, getHash, (), (const, override));
         MOCK_METHOD(unsigned int, getVersion, (), (const, override));
         MOCK_METHOD(unsigned int, getUserVersion, (), (const, override));
         MOCK_METHOD(unsigned int, getBethVersion, (), (const, override));
@@ -382,7 +382,7 @@ namespace
             ),
             btVector3(4, 8, 12)
         };
-        const std::uint64_t mHash = 42;
+        const std::string mHash = "hash";
 
         TestBulletNifLoader()
         {

components/CMakeLists.txt

@@ -289,6 +289,7 @@ target_link_libraries(components
 
     Base64
     SQLite::SQLite3
+    smhasher
     )
 
 target_link_libraries(components ${BULLET_LIBRARIES})

components/files/hash.cpp

@@ -1,17 +1,17 @@
 #include "hash.hpp"
 
-#include <components/misc/hash.hpp>
+#include <extern/smhasher/MurmurHash3.h>
 
+#include <array>
 #include <cstdint>
-#include <functional>
 #include <istream>
 #include <string>
 
 namespace Files
 {
-    std::uint64_t getHash(const std::string& fileName, std::istream& stream)
+    std::array<std::uint64_t, 2> getHash(const std::string& fileName, std::istream& stream)
     {
-        std::uint64_t hash = std::hash<std::string> {}(fileName);
+        std::array<std::uint64_t, 2> hash {0, 0};
         try
         {
             const auto start = stream.tellg();
@@ -19,9 +19,14 @@ namespace Files
             stream.exceptions(std::ios_base::badbit);
             while (stream)
             {
-                std::uint64_t value = 0;
-                stream.read(reinterpret_cast<char*>(&value), sizeof(value));
-                Misc::hashCombine(hash, value);
+                std::array<char, 4096> value;
+                stream.read(value.data(), value.size());
+                const std::streamsize read = stream.gcount();
+                if (read == 0)
+                    break;
+                std::array<std::uint64_t, 2> blockHash {0, 0};
+                MurmurHash3_x64_128(value.data(), static_cast<int>(read), hash.data(), blockHash.data());
+                hash = blockHash;
             }
             stream.exceptions(exceptions);
             stream.clear();

components/files/hash.hpp

@@ -1,13 +1,14 @@
 #ifndef COMPONENTS_FILES_HASH_H
 #define COMPONENTS_FILES_HASH_H
 
+#include <array>
 #include <cstdint>
 #include <istream>
 #include <string>
 
 namespace Files
 {
-    std::uint64_t getHash(const std::string& fileName, std::istream& stream);
+    std::array<std::uint64_t, 2> getHash(const std::string& fileName, std::istream& stream);
 }
 
 #endif

components/nif/niffile.cpp

@@ -173,7 +173,8 @@ std::string NIFFile::printVersion(unsigned int version)
 
 void NIFFile::parse(Files::IStreamPtr stream)
 {
-    hash = Files::getHash(filename, *stream);
+    const std::array<std::uint64_t, 2> fileHash = Files::getHash(filename, *stream);
+    hash.append(reinterpret_cast<const char*>(fileHash.data()), fileHash.size() * sizeof(std::uint64_t));
 
     NIFStream nif (this, stream);
 

components/nif/niffile.hpp

@@ -34,7 +34,7 @@ struct File
 
     virtual std::string getFilename() const = 0;
 
-    virtual std::uint64_t getHash() const = 0;
+    virtual std::string getHash() const = 0;
 
     virtual unsigned int getVersion() const = 0;
 
@@ -52,7 +52,7 @@ class NIFFile final : public File
 
     /// File name, used for error messages and opening the file
     std::string filename;
-    std::uint64_t hash = 0;
+    std::string hash;
 
     /// Record list
     std::vector<Record*> records;
@@ -144,7 +144,7 @@ public:
     /// Get the name of the file
     std::string getFilename() const override { return filename; }
 
-    std::uint64_t getHash() const override { return hash; }
+    std::string getHash() const override { return hash; }
 
     /// Get the version of the NIF format used
     unsigned int getVersion() const override { return ver; }

components/nifosg/nifloader.cpp

@@ -325,9 +325,7 @@ namespace NifOsg
             if (!textkeys->mTextKeys.empty())
                 created->getOrCreateUserDataContainer()->addUserObject(textkeys);
 
-            const std::uint64_t nifHash = nif->getHash();
-            created->setUserValue(Misc::OsgUserValues::sFileHash,
-                std::string(reinterpret_cast<const char*>(&nifHash), sizeof(nifHash)));
+            created->setUserValue(Misc::OsgUserValues::sFileHash, nif->getHash());
 
             return created;
         }

components/resource/bulletshape.hpp

@@ -1,6 +1,7 @@
 #ifndef OPENMW_COMPONENTS_RESOURCE_BULLETSHAPE_H
 #define OPENMW_COMPONENTS_RESOURCE_BULLETSHAPE_H
 
+#include <array>
 #include <map>
 #include <memory>
 
@@ -53,7 +54,7 @@ namespace Resource
         std::map<int, int> mAnimatedShapes;
 
         std::string mFileName;
-        std::uint64_t mFileHash = 0;
+        std::string mFileHash;
 
         void setLocalScaling(const btVector3& scale);
 

components/resource/bulletshapemanager.cpp

@@ -169,10 +169,7 @@ osg::ref_ptr<const BulletShape> BulletShapeManager::getShape(const std::string &
             if (shape != nullptr)
             {
                 shape->mFileName = normalized;
-                std::string fileHash;
-                constNode->getUserValue(Misc::OsgUserValues::sFileHash, fileHash);
-                if (!fileHash.empty())
-                    std::memcpy(&shape->mFileHash, fileHash.data(), std::min(fileHash.size(), sizeof(shape->mFileHash)));
+                constNode->getUserValue(Misc::OsgUserValues::sFileHash, shape->mFileHash);
             }
         }
 

components/resource/scenemanager.cpp

@@ -506,7 +506,7 @@ namespace Resource
             options->setReadFileCallback(new ImageReadCallback(imageManager));
             if (ext == "dae") options->setOptionString("daeUseSequencedTextureUnits");
 
-            const std::uint64_t fileHash = Files::getHash(normalizedFilename, model);
+            const std::array<std::uint64_t, 2> fileHash = Files::getHash(normalizedFilename, model);
 
             osgDB::ReaderWriter::ReadResult result = reader->readNode(model, options);
             if (!result.success())
@@ -538,7 +538,7 @@ namespace Resource
             }
 
             node->setUserValue(Misc::OsgUserValues::sFileHash,
-                std::string(reinterpret_cast<const char*>(&fileHash), sizeof(fileHash)));
+                std::string(reinterpret_cast<const char*>(fileHash.data()), fileHash.size() * sizeof(std::uint64_t)));
 
             return node;
         }