Merge branch 'nifisjustice' into 'master'

NIF updates

See merge request OpenMW/openmw!394

components/nif/data.cpp

@@ -320,7 +320,7 @@ void NiSkinData::read(NIFStream *nif)
 
     int boneNum = nif->getInt();
     if (nif->getVersion() >= NIFFile::NIFVersion::VER_MW && nif->getVersion() <= NIFStream::generateVersion(10,1,0,0))
-        nif->skip(4); // NiSkinPartition link
+        partitions.read(nif);
 
     // Has vertex weights flag
     if (nif->getVersion() > NIFStream::generateVersion(4,2,1,0) && !nif->getBoolean())
@@ -345,6 +345,69 @@ void NiSkinData::read(NIFStream *nif)
     }
 }
 
+void NiSkinData::post(NIFFile *nif)
+{
+    partitions.post(nif);
+}
+
+void NiSkinPartition::read(NIFStream *nif)
+{
+    unsigned int num = nif->getUInt();
+    data.resize(num);
+    for (auto& partition : data)
+        partition.read(nif);
+}
+
+void NiSkinPartition::Partition::read(NIFStream *nif)
+{
+    unsigned short numVertices = nif->getUShort();
+    unsigned short numTriangles = nif->getUShort();
+    unsigned short numBones = nif->getUShort();
+    unsigned short numStrips = nif->getUShort();
+    unsigned short bonesPerVertex = nif->getUShort();
+    if (numBones)
+        nif->getUShorts(bones, numBones);
+
+    bool hasVertexMap = true;
+    if (nif->getVersion() >= NIFStream::generateVersion(10,1,0,0))
+        hasVertexMap = nif->getBoolean();
+    if (hasVertexMap && numVertices)
+        nif->getUShorts(vertexMap, numVertices);
+
+    bool hasVertexWeights = true;
+    if (nif->getVersion() >= NIFStream::generateVersion(10,1,0,0))
+        hasVertexWeights = nif->getBoolean();
+    if (hasVertexWeights && numVertices && bonesPerVertex)
+        nif->getFloats(weights, numVertices * bonesPerVertex);
+
+    std::vector<unsigned short> stripLengths;
+    if (numStrips)
+        nif->getUShorts(stripLengths, numStrips);
+
+    bool hasFaces = true;
+    if (nif->getVersion() >= NIFStream::generateVersion(10,1,0,0))
+        hasFaces = nif->getBoolean();
+    if (hasFaces)
+    {
+        if (numStrips)
+        {
+            strips.resize(numStrips);
+            for (unsigned short i = 0; i < numStrips; i++)
+                nif->getUShorts(strips[i], stripLengths[i]);
+        }
+        else if (numTriangles)
+            nif->getUShorts(triangles, numTriangles * 3);
+    }
+    bool hasBoneIndices = nif->getChar() != 0;
+    if (hasBoneIndices && numVertices && bonesPerVertex)
+        nif->getChars(boneIndices, numVertices * bonesPerVertex);
+    if (nif->getBethVersion() > NIFFile::BethVersion::BETHVER_FO3)
+    {
+        nif->getChar(); // LOD level
+        nif->getBoolean(); // Global VB
+    }
+}
+
 void NiMorphData::read(NIFStream *nif)
 {
     int morphCount = nif->getInt();
@@ -392,4 +455,29 @@ void NiPalette::read(NIFStream *nif)
         colors[i] = nif->getUInt() | alphaMask;
 }
 
+void NiStringPalette::read(NIFStream *nif)
+{
+    unsigned int size = nif->getUInt();
+    if (!size)
+        return;
+    std::vector<char> source;
+    nif->getChars(source, size);
+    if (nif->getUInt() != size)
+        nif->file->warn("Failed size check in NiStringPalette");
+    if (source[source.size()-1] != '\0')
+        source.emplace_back('\0');
+    const char* buffer = source.data();
+    while (static_cast<size_t>(buffer - source.data()) < source.size())
+    {
+        palette.emplace_back(buffer);
+        buffer += palette.back().size() + 1;
+    }
+}
+
+void NiBoolData::read(NIFStream *nif)
+{
+    mKeyList = std::make_shared<ByteKeyMap>();
+    mKeyList->read(nif);
+}
+
 } // Namespace

components/nif/data.hpp

@@ -174,6 +174,7 @@ class NiSkinInstance : public Record
 {
 public:
     NiSkinDataPtr data;
+    NiSkinPartitionPtr partitions;
     NodePtr root;
     NodeList bones;
 
@@ -200,6 +201,25 @@ public:
 
     Transformation trafo;
     std::vector<BoneInfo> bones;
+    NiSkinPartitionPtr partitions;
+
+    void read(NIFStream *nif) override;
+    void post(NIFFile *nif) override;
+};
+
+struct NiSkinPartition : public Record
+{
+    struct Partition
+    {
+        std::vector<unsigned short> bones;
+        std::vector<unsigned short> vertexMap;
+        std::vector<float> weights;
+        std::vector<std::vector<unsigned short>> strips;
+        std::vector<unsigned short> triangles;
+        std::vector<char> boneIndices;
+        void read(NIFStream *nif);
+    };
+    std::vector<Partition> data;
 
     void read(NIFStream *nif) override;
 };
@@ -240,5 +260,17 @@ public:
     void read(NIFStream *nif) override;
 };
 
+struct NiStringPalette : public Record
+{
+    std::vector<std::string> palette;
+    void read(NIFStream *nif) override;
+};
+
+struct NiBoolData : public Record
+{
+    ByteKeyMapPtr mKeyList;
+    void read(NIFStream *nif) override;
+};
+
 } // Namespace
 #endif

components/nif/niffile.cpp

@@ -1,9 +1,12 @@
 #include "niffile.hpp"
 #include "effect.hpp"
 
+#include <array>
 #include <map>
 #include <sstream>
 
+#include <components/settings/settings.hpp>
+
 namespace Nif
 {
 
@@ -113,6 +116,9 @@ static std::map<std::string,RecordFactoryEntry> makeFactory()
     factory["NiColorExtraData"]             = {&construct <NiVectorExtraData>           , RC_NiColorExtraData           };
     factory["NiFloatExtraData"]             = {&construct <NiFloatExtraData>            , RC_NiFloatExtraData           };
     factory["NiFloatsExtraData"]            = {&construct <NiFloatsExtraData>           , RC_NiFloatsExtraData          };
+    factory["NiStringPalette"]              = {&construct <NiStringPalette>             , RC_NiStringPalette            };
+    factory["NiBoolData"]                   = {&construct <NiBoolData>                  , RC_NiBoolData                 };
+    factory["NiSkinPartition"]              = {&construct <NiSkinPartition>             , RC_NiSkinPartition            };
     return factory;
 }
 
@@ -137,15 +143,46 @@ void NIFFile::parse(Files::IStreamPtr stream)
 
     // Check the header string
     std::string head = nif.getVersionString();
-    if(head.compare(0, 22, "NetImmerse File Format") != 0)
+    static const std::array<std::string, 2> verStrings =
+    {
+        "NetImmerse File Format",
+        "Gamebryo File Format"
+    };
+    bool supported = false;
+    for (const std::string& verString : verStrings)
+    {
+        supported = (head.compare(0, verString.size(), verString) == 0);
+        if (supported)
+            break;
+    }
+    if (!supported)
         fail("Invalid NIF header: " + head);
 
+    supported = false;
+
     // Get BCD version
     ver = nif.getUInt();
     // 4.0.0.0 is an older, practically identical version of the format.
     // It's not used by Morrowind assets but Morrowind supports it.
-    if(ver != NIFStream::generateVersion(4,0,0,0) && ver != VER_MW)
-        fail("Unsupported NIF version: " + printVersion(ver));
+    static const std::array<uint32_t, 2> supportedVers =
+    {
+        NIFStream::generateVersion(4,0,0,0),
+        VER_MW
+    };
+    for (uint32_t supportedVer : supportedVers)
+    {
+        supported = (ver == supportedVer);
+        if (supported)
+            break;
+    }
+    if (!supported)
+    {
+        static const bool ignoreUnsupported = Settings::Manager::getBool("load unsupported nif files", "Models");
+        if (ignoreUnsupported)
+            warn("Unsupported NIF version: " + printVersion(ver) + ". Proceed with caution!");
+        else
+            fail("Unsupported NIF version: " + printVersion(ver));
+    }
 
     // NIF data endianness
     if (ver >= NIFStream::generateVersion(20,0,0,4))
@@ -245,6 +282,9 @@ void NIFFile::parse(Files::IStreamPtr stream)
         else
             fail("Unknown record type " + rec);
 
+        if (!supported)
+            Log(Debug::Verbose) << "NIF Debug: Reading record of type " << rec << ", index " << i << " (" << filename << ")";
+
         assert(r != nullptr);
         assert(r->recType != RC_MISSING);
         r->recName = rec;

components/nif/record.hpp

@@ -109,7 +109,10 @@ enum RecordType
   RC_NiVectorExtraData,
   RC_NiColorExtraData,
   RC_NiFloatExtraData,
-  RC_NiFloatsExtraData
+  RC_NiFloatsExtraData,
+  RC_NiStringPalette,
+  RC_NiBoolData,
+  RC_NiSkinPartition
 };
 
 /// Base class for all records

components/nif/recordptr.hpp

@@ -143,6 +143,8 @@ class NiAutoNormalParticlesData;
 class NiPalette;
 struct NiParticleModifier;
 struct NiLinesData;
+struct NiBoolData;
+struct NiSkinPartition;
 
 using NodePtr = RecordPtrT<Node>;
 using ExtraPtr = RecordPtrT<Extra>;
@@ -166,6 +168,8 @@ using NiRotatingParticlesDataPtr = RecordPtrT<NiRotatingParticlesData>;
 using NiAutoNormalParticlesDataPtr = RecordPtrT<NiAutoNormalParticlesData>;
 using NiPalettePtr = RecordPtrT<NiPalette>;
 using NiParticleModifierPtr = RecordPtrT<NiParticleModifier>;
+using NiBoolDataPtr = RecordPtrT<NiBoolData>;
+using NiSkinPartitionPtr = RecordPtrT<NiSkinPartition>;
 
 using NodeList = RecordListT<Node>;
 using PropertyList = RecordListT<Property>;

components/nifbullet/bulletnifloader.cpp

@@ -8,6 +8,7 @@
 
 #include <components/debug/debuglog.hpp>
 
+#include <components/misc/convert.hpp>
 #include <components/misc/stringops.hpp>
 
 #include <components/nif/node.hpp>
@@ -24,11 +25,6 @@ osg::Matrixf getWorldTransform(const Nif::Node *node)
     return node->trafo.toMatrix();
 }
 
-btVector3 getbtVector(const osg::Vec3f &v)
-{
-    return btVector3(v.x(), v.y(), v.z());
-}
-
 bool pathFileNameStartsWithX(const std::string& path)
 {
     const std::size_t slashpos = path.find_last_of("/\\");
@@ -36,7 +32,7 @@ bool pathFileNameStartsWithX(const std::string& path)
     return letterPos < path.size() && (path[letterPos] == 'x' || path[letterPos] == 'X');
 }
 
-void fillTriangleMeshWithTransform(btTriangleMesh& mesh, const Nif::NiTriShapeData& data, const osg::Matrixf &transform)
+void fillTriangleMesh(btTriangleMesh& mesh, const Nif::NiTriShapeData& data, const osg::Matrixf &transform)
 {
     mesh.preallocateVertices(static_cast<int>(data.vertices.size()));
     mesh.preallocateIndices(static_cast<int>(data.triangles.size()));
@@ -47,20 +43,20 @@ void fillTriangleMeshWithTransform(btTriangleMesh& mesh, const Nif::NiTriShapeDa
     for (std::size_t i = 0; i < triangles.size(); i += 3)
     {
         mesh.addTriangle(
-            getbtVector(vertices[triangles[i + 0]] * transform),
-            getbtVector(vertices[triangles[i + 1]] * transform),
-            getbtVector(vertices[triangles[i + 2]] * transform)
+            Misc::Convert::toBullet(vertices[triangles[i + 0]] * transform),
+            Misc::Convert::toBullet(vertices[triangles[i + 1]] * transform),
+            Misc::Convert::toBullet(vertices[triangles[i + 2]] * transform)
         );
     }
 }
 
-void fillTriangleMeshWithTransform(btTriangleMesh& mesh, const Nif::NiTriStripsData& data, const osg::Matrixf &transform)
+void fillTriangleMesh(btTriangleMesh& mesh, const Nif::NiTriStripsData& data, const osg::Matrixf &transform)
 {
     const std::vector<osg::Vec3f> &vertices = data.vertices;
     const std::vector<std::vector<unsigned short>> &strips = data.strips;
     if (vertices.empty() || strips.empty())
         return;
-    mesh.preallocateVertices(static_cast<int>(data.vertices.size()));
+    mesh.preallocateVertices(static_cast<int>(vertices.size()));
     int numTriangles = 0;
     for (const std::vector<unsigned short>& strip : strips)
     {
@@ -88,17 +84,17 @@ void fillTriangleMeshWithTransform(btTriangleMesh& mesh, const Nif::NiTriStripsD
                 if (i%2==0)
                 {
                     mesh.addTriangle(
-                        getbtVector(vertices[a] * transform),
-                        getbtVector(vertices[b] * transform),
-                        getbtVector(vertices[c] * transform)
+                        Misc::Convert::toBullet(vertices[a] * transform),
+                        Misc::Convert::toBullet(vertices[b] * transform),
+                        Misc::Convert::toBullet(vertices[c] * transform)
                     );
                 }
                 else
                 {
                     mesh.addTriangle(
-                        getbtVector(vertices[a] * transform),
-                        getbtVector(vertices[c] * transform),
-                        getbtVector(vertices[b] * transform)
+                        Misc::Convert::toBullet(vertices[a] * transform),
+                        Misc::Convert::toBullet(vertices[c] * transform),
+                        Misc::Convert::toBullet(vertices[b] * transform)
                     );
                 }
             }
@@ -106,17 +102,12 @@ void fillTriangleMeshWithTransform(btTriangleMesh& mesh, const Nif::NiTriStripsD
     }
 }
 
-void fillTriangleMeshWithTransform(btTriangleMesh& mesh, const Nif::Node* nifNode, const osg::Matrixf &transform)
+void fillTriangleMesh(btTriangleMesh& mesh, const Nif::Node* nifNode, const osg::Matrixf &transform = osg::Matrixf())
 {
     if (nifNode->recType == Nif::RC_NiTriShape)
-        fillTriangleMeshWithTransform(mesh, static_cast<const Nif::NiTriShape*>(nifNode)->data.get(), transform);
-    else // if (nifNode->recType == Nif::RC_NiTriStrips)
-        fillTriangleMeshWithTransform(mesh, static_cast<const Nif::NiTriStrips*>(nifNode)->data.get(), transform);
-}
-
-void fillTriangleMesh(btTriangleMesh& mesh, const Nif::Node* node)
-{
-    fillTriangleMeshWithTransform(mesh, node, osg::Matrixf());
+        fillTriangleMesh(mesh, static_cast<const Nif::NiTriShape*>(nifNode)->data.get(), transform);
+    else if (nifNode->recType == Nif::RC_NiTriStrips)
+        fillTriangleMesh(mesh, static_cast<const Nif::NiTriStrips*>(nifNode)->data.get(), transform);
 }
 
 }
@@ -149,10 +140,12 @@ osg::ref_ptr<Resource::BulletShape> BulletNifLoader::load(const Nif::File& nif)
 
     if (findBoundingBox(node))
     {
+        const btVector3 halfExtents = Misc::Convert::toBullet(mShape->mCollisionBoxHalfExtents);
+        const btVector3 origin = Misc::Convert::toBullet(mShape->mCollisionBoxTranslate);
         std::unique_ptr<btCompoundShape> compound (new btCompoundShape);
-        std::unique_ptr<btBoxShape> boxShape(new btBoxShape(getbtVector(mShape->mCollisionBoxHalfExtents)));
+        std::unique_ptr<btBoxShape> boxShape(new btBoxShape(halfExtents));
         btTransform transform = btTransform::getIdentity();
-        transform.setOrigin(getbtVector(mShape->mCollisionBoxTranslate));
+        transform.setOrigin(origin);
         compound->addChildShape(transform, boxShape.get());
         boxShape.release();
 
@@ -383,7 +376,7 @@ void BulletNifLoader::handleNiTriShape(const Nif::Node *nifNode, int flags, cons
         if (!mAvoidStaticMesh)
             mAvoidStaticMesh.reset(new btTriangleMesh(false));
 
-        fillTriangleMeshWithTransform(*mAvoidStaticMesh, nifNode, transform);
+        fillTriangleMesh(*mAvoidStaticMesh, nifNode, transform);
     }
     else
     {
@@ -391,7 +384,7 @@ void BulletNifLoader::handleNiTriShape(const Nif::Node *nifNode, int flags, cons
             mStaticMesh.reset(new btTriangleMesh(false));
 
         // Static shape, just transform all vertices into position
-        fillTriangleMeshWithTransform(*mStaticMesh, nifNode, transform);
+        fillTriangleMesh(*mStaticMesh, nifNode, transform);
     }
 }
 

components/nifosg/nifloader.cpp

@@ -60,6 +60,18 @@ namespace
         }
     }
 
+    bool isTypeGeometry(int type)
+    {
+        switch (type)
+        {
+            case Nif::RC_NiTriShape:
+            case Nif::RC_NiTriStrips:
+            case Nif::RC_NiLines:
+                return true;
+        }
+        return false;
+    }
+
     // Collect all properties affecting the given drawable that should be handled on drawable basis rather than on the node hierarchy above it.
     void collectDrawableProperties(const Nif::Node* nifNode, std::vector<const Nif::Property*>& out)
     {
@@ -528,7 +540,19 @@ namespace NifOsg
             // - finding a random child NiNode in NiBspArrayController
             node->setUserValue("recIndex", nifNode->recIndex);
 
+            std::vector<Nif::ExtraPtr> extraCollection;
+
             for (Nif::ExtraPtr e = nifNode->extra; !e.empty(); e = e->next)
+                extraCollection.emplace_back(e);
+
+            for (size_t i = 0; i < nifNode->extralist.length(); ++i)
+            {
+                Nif::ExtraPtr e = nifNode->extralist[i];
+                if (!e.empty())
+                    extraCollection.emplace_back(e);
+            }
+
+            for (const auto& e : extraCollection)
             {
                 if(e->recType == Nif::RC_NiTextKeyExtraData && textKeys)
                 {
@@ -584,7 +608,7 @@ namespace NifOsg
 
             applyNodeProperties(nifNode, node, composite, imageManager, boundTextures, animflags);
 
-            const bool isGeometry = nifNode->recType == Nif::RC_NiTriShape || nifNode->recType == Nif::RC_NiTriStrips || nifNode->recType == Nif::RC_NiLines;
+            const bool isGeometry = isTypeGeometry(nifNode->recType);
 
             if (isGeometry && !skipMeshes)
             {
@@ -1175,7 +1199,7 @@ namespace NifOsg
 
         void handleGeometry(const Nif::Node* nifNode, osg::Group* parentNode, SceneUtil::CompositeStateSetUpdater* composite, const std::vector<unsigned int>& boundTextures, int animflags)
         {
-            assert(nifNode->recType == Nif::RC_NiTriShape || nifNode->recType == Nif::RC_NiTriStrips || nifNode->recType == Nif::RC_NiLines);
+            assert(isTypeGeometry(nifNode->recType));
             osg::ref_ptr<osg::Drawable> drawable;
             osg::ref_ptr<osg::Geometry> geom (new osg::Geometry);
             handleNiGeometry(nifNode, geom, parentNode, composite, boundTextures, animflags);
@@ -1220,7 +1244,7 @@ namespace NifOsg
         void handleSkinnedGeometry(const Nif::Node *nifNode, osg::Group *parentNode, SceneUtil::CompositeStateSetUpdater* composite,
                                           const std::vector<unsigned int>& boundTextures, int animflags)
         {
-            assert(nifNode->recType == Nif::RC_NiTriShape || nifNode->recType == Nif::RC_NiTriStrips || nifNode->recType == Nif::RC_NiLines);
+            assert(isTypeGeometry(nifNode->recType));
             osg::ref_ptr<osg::Geometry> geometry (new osg::Geometry);
             handleNiGeometry(nifNode, geometry, parentNode, composite, boundTextures, animflags);
             osg::ref_ptr<SceneUtil::RigGeometry> rig(new SceneUtil::RigGeometry);

docs/source/reference/modding/settings/index.rst

@@ -58,3 +58,4 @@ The ranges included with each setting are the physically possible ranges, not re
 	windows
 	navigator
 	physics
+	models

docs/source/reference/modding/settings/models.rst

@@ -0,0 +1,31 @@
+Models Settings
+###############
+
+load unsupported nif files
+--------------------------
+
+:Type:		boolean
+:Range:		True/False
+:Default:	False
+
+Allow the engine to load arbitrary NIF files as long as they appear to be valid.
+
+OpenMW has limited and **experimental** support for NIF files
+that Morrowind itself cannot load, which normally goes unused.
+
+If enabled, this setting allows the NIF loader to make use of that functionality.
+
+.. warning::
+	You must keep in mind that since the mentioned support is experimental,
+	loading unsupported NIF files may fail, and the degree of this failure may vary.
+	
+	In milder cases, OpenMW will reject the file anyway because
+	it lacks a definition for a certain record type that the file may use.
+	
+	In more severe cases OpenMW's incomplete understanding of a record type
+	can lead to memory corruption, freezes or even crashes.
+	
+	**Do not enable** this if you're not so sure that you know what you're doing.
+
+To help debug possible issues OpenMW will log its progress in loading
+every file that uses an unsupported NIF version.

files/settings-default.cfg

@@ -945,3 +945,8 @@ lineofsight keep inactive cache = 0
 
 # Defer bounding boxes update until collision detection.
 defer aabb update = true
+
+[Models]
+# Attempt to load any valid NIF file regardless of its version and track the progress.
+# Loading arbitrary meshes is not advised and may cause instability.
+load unsupported nif files = false