fixes bugs with share state (#3111)

* optimizer.cpp merge fix

* objectpaging.cpp

* optimizer.hpp setSharedStateManager

* optimizer.cpp shareState

* scenemanager.cpp shareState

* scenemanager.cpp

* optimizer.cpp

* optimizer.cpp

* scenemanager.cpp

* optimizer.cpp

apps/openmw/mwrender/objectpaging.cpp

@@ -651,7 +651,7 @@ namespace MWRender
             }
             optimizer.setIsOperationPermissibleForObjectCallback(new CanOptimizeCallback);
             unsigned int options = SceneUtil::Optimizer::FLATTEN_STATIC_TRANSFORMS|SceneUtil::Optimizer::REMOVE_REDUNDANT_NODES|SceneUtil::Optimizer::MERGE_GEOMETRY;
-            mSceneManager->shareState(mergeGroup);
+
             optimizer.optimize(mergeGroup, options);
 
             group->addChild(mergeGroup);

components/resource/scenemanager.cpp

@@ -659,22 +659,18 @@ namespace Resource
             osg::ref_ptr<Shader::ShaderVisitor> shaderVisitor (createShaderVisitor());
             loaded->accept(*shaderVisitor);
 
-            // share state
-            // do this before optimizing so the optimizer will be able to combine nodes more aggressively
-            // note, because StateSets will be shared at this point, StateSets can not be modified inside the optimizer
-            mSharedStateMutex.lock();
-            mSharedStateManager->share(loaded.get());
-            mSharedStateMutex.unlock();
-
             if (canOptimize(normalized))
             {
                 SceneUtil::Optimizer optimizer;
+                optimizer.setSharedStateManager(mSharedStateManager, &mSharedStateMutex);
                 optimizer.setIsOperationPermissibleForObjectCallback(new CanOptimizeCallback);
 
-                static const unsigned int options = getOptimizationOptions();
+                static const unsigned int options = getOptimizationOptions()|SceneUtil::Optimizer::SHARE_DUPLICATE_STATE;
 
                 optimizer.optimize(loaded, options);
             }
+            else
+                shareState(loaded);
 
             if (compile && mIncrementalCompileOperation)
                 mIncrementalCompileOperation->add(loaded);

components/sceneutil/optimizer.cpp

@@ -30,6 +30,8 @@
 #include <osg/io_utils>
 #include <osg/Depth>
 
+#include <osgDB/SharedStateManager>
+
 #include <osgUtil/TransformAttributeFunctor>
 #include <osgUtil/Statistics>
 #include <osgUtil/MeshOptimizers>
@@ -84,6 +86,13 @@ void Optimizer::optimize(osg::Node* node, unsigned int options)
         cstv.removeTransforms(node);
     }
 
+    if (options & SHARE_DUPLICATE_STATE && _sharedStateManager)
+    {
+        if (_sharedStateMutex) _sharedStateMutex->lock();
+        _sharedStateManager->share(node);
+        if (_sharedStateMutex) _sharedStateMutex->unlock();
+    }
+
     if (options & REMOVE_REDUNDANT_NODES)
     {
         OSG_INFO<<"Optimizer::optimize() doing REMOVE_REDUNDANT_NODES"<<std::endl;
@@ -741,7 +750,8 @@ bool Optimizer::CombineStaticTransformsVisitor::removeTransforms(osg::Node* node
         if (transform->getNumChildren()==1 &&
             transform->getChild(0)->asTransform()!=0 &&
             transform->getChild(0)->asTransform()->asMatrixTransform()!=0 &&
-            transform->getChild(0)->asTransform()->getDataVariance()==osg::Object::STATIC)
+            (!transform->getChild(0)->getStateSet() || transform->getChild(0)->getStateSet()->referenceCount()==1) &&
+            transform->getChild(0)->getDataVariance()==osg::Object::STATIC)
         {
             // now combine with its child.
             osg::MatrixTransform* child = transform->getChild(0)->asTransform()->asMatrixTransform();

components/sceneutil/optimizer.hpp

@@ -25,6 +25,12 @@
 //#include <osgUtil/Export>
 
 #include <set>
+#include <mutex>
+
+namespace osgDB
+{
+    class SharedStateManager;
+}
 
 //namespace osgUtil {
 namespace SceneUtil {
@@ -65,7 +71,7 @@ class Optimizer
 
     public:
 
-        Optimizer() : _mergeAlphaBlending(false) {}
+        Optimizer() : _mergeAlphaBlending(false), _sharedStateManager(nullptr), _sharedStateMutex(nullptr) {}
         virtual ~Optimizer() {}
 
         enum OptimizationOptions
@@ -121,6 +127,8 @@ class Optimizer
         void setMergeAlphaBlending(bool merge) { _mergeAlphaBlending = merge; }
         void setViewPoint(const osg::Vec3f& viewPoint) { _viewPoint = viewPoint; }
 
+        void setSharedStateManager(osgDB::SharedStateManager* sharedStateManager, std::mutex* sharedStateMutex) { _sharedStateMutex = sharedStateMutex; _sharedStateManager = sharedStateManager; }
+
         /** Reset internal data to initial state - the getPermissibleOptionsMap is cleared.*/
         void reset();
 
@@ -258,6 +266,9 @@ class Optimizer
         osg::Vec3f _viewPoint;
         bool _mergeAlphaBlending;
 
+        osgDB::SharedStateManager* _sharedStateManager;
+        mutable std::mutex* _sharedStateMutex;
+
     public:
 
         /** Flatten Static Transform nodes by applying their transform to the