From e4c12e5e56fc86b6b341a503e6c7e8618f0323b0 Mon Sep 17 00:00:00 2001
From: scrawl <scrawl@baseoftrash.de>
Date: Thu, 23 Feb 2017 20:41:07 +0100
Subject: [PATCH] optimizer: don't merge Geometry that has transparency sorting
 enabled

---
 components/sceneutil/optimizer.cpp | 45 ++++++++++++++++++++++++++++++
 components/sceneutil/optimizer.hpp | 11 ++++++--
 2 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/components/sceneutil/optimizer.cpp b/components/sceneutil/optimizer.cpp
index 011669a54..b975e5463 100644
--- a/components/sceneutil/optimizer.cpp
+++ b/components/sceneutil/optimizer.cpp
@@ -1032,6 +1032,51 @@ bool isAbleToMerge(const osg::Geometry& g1, const osg::Geometry& g2)
 }
 
 
+void Optimizer::MergeGeometryVisitor::pushStateSet(osg::StateSet *stateSet)
+{
+    _stateSetStack.push_back(stateSet);
+    checkAllowedToMerge();
+}
+
+void Optimizer::MergeGeometryVisitor::popStateSet()
+{
+    _stateSetStack.pop_back();
+    checkAllowedToMerge();
+}
+
+void Optimizer::MergeGeometryVisitor::checkAllowedToMerge()
+{
+    int renderingHint = 0;
+    bool override = false;
+    for (std::vector<osg::StateSet*>::const_iterator it = _stateSetStack.begin(); it != _stateSetStack.end(); ++it)
+    {
+        osg::StateSet* stateSet = *it;
+        osg::StateSet::RenderBinMode mode = stateSet->getRenderBinMode();
+        if (override && (!mode & osg::StateSet::PROTECTED_RENDERBIN_DETAILS))
+            continue;
+        if (mode & osg::StateSet::USE_RENDERBIN_DETAILS)
+            renderingHint = stateSet->getRenderingHint();
+        if (mode & osg::StateSet::OVERRIDE_RENDERBIN_DETAILS)
+            override = true;
+    }
+    // Can't merge Geometry that are using a transparent sorting bin as that would cause the sorting to break.
+    _allowedToMerge = renderingHint != osg::StateSet::TRANSPARENT_BIN;
+}
+
+void Optimizer::MergeGeometryVisitor::apply(osg::Group &group)
+{
+    if (group.getStateSet())
+        pushStateSet(group.getStateSet());
+
+    if (_allowedToMerge)
+        mergeGroup(group);
+
+    traverse(group);
+
+    if (group.getStateSet())
+        popStateSet();
+}
+
 bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
 {
     if (!isOperationPermissibleForObject(&group)) return false;
diff --git a/components/sceneutil/optimizer.hpp b/components/sceneutil/optimizer.hpp
index e6ae6768b..804938074 100644
--- a/components/sceneutil/optimizer.hpp
+++ b/components/sceneutil/optimizer.hpp
@@ -367,7 +367,7 @@ class Optimizer
                 /// default to traversing all children.
                 MergeGeometryVisitor(Optimizer* optimizer=0) :
                     BaseOptimizerVisitor(optimizer, MERGE_GEOMETRY),
-                    _targetMaximumNumberOfVertices(10000) {}
+                    _targetMaximumNumberOfVertices(10000), _allowedToMerge(true) {}
 
                 void setTargetMaximumNumberOfVertices(unsigned int num)
                 {
@@ -379,7 +379,11 @@ class Optimizer
                     return _targetMaximumNumberOfVertices;
                 }
 
-                virtual void apply(osg::Group& group) { mergeGroup(group); traverse(group); }
+                void pushStateSet(osg::StateSet* stateSet);
+                void popStateSet();
+                void checkAllowedToMerge();
+
+                virtual void apply(osg::Group& group);
                 virtual void apply(osg::Billboard&) { /* don't do anything*/ }
 
                 bool mergeGroup(osg::Group& group);
@@ -397,7 +401,8 @@ class Optimizer
             protected:
 
                 unsigned int _targetMaximumNumberOfVertices;
-
+                std::vector<osg::StateSet*> _stateSetStack;
+                bool _allowedToMerge;
         };
 
 };