Merge branch '3' of https://gitlab.com/bzzt/openmw into shadows-bin

This won't actually work.

components/CMakeLists.txt

@@ -51,7 +51,7 @@ add_component_dir (shader
 add_component_dir (sceneutil
     clone attach visitor util statesetupdater controller skeleton riggeometry morphgeometry lightcontroller
     lightmanager lightutil positionattitudetransform workqueue unrefqueue pathgridutil waterutil writescene serialize optimizer
-    actorutil detourdebugdraw navmesh agentpath shadow mwshadowtechnique recastmesh
+    actorutil detourdebugdraw navmesh agentpath shadow mwshadowtechnique recastmesh shadowsbin
     )
 
 add_component_dir (nif

components/sceneutil/mwshadowtechnique.cpp

@@ -25,6 +25,7 @@
 #include <osg/Depth>
 
 #include <sstream>
+#include "shadowsbin.hpp"
 
 namespace {
 
@@ -273,10 +274,20 @@ void VDSMCameraCullCallback::operator()(osg::Node* node, osg::NodeVisitor* nv)
         cv->pushCullingSet();
     }
 #endif
+    // bin has to go inside camera cull or the rendertexture stage will override it
+    static osg::ref_ptr<osg::StateSet> ss;
+    if (!ss)
+    {
+        ShadowsBinAdder adder("ShadowsBin");
+        ss = new osg::StateSet;
+        ss->setRenderBinDetails(osg::StateSet::OPAQUE_BIN, "ShadowsBin", osg::StateSet::OVERRIDE_PROTECTED_RENDERBIN_DETAILS);
+    }
+    cv->pushStateSet(ss);
     if (_vdsm->getShadowedScene())
     {
         _vdsm->getShadowedScene()->osg::Group::traverse(*nv);
     }
+    cv->popStateSet();
 #if 1
     if (!_polytope.empty())
     {
@@ -1583,17 +1594,14 @@ void MWShadowTechnique::createShaders()
     _shadowCastingStateSet->setAttributeAndModes(_castingProgram, osg::StateAttribute::ON | osg::StateAttribute::OVERRIDE);
     // The casting program uses a sampler, so to avoid undefined behaviour, we must bind a dummy texture in case no other is supplied
     _shadowCastingStateSet->setTextureAttributeAndModes(0, _fallbackBaseTexture.get(), osg::StateAttribute::ON);
-    _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
+    _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", true));
     _shadowCastingStateSet->addUniform(_shadowMapAlphaTestDisableUniform);
     osg::ref_ptr<osg::Depth> depth = new osg::Depth;
     depth->setWriteMask(true);
     _shadowCastingStateSet->setAttribute(depth, osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE);
     _shadowCastingStateSet->setMode(GL_DEPTH_CLAMP, osg::StateAttribute::ON);
 
-    _shadowCastingStateSet->setRenderBinDetails(osg::StateSet::OPAQUE_BIN, "RenderBin", osg::StateSet::OVERRIDE_PROTECTED_RENDERBIN_DETAILS);
-
     // TODO: compare performance when alpha testing is handled here versus using a discard in the fragment shader
-    // TODO: compare performance when we set a bunch of GL state to the default here with OVERRIDE set so that there are fewer pointless state switches
 }
 
 osg::Polytope MWShadowTechnique::computeLightViewFrustumPolytope(Frustum& frustum, LightData& positionedLight)

components/sceneutil/shadowsbin.cpp

@@ -0,0 +1,133 @@
+#include "shadowsbin.hpp"
+#include <unordered_set>
+#include <osg/StateSet>
+#include <osg/Material>
+#include <osgUtil/StateGraph>
+
+using namespace osgUtil;
+
+namespace
+{
+    template <typename T>
+    inline void accumulateState(T& currentValue, T newValue, bool& isOverride, unsigned int overrideFlags)
+    {
+        if (isOverride && !(overrideFlags & osg::StateAttribute::PROTECTED)) return;
+
+        if (overrideFlags & osg::StateAttribute::OVERRIDE)
+            isOverride = true;
+
+        currentValue = newValue;
+    }
+
+    inline void accumulateModeState(const osg::StateSet* ss, bool& currentValue, bool& isOverride, int mode)
+    {
+        const osg::StateSet::ModeList& l = ss->getModeList();
+        osg::StateSet::ModeList::const_iterator mf = l.find(mode);
+        if (mf == l.end())
+            return;
+        int flags = mf->second;
+        bool newValue = flags & osg::StateAttribute::ON;
+        accumulateState(currentValue, newValue, isOverride, ss->getMode(mode));
+    }
+
+    inline bool materialNeedShadows(osg::Material* m)
+    {
+        return m->getDiffuse(osg::Material::FRONT).a() > 0.5;
+    }
+}
+
+namespace SceneUtil
+{
+
+ShadowsBin::ShadowsBin()
+{
+    mStateSet = new osg::StateSet;
+    mStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
+}
+
+bool ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::unordered_set<StateGraph*>& uninteresting)
+{
+    std::vector<StateGraph*> return_path;
+    State state;
+    StateGraph* sg_new = sg;
+    do
+    {
+        if (uninteresting.find(sg_new) != uninteresting.end())
+            break;
+        return_path.push_back(sg_new);
+        sg_new = sg_new->_parent;
+    } while (sg_new && sg_new != root);
+    for(std::vector<StateGraph*>::reverse_iterator itr=return_path.rbegin(); itr!=return_path.rend(); ++itr)
+    {
+        const osg::StateSet* ss = (*itr)->getStateSet();
+        if (!ss) continue;
+        accumulateModeState(ss, state.mAlphaBlend, state.mAlphaBlendOverride, GL_BLEND);
+        accumulateModeState(ss, state.mAlphaTest, state.mAlphaTestOverride, GL_ALPHA_TEST);
+        const osg::StateSet::AttributeList& l = ss->getAttributeList();
+        osg::StateSet::AttributeList::const_iterator f = l.find(std::make_pair(osg::StateAttribute::MATERIAL, 0));
+        if (f != l.end())
+        {
+            const osg::StateSet::RefAttributePair* rap = &f->second;
+            accumulateState(state.mMaterial, static_cast<osg::Material*>(rap->first.get()), state.mMaterialOverride, rap->second);
+            if (state.mMaterial && !materialNeedShadows(state.mMaterial))
+                state.mMaterial = nullptr;
+        }
+        f = l.find(std::make_pair(osg::StateAttribute::FRONTFACE, 0));
+        if (f != l.end())
+            state.mImportantState = true;
+        if ((*itr) != sg && !state.interesting())
+            uninteresting.insert(*itr);
+    }
+
+    if (!state.needShadows())
+        return true;
+
+    if (!state.needTexture() && !state.mImportantState)
+    {
+        for (RenderLeaf* leaf : sg->_leaves)
+        {
+            leaf->_parent = root;
+            root->_leaves.push_back(leaf);
+        }
+        return true;
+    }
+    return false;
+}
+
+bool ShadowsBin::State::needShadows() const
+{
+    if (!mMaterial)
+        return true;
+    return materialNeedShadows(mMaterial);
+}
+
+void ShadowsBin::sortImplementation()
+{
+    if (!_stateGraphList.size())
+        return;
+    StateGraph* root = _stateGraphList[0];
+    while (root->_parent)
+    {
+        root = root->_parent;
+        const osg::StateSet* ss = root->getStateSet();
+        if (ss->getMode(GL_NORMALIZE) & osg::StateAttribute::ON // that is root stategraph of renderingmanager cpp
+           || ss->getAttribute(osg::StateAttribute::VIEWPORT)) // fallback to rendertargets sg just in case
+            break;
+        if (!root->_parent)
+            return;
+    }
+    root = root->find_or_insert(mStateSet.get());
+    root->_leaves.reserve(_stateGraphList.size());
+    StateGraphList newList;
+    std::unordered_set<StateGraph*> uninteresting;
+    for (StateGraph* graph : _stateGraphList)
+    {
+        if (!cullStateGraph(graph, root, uninteresting))
+            newList.push_back(graph);
+    }
+    if (!root->_leaves.empty())
+        newList.push_back(root);
+    _stateGraphList = newList;
+}
+
+}

components/sceneutil/shadowsbin.hpp

@@ -0,0 +1,58 @@
+#ifndef OPENMW_COMPONENTS_SCENEUTIL_SHADOWBIN_H
+#define OPENMW_COMPONENTS_SCENEUTIL_SHADOWBIN_H
+#include <unordered_set>
+#include <osgUtil/RenderBin>
+
+namespace osg
+{
+    class Material;
+}
+
+namespace SceneUtil
+{
+
+    /// renderbin which culls redundent state for shadows rendering
+    class ShadowsBin : public osgUtil::RenderBin
+    {
+    private:
+        osg::ref_ptr<osg::StateSet> mStateSet;
+    public:
+         META_Object(SceneUtil, ShadowsBin)
+         ShadowsBin();
+         ShadowsBin(const ShadowsBin& rhs, const osg::CopyOp& copyop) : osgUtil::RenderBin(rhs, copyop), mStateSet(rhs.mStateSet) {}
+
+         virtual void sortImplementation();
+
+        struct State
+        {
+            State():mAlphaBlend(false),mAlphaBlendOverride(false),mAlphaTest(false),mAlphaTestOverride(false),mMaterial(nullptr),mMaterialOverride(false),mImportantState(false){}
+            bool mAlphaBlend;
+            bool mAlphaBlendOverride;
+            bool mAlphaTest;
+            bool mAlphaTestOverride;
+            osg::Material* mMaterial;
+            bool mMaterialOverride;
+            bool mImportantState;
+            bool needTexture() const { return mAlphaBlend || mAlphaTest; }
+            bool needShadows() const;
+            bool interesting() const { return !needShadows() || needTexture() || mAlphaBlendOverride || mAlphaTestOverride || mMaterialOverride || mImportantState; }
+        };
+
+        bool cullStateGraph(osgUtil::StateGraph* sg, osgUtil::StateGraph* root, std::unordered_set<osgUtil::StateGraph*>& uninteresting);
+
+         static void addPrototype(const std::string& name)
+         {
+             osg::ref_ptr<osgUtil::RenderBin> bin (new ShadowsBin);
+             osgUtil::RenderBin::addRenderBinPrototype(name, bin);
+         }
+    };
+
+    class ShadowsBinAdder
+    {
+        public:
+        ShadowsBinAdder(const std::string& name){ ShadowsBin::addPrototype(name); }
+    };
+
+}
+
+#endif