Merge branch 'shadows-bin' into 'master'

Use a custom renderbin to avoid pointless OpenGL state switches

See merge request OpenMW/openmw!402

apps/openmw/mwrender/animation.cpp

@@ -16,7 +16,6 @@
 
 #include <components/resource/scenemanager.hpp>
 #include <components/resource/keyframemanager.hpp>
-#include <components/resource/resourcesystem.hpp>
 
 #include <components/misc/constants.hpp>
 #include <components/misc/resourcehelpers.hpp>
@@ -37,8 +36,6 @@
 
 #include <components/settings/settings.hpp>
 
-#include <components/shader/shadermanager.hpp>
-
 #include "../mwbase/environment.hpp"
 #include "../mwbase/world.hpp"
 #include "../mwworld/esmstore.hpp"
@@ -493,9 +490,8 @@ namespace MWRender
     class TransparencyUpdater : public SceneUtil::StateSetUpdater
     {
     public:
-        TransparencyUpdater(const float alpha, osg::ref_ptr<osg::Uniform> shadowUniform)
+        TransparencyUpdater(const float alpha)
             : mAlpha(alpha)
-            , mShadowUniform(shadowUniform)
         {
         }
 
@@ -509,9 +505,6 @@ namespace MWRender
         {
             osg::BlendFunc* blendfunc (new osg::BlendFunc);
             stateset->setAttributeAndModes(blendfunc, osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE);
-            // TODO: don't do this anymore once custom shadow renderbin is handling it
-            if (mShadowUniform)
-                stateset->addUniform(mShadowUniform);
 
             stateset->setRenderingHint(osg::StateSet::TRANSPARENT_BIN);
             stateset->setRenderBinMode(osg::StateSet::OVERRIDE_RENDERBIN_DETAILS);
@@ -533,7 +526,6 @@ namespace MWRender
 
     private:
         float mAlpha;
-        osg::ref_ptr<osg::Uniform> mShadowUniform;
     };
 
     struct Animation::AnimSource
@@ -1773,7 +1765,7 @@ namespace MWRender
         {
             if (mTransparencyUpdater == nullptr)
             {
-                mTransparencyUpdater = new TransparencyUpdater(alpha, mResourceSystem->getSceneManager()->getShaderManager().getShadowMapAlphaTestEnableUniform());
+                mTransparencyUpdater = new TransparencyUpdater(alpha);
                 mObjectRoot->addCullCallback(mTransparencyUpdater);
             }
             else

components/CMakeLists.txt

@@ -51,7 +51,7 @@ add_component_dir (shader
 add_component_dir (sceneutil
     clone attach visitor util statesetupdater controller skeleton riggeometry morphgeometry lightcontroller
     lightmanager lightutil positionattitudetransform workqueue unrefqueue pathgridutil waterutil writescene serialize optimizer
-    actorutil detourdebugdraw navmesh agentpath shadow mwshadowtechnique recastmesh
+    actorutil detourdebugdraw navmesh agentpath shadow mwshadowtechnique recastmesh shadowsbin
     )
 
 add_component_dir (nif

components/sceneutil/mwshadowtechnique.cpp

@@ -25,6 +25,7 @@
 #include <osg/Depth>
 
 #include <sstream>
+#include "shadowsbin.hpp"
 
 namespace {
 
@@ -273,10 +274,20 @@ void VDSMCameraCullCallback::operator()(osg::Node* node, osg::NodeVisitor* nv)
         cv->pushCullingSet();
     }
 #endif
+    // bin has to go inside camera cull or the rendertexture stage will override it
+    static osg::ref_ptr<osg::StateSet> ss;
+    if (!ss)
+    {
+        ShadowsBinAdder adder("ShadowsBin");
+        ss = new osg::StateSet;
+        ss->setRenderBinDetails(osg::StateSet::OPAQUE_BIN, "ShadowsBin", osg::StateSet::OVERRIDE_PROTECTED_RENDERBIN_DETAILS);
+    }
+    cv->pushStateSet(ss);
     if (_vdsm->getShadowedScene())
     {
         _vdsm->getShadowedScene()->osg::Group::traverse(*nv);
     }
+    cv->popStateSet();
 #if 1
     if (!_polytope.empty())
     {
@@ -875,15 +886,6 @@ void SceneUtil::MWShadowTechnique::setupCastingShader(Shader::ShaderManager & sh
 
     _castingProgram->addShader(shaderManager.getShader("shadowcasting_vertex.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::VERTEX));
     _castingProgram->addShader(shaderManager.getShader("shadowcasting_fragment.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::FRAGMENT));
-
-    _shadowMapAlphaTestDisableUniform = shaderManager.getShadowMapAlphaTestDisableUniform();
-    _shadowMapAlphaTestDisableUniform->setName("alphaTestShadows");
-    _shadowMapAlphaTestDisableUniform->setType(osg::Uniform::BOOL);
-    _shadowMapAlphaTestDisableUniform->set(false);
-
-    shaderManager.getShadowMapAlphaTestEnableUniform()->setName("alphaTestShadows");
-    shaderManager.getShadowMapAlphaTestEnableUniform()->setType(osg::Uniform::BOOL);
-    shaderManager.getShadowMapAlphaTestEnableUniform()->set(true);
 }
 
 MWShadowTechnique::ViewDependentData* MWShadowTechnique::createViewDependentData(osgUtil::CullVisitor* /*cv*/)
@@ -1584,17 +1586,14 @@ void MWShadowTechnique::createShaders()
     _shadowCastingStateSet->setAttributeAndModes(_castingProgram, osg::StateAttribute::ON | osg::StateAttribute::OVERRIDE);
     // The casting program uses a sampler, so to avoid undefined behaviour, we must bind a dummy texture in case no other is supplied
     _shadowCastingStateSet->setTextureAttributeAndModes(0, _fallbackBaseTexture.get(), osg::StateAttribute::ON);
-    _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
-    _shadowCastingStateSet->addUniform(_shadowMapAlphaTestDisableUniform);
+    _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", true));
+    _shadowCastingStateSet->addUniform(new osg::Uniform("alphaTestShadows", false));
     osg::ref_ptr<osg::Depth> depth = new osg::Depth;
     depth->setWriteMask(true);
     _shadowCastingStateSet->setAttribute(depth, osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE);
     _shadowCastingStateSet->setMode(GL_DEPTH_CLAMP, osg::StateAttribute::ON);
 
-    _shadowCastingStateSet->setRenderBinDetails(osg::StateSet::OPAQUE_BIN, "RenderBin", osg::StateSet::OVERRIDE_PROTECTED_RENDERBIN_DETAILS);
-
     // TODO: compare performance when alpha testing is handled here versus using a discard in the fragment shader
-    // TODO: compare performance when we set a bunch of GL state to the default here with OVERRIDE set so that there are fewer pointless state switches
 }
 
 osg::Polytope MWShadowTechnique::computeLightViewFrustumPolytope(Frustum& frustum, LightData& positionedLight)

components/sceneutil/mwshadowtechnique.hpp

@@ -288,7 +288,6 @@ namespace SceneUtil {
 
         osg::ref_ptr<DebugHUD>                  _debugHud;
         osg::ref_ptr<osg::Program>              _castingProgram;
-        osg::ref_ptr<osg::Uniform>              _shadowMapAlphaTestDisableUniform;
     };
 
 }

components/sceneutil/shadowsbin.cpp

@@ -0,0 +1,166 @@
+#include "shadowsbin.hpp"
+#include <unordered_set>
+#include <osg/StateSet>
+#include <osg/Material>
+#include <osgUtil/StateGraph>
+
+using namespace osgUtil;
+
+namespace
+{
+    template <typename T>
+    inline void accumulateState(T& currentValue, T newValue, bool& isOverride, unsigned int overrideFlags)
+    {
+        if (isOverride && !(overrideFlags & osg::StateAttribute::PROTECTED)) return;
+
+        if (overrideFlags & osg::StateAttribute::OVERRIDE)
+            isOverride = true;
+
+        currentValue = newValue;
+    }
+
+    inline void accumulateModeState(const osg::StateSet* ss, bool& currentValue, bool& isOverride, int mode)
+    {
+        const osg::StateSet::ModeList& l = ss->getModeList();
+        osg::StateSet::ModeList::const_iterator mf = l.find(mode);
+        if (mf == l.end())
+            return;
+        int flags = mf->second;
+        bool newValue = flags & osg::StateAttribute::ON;
+        accumulateState(currentValue, newValue, isOverride, ss->getMode(mode));
+    }
+
+    inline bool materialNeedShadows(osg::Material* m)
+    {
+        // I'm pretty sure this needs to check the colour mode - vertex colours might override this value.
+        return m->getDiffuse(osg::Material::FRONT).a() > 0.5;
+    }
+}
+
+namespace SceneUtil
+{
+
+ShadowsBin::ShadowsBin()
+{
+    mNoTestStateSet = new osg::StateSet;
+    mNoTestStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
+    mNoTestStateSet->addUniform(new osg::Uniform("alphaTestShadows", false));
+
+    mShaderAlphaTestStateSet = new osg::StateSet;
+    mShaderAlphaTestStateSet->addUniform(new osg::Uniform("alphaTestShadows", true));
+    mShaderAlphaTestStateSet->setMode(GL_BLEND, osg::StateAttribute::OFF | osg::StateAttribute::OVERRIDE);
+}
+
+StateGraph* ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::unordered_set<StateGraph*>& uninterestingCache)
+{
+    std::vector<StateGraph*> return_path;
+    State state;
+    StateGraph* sg_new = sg;
+    do
+    {
+        if (uninterestingCache.find(sg_new) != uninterestingCache.end())
+            break;
+        return_path.push_back(sg_new);
+        sg_new = sg_new->_parent;
+    } while (sg_new && sg_new != root);
+
+    for(auto itr=return_path.rbegin(); itr!=return_path.rend(); ++itr)
+    {
+        const osg::StateSet* ss = (*itr)->getStateSet();
+        if (!ss)
+            continue;
+
+        accumulateModeState(ss, state.mAlphaBlend, state.mAlphaBlendOverride, GL_BLEND);
+        accumulateModeState(ss, state.mAlphaTest, state.mAlphaTestOverride, GL_ALPHA_TEST);
+
+        const osg::StateSet::AttributeList& attributes = ss->getAttributeList();
+        osg::StateSet::AttributeList::const_iterator found = attributes.find(std::make_pair(osg::StateAttribute::MATERIAL, 0));
+        if (found != attributes.end())
+        {
+            const osg::StateSet::RefAttributePair& rap = found->second;
+            accumulateState(state.mMaterial, static_cast<osg::Material*>(rap.first.get()), state.mMaterialOverride, rap.second);
+            if (state.mMaterial && !materialNeedShadows(state.mMaterial))
+                state.mMaterial = nullptr;
+        }
+
+        // osg::FrontFace specifies triangle winding, not front-face culling. We can't safely reparent anything under it.
+        found = attributes.find(std::make_pair(osg::StateAttribute::FRONTFACE, 0));
+        if (found != attributes.end())
+            state.mImportantState = true;
+
+        if ((*itr) != sg && !state.interesting())
+            uninterestingCache.insert(*itr);
+    }
+
+    if (!state.needShadows())
+        return nullptr;
+
+    if (!state.needTexture() && !state.mImportantState)
+    {
+        for (RenderLeaf* leaf : sg->_leaves)
+        {
+            leaf->_parent = root;
+            root->_leaves.push_back(leaf);
+        }
+        return nullptr;
+    }
+
+    if (state.mAlphaBlend)
+    {
+        sg_new = sg->find_or_insert(mShaderAlphaTestStateSet);
+        for (RenderLeaf* leaf : sg->_leaves)
+        {
+            leaf->_parent = sg_new;
+            sg_new->_leaves.push_back(leaf);
+        }
+        return sg_new;
+    }
+    return sg;
+}
+
+bool ShadowsBin::State::needShadows() const
+{
+    if (!mMaterial)
+        return true;
+    return materialNeedShadows(mMaterial);
+}
+
+void ShadowsBin::sortImplementation()
+{
+    // The cull visitor contains a stategraph.
+    // When a stateset is pushed, it's added/found as a child of the current stategraph node, then that node becomes the new current stategraph node.
+    // When a drawable is added, the current stategraph node is added to the current renderbin (if it's not there already) and the drawable is added as a renderleaf to the stategraph
+    // This means our list only contains stategraph nodes with directly-attached renderleaves, but they might have parents with more state set that needs to be considered.
+    if (!_stateGraphList.size())
+        return;
+    StateGraph* root = _stateGraphList[0];
+    while (root->_parent)
+    {
+        root = root->_parent;
+        const osg::StateSet* ss = root->getStateSet();
+        if (ss->getMode(GL_NORMALIZE) & osg::StateAttribute::ON // that is root stategraph of renderingmanager cpp
+           || ss->getAttribute(osg::StateAttribute::VIEWPORT)) // fallback to rendertargets sg just in case
+            break;
+        if (!root->_parent)
+            return;
+    }
+    StateGraph* noTestRoot = root->find_or_insert(mNoTestStateSet.get());
+    // root is now a stategraph with useDiffuseMapForShadowAlpha disabled but minimal other state
+    noTestRoot->_leaves.reserve(_stateGraphList.size());
+    StateGraphList newList;
+    std::unordered_set<StateGraph*> uninterestingCache;
+    for (StateGraph* graph : _stateGraphList)
+    {
+        // Render leaves which shouldn't use the diffuse map for shadow alpha but do cast shadows become children of root, so graph is now empty. Don't add to newList.
+        // Graphs containing just render leaves which don't cast shadows are discarded. Don't add to newList.
+        // Graphs containing other leaves need to be in newList.
+        StateGraph* graphToAdd = cullStateGraph(graph, noTestRoot, uninterestingCache);
+        if (graphToAdd)
+            newList.push_back(graphToAdd);
+    }
+    if (!noTestRoot->_leaves.empty())
+        newList.push_back(noTestRoot);
+    _stateGraphList = newList;
+}
+
+}

components/sceneutil/shadowsbin.hpp

@@ -0,0 +1,76 @@
+#ifndef OPENMW_COMPONENTS_SCENEUTIL_SHADOWBIN_H
+#define OPENMW_COMPONENTS_SCENEUTIL_SHADOWBIN_H
+#include <unordered_set>
+#include <osgUtil/RenderBin>
+
+namespace osg
+{
+    class Material;
+}
+
+namespace SceneUtil
+{
+
+    /// renderbin which culls redundant state for shadow map rendering
+    class ShadowsBin : public osgUtil::RenderBin
+    {
+    private:
+        osg::ref_ptr<osg::StateSet> mNoTestStateSet;
+        osg::ref_ptr<osg::StateSet> mShaderAlphaTestStateSet;
+    public:
+        META_Object(SceneUtil, ShadowsBin)
+        ShadowsBin();
+        ShadowsBin(const ShadowsBin& rhs, const osg::CopyOp& copyop)
+            : osgUtil::RenderBin(rhs, copyop)
+            , mNoTestStateSet(rhs.mNoTestStateSet)
+            , mShaderAlphaTestStateSet(rhs.mShaderAlphaTestStateSet)
+            {}
+
+        void sortImplementation() override;
+
+        struct State
+        {
+            State()
+                : mAlphaBlend(false)
+                , mAlphaBlendOverride(false)
+                , mAlphaTest(false)
+                , mAlphaTestOverride(false)
+                , mMaterial(nullptr)
+                , mMaterialOverride(false)
+                , mImportantState(false)
+                {}
+
+            bool mAlphaBlend;
+            bool mAlphaBlendOverride;
+            bool mAlphaTest;
+            bool mAlphaTestOverride;
+            osg::Material* mMaterial;
+            bool mMaterialOverride;
+            bool mImportantState;
+            bool needTexture() const { return mAlphaBlend || mAlphaTest; }
+            bool needShadows() const;
+            // A state is interesting if there's anything about it that might affect whether we can optimise child state
+            bool interesting() const
+            {
+                return !needShadows() || needTexture() || mAlphaBlendOverride || mAlphaTestOverride || mMaterialOverride || mImportantState;
+            }
+        };
+
+        osgUtil::StateGraph* cullStateGraph(osgUtil::StateGraph* sg, osgUtil::StateGraph* root, std::unordered_set<osgUtil::StateGraph*>& uninteresting);
+
+        static void addPrototype(const std::string& name)
+        {
+            osg::ref_ptr<osgUtil::RenderBin> bin (new ShadowsBin);
+            osgUtil::RenderBin::addRenderBinPrototype(name, bin);
+        }
+    };
+
+    class ShadowsBinAdder
+    {
+        public:
+        ShadowsBinAdder(const std::string& name){ ShadowsBin::addPrototype(name); }
+    };
+
+}
+
+#endif

components/shader/shadermanager.cpp

@@ -384,14 +384,4 @@ namespace Shader
             program.second->releaseGLObjects(state);
     }
 
-    const osg::ref_ptr<osg::Uniform> ShaderManager::getShadowMapAlphaTestEnableUniform()
-    {
-        return mShadowMapAlphaTestEnableUniform;
-    }
-
-    const osg::ref_ptr<osg::Uniform> ShaderManager::getShadowMapAlphaTestDisableUniform()
-    {
-        return mShadowMapAlphaTestDisableUniform;
-    }
-
 }

components/shader/shadermanager.hpp

@@ -43,9 +43,6 @@ namespace Shader
 
         void releaseGLObjects(osg::State* state);
 
-        const osg::ref_ptr<osg::Uniform> getShadowMapAlphaTestEnableUniform();
-        const osg::ref_ptr<osg::Uniform> getShadowMapAlphaTestDisableUniform();
-
     private:
         std::string mPath;
 
@@ -63,9 +60,6 @@ namespace Shader
         ProgramMap mPrograms;
 
         std::mutex mMutex;
-
-        const osg::ref_ptr<osg::Uniform> mShadowMapAlphaTestEnableUniform = new osg::Uniform();
-        const osg::ref_ptr<osg::Uniform> mShadowMapAlphaTestDisableUniform = new osg::Uniform();
     };
 
     bool parseFors(std::string& source, const std::string& templateName);

components/shader/shadervisitor.cpp

@@ -1,7 +1,5 @@
 #include "shadervisitor.hpp"
 
-#include <osg/AlphaFunc>
-#include <osg/BlendFunc>
 #include <osg/Geometry>
 #include <osg/Material>
 #include <osg/Texture>
@@ -24,7 +22,6 @@ namespace Shader
         : mShaderRequired(false)
         , mColorMode(0)
         , mMaterialOverridden(false)
-        , mBlendFuncOverridden(false)
         , mNormalHeight(false)
         , mTexStageRequiringTangents(-1)
         , mNode(nullptr)
@@ -231,14 +228,11 @@ namespace Shader
                 if (!writableStateSet)
                     writableStateSet = getWritableStateSet(node);
                 // We probably shouldn't construct a new version of this each time as Uniforms use pointer comparison for early-out.
-                // Also it should probably belong to the shader manager
+                // Also it should probably belong to the shader manager or be applied by the shadows bin
                 writableStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", true));
             }
         }
 
-        bool alphaSettingsChanged = false;
-        bool alphaTestShadows = false;
-
         const osg::StateSet::AttributeList& attributes = stateset->getAttributeList();
         for (osg::StateSet::AttributeList::const_iterator it = attributes.begin(); it != attributes.end(); ++it)
         {
@@ -282,28 +276,8 @@ namespace Shader
                     mRequirements.back().mColorMode = colorMode;
                 }
             }
-            else if (it->first.first == osg::StateAttribute::BLENDFUNC)
-            {
-                if (!mRequirements.back().mBlendFuncOverridden || it->second.second & osg::StateAttribute::PROTECTED)
-                {
-                    if (it->second.second & osg::StateAttribute::OVERRIDE)
-                        mRequirements.back().mBlendFuncOverridden = true;
-
-                    const osg::BlendFunc* blend = static_cast<const osg::BlendFunc*>(it->second.first.get());
-                    if (blend->getSource() == osg::BlendFunc::SRC_ALPHA || blend->getSource() == osg::BlendFunc::SRC_COLOR)
-                        alphaTestShadows = true;
-                    alphaSettingsChanged = true;
-                }
-            }
             // Eventually, move alpha testing to discard in shader adn remove deprecated state here
         }
-        // we don't need to check for glEnable/glDisable of blending as we always set it at the same time
-        if (alphaSettingsChanged)
-        {
-            if (!writableStateSet)
-                writableStateSet = getWritableStateSet(node);
-            writableStateSet->addUniform(alphaTestShadows ? mShaderManager.getShadowMapAlphaTestEnableUniform() : mShaderManager.getShadowMapAlphaTestDisableUniform());
-        }
     }
 
     void ShaderVisitor::pushRequirements(osg::Node& node)

components/shader/shadervisitor.hpp

@@ -79,7 +79,6 @@ namespace Shader
             int mColorMode;
             
             bool mMaterialOverridden;
-            bool mBlendFuncOverridden;
 
             bool mNormalHeight; // true if normal map has height info in alpha channel