Handle all shadow alpha uniforms in shadowsbin

apps/openmw/mwrender/animation.cpp

@@ -16,7 +16,6 @@
 
 #include <components/resource/scenemanager.hpp>
 #include <components/resource/keyframemanager.hpp>
-#include <components/resource/resourcesystem.hpp>
 
 #include <components/misc/constants.hpp>
 #include <components/misc/resourcehelpers.hpp>
@@ -37,8 +36,6 @@
 
 #include <components/settings/settings.hpp>
 
-#include <components/shader/shadermanager.hpp>
-
 #include "../mwbase/environment.hpp"
 #include "../mwbase/world.hpp"
 #include "../mwworld/esmstore.hpp"
@@ -493,9 +490,8 @@ namespace MWRender
     class TransparencyUpdater : public SceneUtil::StateSetUpdater
     {
     public:
-        TransparencyUpdater(const float alpha, osg::ref_ptr<osg::Uniform> shadowUniform)
+        TransparencyUpdater(const float alpha)
             : mAlpha(alpha)
-            , mShadowUniform(shadowUniform)
         {
         }
 
@@ -509,9 +505,6 @@ namespace MWRender
         {
             osg::BlendFunc* blendfunc (new osg::BlendFunc);
             stateset->setAttributeAndModes(blendfunc, osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE);
-            // TODO: don't do this anymore once custom shadow renderbin is handling it
-            if (mShadowUniform)
-                stateset->addUniform(mShadowUniform);
 
             stateset->setRenderingHint(osg::StateSet::TRANSPARENT_BIN);
             stateset->setRenderBinMode(osg::StateSet::OVERRIDE_RENDERBIN_DETAILS);
@@ -533,7 +526,6 @@ namespace MWRender
 
     private:
         float mAlpha;
-        osg::ref_ptr<osg::Uniform> mShadowUniform;
     };
 
     struct Animation::AnimSource
@@ -1773,7 +1765,7 @@ namespace MWRender
         {
             if (mTransparencyUpdater == nullptr)
             {
-                mTransparencyUpdater = new TransparencyUpdater(alpha, mResourceSystem->getSceneManager()->getShaderManager().getShadowMapAlphaTestEnableUniform());
+                mTransparencyUpdater = new TransparencyUpdater(alpha);
                 mObjectRoot->addCullCallback(mTransparencyUpdater);
             }
             else

components/sceneutil/mwshadowtechnique.cpp

@@ -885,15 +885,6 @@ void SceneUtil::MWShadowTechnique::setupCastingShader(Shader::ShaderManager & sh
 
     _castingProgram->addShader(shaderManager.getShader("shadowcasting_vertex.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::VERTEX));
     _castingProgram->addShader(shaderManager.getShader("shadowcasting_fragment.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::FRAGMENT));
-
-    _shadowMapAlphaTestDisableUniform = shaderManager.getShadowMapAlphaTestDisableUniform();
-    _shadowMapAlphaTestDisableUniform->setName("alphaTestShadows");
-    _shadowMapAlphaTestDisableUniform->setType(osg::Uniform::BOOL);
-    _shadowMapAlphaTestDisableUniform->set(false);
-
-    shaderManager.getShadowMapAlphaTestEnableUniform()->setName("alphaTestShadows");
-    shaderManager.getShadowMapAlphaTestEnableUniform()->setType(osg::Uniform::BOOL);
-    shaderManager.getShadowMapAlphaTestEnableUniform()->set(true);
 }
 
 MWShadowTechnique::ViewDependentData* MWShadowTechnique::createViewDependentData(osgUtil::CullVisitor* /*cv*/)
@@ -1595,7 +1586,7 @@ void MWShadowTechnique::createShaders()
     // The casting program uses a sampler, so to avoid undefined behaviour, we must bind a dummy texture in case no other is supplied
     _shadowCastingStateSet->setTextureAttributeAndModes(0, _fallbackBaseTexture.get(), osg::StateAttribute::ON);
     _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", true));
-    _shadowCastingStateSet->addUniform(_shadowMapAlphaTestDisableUniform);
+    _shadowCastingStateSet->addUniform(new osg::Uniform("alphaTestShadows", false));
     osg::ref_ptr<osg::Depth> depth = new osg::Depth;
     depth->setWriteMask(true);
     _shadowCastingStateSet->setAttribute(depth, osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE);

components/sceneutil/mwshadowtechnique.hpp

@@ -288,7 +288,6 @@ namespace SceneUtil {
 
         osg::ref_ptr<DebugHUD>                  _debugHud;
         osg::ref_ptr<osg::Program>              _castingProgram;
-        osg::ref_ptr<osg::Uniform>              _shadowMapAlphaTestDisableUniform;
     };
 
 }

components/sceneutil/shadowsbin.cpp

@@ -42,11 +42,16 @@ namespace SceneUtil
 
 ShadowsBin::ShadowsBin()
 {
-    mStateSet = new osg::StateSet;
-    mStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
+    mNoTestStateSet = new osg::StateSet;
+    mNoTestStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
+    mNoTestStateSet->addUniform(new osg::Uniform("alphaTestShadows", false));
+
+    mShaderAlphaTestStateSet = new osg::StateSet;
+    mShaderAlphaTestStateSet->addUniform(new osg::Uniform("alphaTestShadows", true));
+    mShaderAlphaTestStateSet->setMode(GL_BLEND, osg::StateAttribute::OFF | osg::StateAttribute::OVERRIDE);
 }
 
-bool ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::unordered_set<StateGraph*>& uninterestingCache)
+StateGraph* ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::unordered_set<StateGraph*>& uninterestingCache)
 {
     std::vector<StateGraph*> return_path;
     State state;
@@ -88,7 +93,7 @@ bool ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::unordered
     }
 
     if (!state.needShadows())
-        return true;
+        return nullptr;
 
     if (!state.needTexture() && !state.mImportantState)
     {
@@ -97,9 +102,12 @@ bool ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::unordered
             leaf->_parent = root;
             root->_leaves.push_back(leaf);
         }
-        return true;
+        return nullptr;
     }
-    return false;
+
+    if (state.mAlphaBlend)
+        return sg->find_or_insert(mShaderAlphaTestStateSet);
+    return sg;
 }
 
 bool ShadowsBin::State::needShadows() const
@@ -128,9 +136,9 @@ void ShadowsBin::sortImplementation()
         if (!root->_parent)
             return;
     }
-    root = root->find_or_insert(mStateSet.get());
+    StateGraph* noTestRoot = root->find_or_insert(mNoTestStateSet.get());
     // root is now a stategraph with useDiffuseMapForShadowAlpha disabled but minimal other state
-    root->_leaves.reserve(_stateGraphList.size());
+    noTestRoot->_leaves.reserve(_stateGraphList.size());
     StateGraphList newList;
     std::unordered_set<StateGraph*> uninterestingCache;
     for (StateGraph* graph : _stateGraphList)
@@ -138,11 +146,12 @@ void ShadowsBin::sortImplementation()
         // Render leaves which shouldn't use the diffuse map for shadow alpha but do cast shadows become children of root, so graph is now empty. Don't add to newList.
         // Graphs containing just render leaves which don't cast shadows are discarded. Don't add to newList.
         // Graphs containing other leaves need to be in newList.
-        if (!cullStateGraph(graph, root, uninterestingCache))
-            newList.push_back(graph);
+        StateGraph* graphToAdd = cullStateGraph(graph, noTestRoot, uninterestingCache);
+        if (graphToAdd)
+            newList.push_back(graphToAdd);
     }
-    if (!root->_leaves.empty())
-        newList.push_back(root);
+    if (!noTestRoot->_leaves.empty())
+        newList.push_back(noTestRoot);
     _stateGraphList = newList;
 }
 

components/sceneutil/shadowsbin.hpp

@@ -15,11 +15,12 @@ namespace SceneUtil
     class ShadowsBin : public osgUtil::RenderBin
     {
     private:
-        osg::ref_ptr<osg::StateSet> mStateSet;
+        osg::ref_ptr<osg::StateSet> mNoTestStateSet;
+        osg::ref_ptr<osg::StateSet> mShaderAlphaTestStateSet;
     public:
         META_Object(SceneUtil, ShadowsBin)
         ShadowsBin();
-        ShadowsBin(const ShadowsBin& rhs, const osg::CopyOp& copyop) : osgUtil::RenderBin(rhs, copyop), mStateSet(rhs.mStateSet) {}
+        ShadowsBin(const ShadowsBin& rhs, const osg::CopyOp& copyop) : osgUtil::RenderBin(rhs, copyop), mNoTestStateSet(rhs.mNoTestStateSet), mShaderAlphaTestStateSet(rhs.mShaderAlphaTestStateSet) {}
 
         void sortImplementation() override;
 
@@ -39,7 +40,7 @@ namespace SceneUtil
             bool interesting() const { return !needShadows() || needTexture() || mAlphaBlendOverride || mAlphaTestOverride || mMaterialOverride || mImportantState; }
         };
 
-        bool cullStateGraph(osgUtil::StateGraph* sg, osgUtil::StateGraph* root, std::unordered_set<osgUtil::StateGraph*>& uninteresting);
+        osgUtil::StateGraph* cullStateGraph(osgUtil::StateGraph* sg, osgUtil::StateGraph* root, std::unordered_set<osgUtil::StateGraph*>& uninteresting);
 
         static void addPrototype(const std::string& name)
         {

components/shader/shadermanager.cpp

@@ -384,14 +384,4 @@ namespace Shader
             program.second->releaseGLObjects(state);
     }
 
-    const osg::ref_ptr<osg::Uniform> ShaderManager::getShadowMapAlphaTestEnableUniform()
-    {
-        return mShadowMapAlphaTestEnableUniform;
-    }
-
-    const osg::ref_ptr<osg::Uniform> ShaderManager::getShadowMapAlphaTestDisableUniform()
-    {
-        return mShadowMapAlphaTestDisableUniform;
-    }
-
 }

components/shader/shadermanager.hpp

@@ -43,9 +43,6 @@ namespace Shader
 
         void releaseGLObjects(osg::State* state);
 
-        const osg::ref_ptr<osg::Uniform> getShadowMapAlphaTestEnableUniform();
-        const osg::ref_ptr<osg::Uniform> getShadowMapAlphaTestDisableUniform();
-
     private:
         std::string mPath;
 
@@ -63,9 +60,6 @@ namespace Shader
         ProgramMap mPrograms;
 
         std::mutex mMutex;
-
-        const osg::ref_ptr<osg::Uniform> mShadowMapAlphaTestEnableUniform = new osg::Uniform();
-        const osg::ref_ptr<osg::Uniform> mShadowMapAlphaTestDisableUniform = new osg::Uniform();
     };
 
     bool parseFors(std::string& source, const std::string& templateName);

components/shader/shadervisitor.cpp

@@ -1,7 +1,5 @@
 #include "shadervisitor.hpp"
 
-#include <osg/AlphaFunc>
-#include <osg/BlendFunc>
 #include <osg/Geometry>
 #include <osg/Material>
 #include <osg/Texture>
@@ -25,7 +23,6 @@ namespace Shader
         : mShaderRequired(false)
         , mColorMode(0)
         , mMaterialOverridden(false)
-        , mBlendFuncOverridden(false)
         , mNormalHeight(false)
         , mTexStageRequiringTangents(-1)
         , mNode(nullptr)
@@ -233,14 +230,11 @@ namespace Shader
                 if (!writableStateSet)
                     writableStateSet = getWritableStateSet(node);
                 // We probably shouldn't construct a new version of this each time as Uniforms use pointer comparison for early-out.
-                // Also it should probably belong to the shader manager
+                // Also it should probably belong to the shader manager or be applied by the shadows bin
                 writableStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", true));
             }
         }
 
-        bool alphaSettingsChanged = false;
-        bool alphaTestShadows = false;
-
         const osg::StateSet::AttributeList& attributes = stateset->getAttributeList();
         for (osg::StateSet::AttributeList::const_iterator it = attributes.begin(); it != attributes.end(); ++it)
         {
@@ -284,28 +278,8 @@ namespace Shader
                     mRequirements.back().mColorMode = colorMode;
                 }
             }
-            else if (it->first.first == osg::StateAttribute::BLENDFUNC)
-            {
-                if (!mRequirements.back().mBlendFuncOverridden || it->second.second & osg::StateAttribute::PROTECTED)
-                {
-                    if (it->second.second & osg::StateAttribute::OVERRIDE)
-                        mRequirements.back().mBlendFuncOverridden = true;
-
-                    const osg::BlendFunc* blend = static_cast<const osg::BlendFunc*>(it->second.first.get());
-                    if (blend->getSource() == osg::BlendFunc::SRC_ALPHA || blend->getSource() == osg::BlendFunc::SRC_COLOR)
-                        alphaTestShadows = true;
-                    alphaSettingsChanged = true;
-                }
-            }
             // Eventually, move alpha testing to discard in shader adn remove deprecated state here
         }
-        // we don't need to check for glEnable/glDisable of blending as we always set it at the same time
-        if (alphaSettingsChanged)
-        {
-            if (!writableStateSet)
-                writableStateSet = getWritableStateSet(node);
-            writableStateSet->addUniform(alphaTestShadows ? mShaderManager.getShadowMapAlphaTestEnableUniform() : mShaderManager.getShadowMapAlphaTestDisableUniform());
-        }
     }
 
     void ShaderVisitor::pushRequirements(osg::Node& node)

components/shader/shadervisitor.hpp

@@ -75,7 +75,6 @@ namespace Shader
             int mColorMode;
             
             bool mMaterialOverridden;
-            bool mBlendFuncOverridden;
 
             bool mNormalHeight; // true if normal map has height info in alpha channel