Only alpha-test shadows when necessary

Previously we always discarded shadow map fragments if the alpha channel of the output would have been low, but there were some (modded) assets that have non-one alpha but have testing or blending disabled so end up opaque anyway. This lets the shadows of those objects match.

components/sceneutil/mwshadowtechnique.cpp

@@ -872,6 +872,15 @@ void SceneUtil::MWShadowTechnique::setupCastingShader(Shader::ShaderManager & sh
 
     _castingProgram->addShader(shaderManager.getShader("shadowcasting_vertex.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::VERTEX));
     _castingProgram->addShader(shaderManager.getShader("shadowcasting_fragment.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::FRAGMENT));
+
+    _shadowMapAlphaTestDisableUniform = shaderManager.getShadowMapAlphaTestDisableUniform();
+    _shadowMapAlphaTestDisableUniform->setName("alphaTestShadows");
+    _shadowMapAlphaTestDisableUniform->setType(osg::Uniform::BOOL);
+    _shadowMapAlphaTestDisableUniform->set(false);
+
+    shaderManager.getShadowMapAlphaTestEnableUniform()->setName("alphaTestShadows");
+    shaderManager.getShadowMapAlphaTestEnableUniform()->setType(osg::Uniform::BOOL);
+    shaderManager.getShadowMapAlphaTestEnableUniform()->set(true);
 }
 
 MWShadowTechnique::ViewDependentData* MWShadowTechnique::createViewDependentData(osgUtil::CullVisitor* /*cv*/)
@@ -1570,6 +1579,7 @@ void MWShadowTechnique::createShaders()
     // The casting program uses a sampler, so to avoid undefined behaviour, we must bind a dummy texture in case no other is supplied
     _shadowCastingStateSet->setTextureAttributeAndModes(0, _fallbackBaseTexture.get(), osg::StateAttribute::ON);
     _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
+    _shadowCastingStateSet->addUniform(_shadowMapAlphaTestDisableUniform);
 
     _shadowCastingStateSet->setMode(GL_DEPTH_CLAMP, osg::StateAttribute::ON);
 

components/sceneutil/mwshadowtechnique.hpp

@@ -286,6 +286,7 @@ namespace SceneUtil {
 
         osg::ref_ptr<DebugHUD>                  _debugHud;
         osg::ref_ptr<osg::Program>              _castingProgram;
+        osg::ref_ptr<osg::Uniform>              _shadowMapAlphaTestDisableUniform;
     };
 
 }

components/shader/shadermanager.cpp

@@ -370,4 +370,14 @@ namespace Shader
             program.second->releaseGLObjects(state);
     }
 
+    const osg::ref_ptr<osg::Uniform> ShaderManager::getShadowMapAlphaTestEnableUniform()
+    {
+        return mShadowMapAlphaTestEnableUniform;
+    }
+
+    const osg::ref_ptr<osg::Uniform> ShaderManager::getShadowMapAlphaTestDisableUniform()
+    {
+        return mShadowMapAlphaTestDisableUniform;
+    }
+
 }

components/shader/shadermanager.hpp

@@ -44,6 +44,9 @@ namespace Shader
 
         void releaseGLObjects(osg::State* state);
 
+        const osg::ref_ptr<osg::Uniform> getShadowMapAlphaTestEnableUniform();
+        const osg::ref_ptr<osg::Uniform> getShadowMapAlphaTestDisableUniform();
+
     private:
         std::string mPath;
 
@@ -61,6 +64,9 @@ namespace Shader
         ProgramMap mPrograms;
 
         OpenThreads::Mutex mMutex;
+
+        const osg::ref_ptr<osg::Uniform> mShadowMapAlphaTestEnableUniform = new osg::Uniform();
+        const osg::ref_ptr<osg::Uniform> mShadowMapAlphaTestDisableUniform = new osg::Uniform();
     };
 
 }

components/shader/shadervisitor.cpp

@@ -1,8 +1,10 @@
 #include "shadervisitor.hpp"
 
-#include <osg/Texture>
-#include <osg/Material>
+#include <osg/AlphaFunc>
+#include <osg/BlendFunc>
 #include <osg/Geometry>
+#include <osg/Material>
+#include <osg/Texture>
 
 #include <osgUtil/TangentSpaceGenerator>
 
@@ -23,6 +25,8 @@ namespace Shader
         : mShaderRequired(false)
         , mColorMode(0)
         , mMaterialOverridden(false)
+        , mAlphaFuncOverridden(false)
+        , mBlendFuncOverridden(false)
         , mNormalHeight(false)
         , mTexStageRequiringTangents(-1)
         , mNode(nullptr)
@@ -229,15 +233,21 @@ namespace Shader
             {
                 if (!writableStateSet)
                     writableStateSet = getWritableStateSet(node);
+                // We probably shouldn't construct a new version of this each time as StateSets only use pointer comparison by default.
+                // Also it should probably belong to the shader manager
                 writableStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", true));
             }
         }
 
+        bool alphaSettingsChanged = false;
+        bool alphaTestShadows = false;
+
         const osg::StateSet::AttributeList& attributes = stateset->getAttributeList();
         for (osg::StateSet::AttributeList::const_iterator it = attributes.begin(); it != attributes.end(); ++it)
         {
             if (it->first.first == osg::StateAttribute::MATERIAL)
             {
+                // This should probably be moved out of ShaderRequirements and be applied directly now it's a uniform instead of a define
                 if (!mRequirements.back().mMaterialOverridden || it->second.second & osg::StateAttribute::PROTECTED)
                 {
                     if (it->second.second & osg::StateAttribute::OVERRIDE)
@@ -269,6 +279,39 @@ namespace Shader
                     mRequirements.back().mColorMode = colorMode;
                 }
             }
+            else if (it->first.first == osg::StateAttribute::ALPHAFUNC)
+            {
+                if (!mRequirements.back().mAlphaFuncOverridden || it->second.second & osg::StateAttribute::PROTECTED)
+                {
+                    if (it->second.second & osg::StateAttribute::OVERRIDE)
+                        mRequirements.back().mAlphaFuncOverridden = true;
+
+                    const osg::AlphaFunc* test = static_cast<const osg::AlphaFunc*>(it->second.first.get());
+                    if (test->getFunction() == osg::AlphaFunc::GREATER || test->getFunction() == osg::AlphaFunc::GEQUAL)
+                        alphaTestShadows = true;
+                    alphaSettingsChanged = true;
+                }
+            }
+            else if (it->first.first == osg::StateAttribute::BLENDFUNC)
+            {
+                if (!mRequirements.back().mBlendFuncOverridden || it->second.second & osg::StateAttribute::PROTECTED)
+                {
+                    if (it->second.second & osg::StateAttribute::OVERRIDE)
+                        mRequirements.back().mBlendFuncOverridden = true;
+
+                    const osg::BlendFunc* blend = static_cast<const osg::BlendFunc*>(it->second.first.get());
+                    if (blend->getSource() == osg::BlendFunc::SRC_ALPHA || blend->getSource() == osg::BlendFunc::SRC_COLOR)
+                        alphaTestShadows = true;
+                    alphaSettingsChanged = true;
+                }
+            }
+        }
+        // we don't need to check for glEnable/glDisable of blending and testing as we always set it at the same time
+        if (alphaSettingsChanged)
+        {
+            if (!writableStateSet)
+                writableStateSet = getWritableStateSet(node);
+            writableStateSet->addUniform(alphaTestShadows ? mShaderManager.getShadowMapAlphaTestEnableUniform() : mShaderManager.getShadowMapAlphaTestDisableUniform());
         }
     }
 

components/shader/shadervisitor.hpp

@@ -75,6 +75,9 @@ namespace Shader
             int mColorMode;
             
             bool mMaterialOverridden;
+            bool mAlphaFuncOverridden;
+            bool mBlendFuncOverridden;
+
             bool mNormalHeight; // true if normal map has height info in alpha channel
 
             // -1 == no tangents required

files/shaders/shadowcasting_fragment.glsl

@@ -6,10 +6,17 @@ varying vec2 diffuseMapUV;
 varying float alphaPassthrough;
 
 uniform bool useDiffuseMapForShadowAlpha;
+uniform bool alphaTestShadows;
 
 void main()
 {
     gl_FragData[0].rgb = vec3(1.0);
+    if (!alphaTestShadows)
+    {
+        gl_FragData[0].a = 1.0;
+        return;
+    }
+
     if (useDiffuseMapForShadowAlpha)
         gl_FragData[0].a = texture2D(diffuseMap, diffuseMapUV).a * alphaPassthrough;
     else

files/shaders/shadowcasting_vertex.glsl

@@ -6,6 +6,7 @@ varying float alphaPassthrough;
 
 uniform int colorMode;
 uniform bool useDiffuseMapForShadowAlpha;
+uniform bool alphaTestShadows;
 
 void main(void)
 {
@@ -14,12 +15,13 @@ void main(void)
     vec4 viewPos = (gl_ModelViewMatrix * gl_Vertex);
     gl_ClipVertex = viewPos;
 
-    if (useDiffuseMapForShadowAlpha)
+    if (alphaTestShadows && useDiffuseMapForShadowAlpha)
         diffuseMapUV = (gl_TextureMatrix[0] * gl_MultiTexCoord0).xy;
     else
         diffuseMapUV = vec2(0.0); // Avoid undefined behaviour if running on hardware predating the concept of dynamically uniform expressions
-
-    if (colorMode == 2)
+    if (!alphaTestShadows)
+        alphaPassthrough = 1.0;
+    else if (colorMode == 2)
         alphaPassthrough = gl_Color.a;
     else
         // This is uniform, so if it's too low, we might be able to put the position/clip vertex outside the view frustum and skip the fragment shader and rasteriser