Merge branch 'shadow-transparency-rework' into 'master'

Only alpha-test shadows when necessary

See merge request OpenMW/openmw!170

apps/openmw/mwrender/animation.cpp

@@ -16,6 +16,7 @@
 
 #include <components/resource/scenemanager.hpp>
 #include <components/resource/keyframemanager.hpp>
+#include <components/resource/resourcesystem.hpp>
 
 #include <components/misc/constants.hpp>
 #include <components/misc/resourcehelpers.hpp>
@@ -36,6 +37,8 @@
 
 #include <components/settings/settings.hpp>
 
+#include <components/shader/shadermanager.hpp>
+
 #include "../mwbase/environment.hpp"
 #include "../mwbase/world.hpp"
 #include "../mwworld/esmstore.hpp"
@@ -502,8 +505,9 @@ namespace MWRender
     class TransparencyUpdater : public SceneUtil::StateSetUpdater
     {
     public:
-        TransparencyUpdater(const float alpha)
+        TransparencyUpdater(const float alpha, osg::ref_ptr<osg::Uniform> shadowUniform)
             : mAlpha(alpha)
+            , mShadowUniform(shadowUniform)
         {
         }
 
@@ -517,6 +521,9 @@ namespace MWRender
         {
             osg::BlendFunc* blendfunc (new osg::BlendFunc);
             stateset->setAttributeAndModes(blendfunc, osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE);
+            // TODO: don't do this anymore once custom shadow renderbin is handling it
+            if (mShadowUniform)
+                stateset->addUniform(mShadowUniform);
 
             // FIXME: overriding diffuse/ambient/emissive colors
             osg::Material* material = new osg::Material;
@@ -535,6 +542,7 @@ namespace MWRender
 
     private:
         float mAlpha;
+        osg::ref_ptr<osg::Uniform> mShadowUniform;
     };
 
     struct Animation::AnimSource
@@ -1744,7 +1752,7 @@ namespace MWRender
         {
             if (mTransparencyUpdater == nullptr)
             {
-                mTransparencyUpdater = new TransparencyUpdater(alpha);
+                mTransparencyUpdater = new TransparencyUpdater(alpha, mResourceSystem->getSceneManager()->getShaderManager().getShadowMapAlphaTestEnableUniform());
                 mObjectRoot->addUpdateCallback(mTransparencyUpdater);
             }
             else

components/sceneutil/mwshadowtechnique.cpp

@@ -872,6 +872,15 @@ void SceneUtil::MWShadowTechnique::setupCastingShader(Shader::ShaderManager & sh
 
     _castingProgram->addShader(shaderManager.getShader("shadowcasting_vertex.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::VERTEX));
     _castingProgram->addShader(shaderManager.getShader("shadowcasting_fragment.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::FRAGMENT));
+
+    _shadowMapAlphaTestDisableUniform = shaderManager.getShadowMapAlphaTestDisableUniform();
+    _shadowMapAlphaTestDisableUniform->setName("alphaTestShadows");
+    _shadowMapAlphaTestDisableUniform->setType(osg::Uniform::BOOL);
+    _shadowMapAlphaTestDisableUniform->set(false);
+
+    shaderManager.getShadowMapAlphaTestEnableUniform()->setName("alphaTestShadows");
+    shaderManager.getShadowMapAlphaTestEnableUniform()->setType(osg::Uniform::BOOL);
+    shaderManager.getShadowMapAlphaTestEnableUniform()->set(true);
 }
 
 MWShadowTechnique::ViewDependentData* MWShadowTechnique::createViewDependentData(osgUtil::CullVisitor* /*cv*/)
@@ -1570,6 +1579,7 @@ void MWShadowTechnique::createShaders()
     // The casting program uses a sampler, so to avoid undefined behaviour, we must bind a dummy texture in case no other is supplied
     _shadowCastingStateSet->setTextureAttributeAndModes(0, _fallbackBaseTexture.get(), osg::StateAttribute::ON);
     _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
+    _shadowCastingStateSet->addUniform(_shadowMapAlphaTestDisableUniform);
 
     _shadowCastingStateSet->setMode(GL_DEPTH_CLAMP, osg::StateAttribute::ON);
 

components/sceneutil/mwshadowtechnique.hpp

@@ -286,6 +286,7 @@ namespace SceneUtil {
 
         osg::ref_ptr<DebugHUD>                  _debugHud;
         osg::ref_ptr<osg::Program>              _castingProgram;
+        osg::ref_ptr<osg::Uniform>              _shadowMapAlphaTestDisableUniform;
     };
 
 }

components/shader/shadermanager.cpp

@@ -372,4 +372,14 @@ namespace Shader
             program.second->releaseGLObjects(state);
     }
 
+    const osg::ref_ptr<osg::Uniform> ShaderManager::getShadowMapAlphaTestEnableUniform()
+    {
+        return mShadowMapAlphaTestEnableUniform;
+    }
+
+    const osg::ref_ptr<osg::Uniform> ShaderManager::getShadowMapAlphaTestDisableUniform()
+    {
+        return mShadowMapAlphaTestDisableUniform;
+    }
+
 }

components/shader/shadermanager.hpp

@@ -44,6 +44,9 @@ namespace Shader
 
         void releaseGLObjects(osg::State* state);
 
+        const osg::ref_ptr<osg::Uniform> getShadowMapAlphaTestEnableUniform();
+        const osg::ref_ptr<osg::Uniform> getShadowMapAlphaTestDisableUniform();
+
     private:
         std::string mPath;
 
@@ -61,6 +64,9 @@ namespace Shader
         ProgramMap mPrograms;
 
         OpenThreads::Mutex mMutex;
+
+        const osg::ref_ptr<osg::Uniform> mShadowMapAlphaTestEnableUniform = new osg::Uniform();
+        const osg::ref_ptr<osg::Uniform> mShadowMapAlphaTestDisableUniform = new osg::Uniform();
     };
 
     bool parseFors(std::string& source, const std::string& templateName);

components/shader/shadervisitor.cpp

@@ -1,8 +1,10 @@
 #include "shadervisitor.hpp"
 
-#include <osg/Texture>
-#include <osg/Material>
+#include <osg/AlphaFunc>
+#include <osg/BlendFunc>
 #include <osg/Geometry>
+#include <osg/Material>
+#include <osg/Texture>
 
 #include <osgUtil/TangentSpaceGenerator>
 
@@ -23,6 +25,7 @@ namespace Shader
         : mShaderRequired(false)
         , mColorMode(0)
         , mMaterialOverridden(false)
+        , mBlendFuncOverridden(false)
         , mNormalHeight(false)
         , mTexStageRequiringTangents(-1)
         , mNode(nullptr)
@@ -229,15 +232,21 @@ namespace Shader
             {
                 if (!writableStateSet)
                     writableStateSet = getWritableStateSet(node);
+                // We probably shouldn't construct a new version of this each time as Uniforms use pointer comparison for early-out.
+                // Also it should probably belong to the shader manager
                 writableStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", true));
             }
         }
 
+        bool alphaSettingsChanged = false;
+        bool alphaTestShadows = false;
+
         const osg::StateSet::AttributeList& attributes = stateset->getAttributeList();
         for (osg::StateSet::AttributeList::const_iterator it = attributes.begin(); it != attributes.end(); ++it)
         {
             if (it->first.first == osg::StateAttribute::MATERIAL)
             {
+                // This should probably be moved out of ShaderRequirements and be applied directly now it's a uniform instead of a define
                 if (!mRequirements.back().mMaterialOverridden || it->second.second & osg::StateAttribute::PROTECTED)
                 {
                     if (it->second.second & osg::StateAttribute::OVERRIDE)
@@ -269,6 +278,27 @@ namespace Shader
                     mRequirements.back().mColorMode = colorMode;
                 }
             }
+            else if (it->first.first == osg::StateAttribute::BLENDFUNC)
+            {
+                if (!mRequirements.back().mBlendFuncOverridden || it->second.second & osg::StateAttribute::PROTECTED)
+                {
+                    if (it->second.second & osg::StateAttribute::OVERRIDE)
+                        mRequirements.back().mBlendFuncOverridden = true;
+
+                    const osg::BlendFunc* blend = static_cast<const osg::BlendFunc*>(it->second.first.get());
+                    if (blend->getSource() == osg::BlendFunc::SRC_ALPHA || blend->getSource() == osg::BlendFunc::SRC_COLOR)
+                        alphaTestShadows = true;
+                    alphaSettingsChanged = true;
+                }
+            }
+            // Eventually, move alpha testing to discard in shader adn remove deprecated state here
+        }
+        // we don't need to check for glEnable/glDisable of blending as we always set it at the same time
+        if (alphaSettingsChanged)
+        {
+            if (!writableStateSet)
+                writableStateSet = getWritableStateSet(node);
+            writableStateSet->addUniform(alphaTestShadows ? mShaderManager.getShadowMapAlphaTestEnableUniform() : mShaderManager.getShadowMapAlphaTestDisableUniform());
         }
     }
 

components/shader/shadervisitor.hpp

@@ -75,6 +75,8 @@ namespace Shader
             int mColorMode;
             
             bool mMaterialOverridden;
+            bool mBlendFuncOverridden;
+
             bool mNormalHeight; // true if normal map has height info in alpha channel
 
             // -1 == no tangents required

files/shaders/shadowcasting_fragment.glsl

@@ -6,6 +6,7 @@ varying vec2 diffuseMapUV;
 varying float alphaPassthrough;
 
 uniform bool useDiffuseMapForShadowAlpha;
+uniform bool alphaTestShadows;
 
 void main()
 {
@@ -15,7 +16,7 @@ void main()
     else
         gl_FragData[0].a = alphaPassthrough;
 
-    // Prevent translucent things casting shadow (including the player using an invisibility effect)
-    if (gl_FragData[0].a <= 0.5)
+    // Prevent translucent things casting shadow (including the player using an invisibility effect). For now, rely on the deprecated FF test for non-blended stuff.
+    if (alphaTestShadows && gl_FragData[0].a <= 0.5)
         discard;
 }

files/shaders/shadowcasting_vertex.glsl

@@ -5,7 +5,8 @@ varying vec2 diffuseMapUV;
 varying float alphaPassthrough;
 
 uniform int colorMode;
-uniform bool useDiffuseMapForShadowAlpha;
+uniform bool useDiffuseMapForShadowAlpha = true;
+uniform bool alphaTestShadows = true;
 
 void main(void)
 {
@@ -18,7 +19,6 @@ void main(void)
         diffuseMapUV = (gl_TextureMatrix[0] * gl_MultiTexCoord0).xy;
     else
         diffuseMapUV = vec2(0.0); // Avoid undefined behaviour if running on hardware predating the concept of dynamically uniform expressions
-
     if (colorMode == 2)
         alphaPassthrough = gl_Color.a;
     else