From 53b9b411591b2bc22e78ba2cc15fa422449d89a1 Mon Sep 17 00:00:00 2001
From: AnyOldName3 <krizdjali+github@gmail.com>
Date: Tue, 21 Apr 2020 18:18:55 +0100
Subject: [PATCH] Rely on existing alpha test for non-blended shadow casting

---
 components/shader/shadervisitor.cpp       | 17 ++---------------
 components/shader/shadervisitor.hpp       |  1 -
 files/shaders/shadowcasting_fragment.glsl | 10 ++--------
 files/shaders/shadowcasting_vertex.glsl   | 10 ++++------
 4 files changed, 8 insertions(+), 30 deletions(-)

diff --git a/components/shader/shadervisitor.cpp b/components/shader/shadervisitor.cpp
index b2a6d6f63..639a7ecca 100644
--- a/components/shader/shadervisitor.cpp
+++ b/components/shader/shadervisitor.cpp
@@ -25,7 +25,6 @@ namespace Shader
         : mShaderRequired(false)
         , mColorMode(0)
         , mMaterialOverridden(false)
-        , mAlphaFuncOverridden(false)
         , mBlendFuncOverridden(false)
         , mNormalHeight(false)
         , mTexStageRequiringTangents(-1)
@@ -279,19 +278,6 @@ namespace Shader
                     mRequirements.back().mColorMode = colorMode;
                 }
             }
-            else if (it->first.first == osg::StateAttribute::ALPHAFUNC)
-            {
-                if (!mRequirements.back().mAlphaFuncOverridden || it->second.second & osg::StateAttribute::PROTECTED)
-                {
-                    if (it->second.second & osg::StateAttribute::OVERRIDE)
-                        mRequirements.back().mAlphaFuncOverridden = true;
-
-                    const osg::AlphaFunc* test = static_cast<const osg::AlphaFunc*>(it->second.first.get());
-                    if (test->getFunction() == osg::AlphaFunc::GREATER || test->getFunction() == osg::AlphaFunc::GEQUAL)
-                        alphaTestShadows = true;
-                    alphaSettingsChanged = true;
-                }
-            }
             else if (it->first.first == osg::StateAttribute::BLENDFUNC)
             {
                 if (!mRequirements.back().mBlendFuncOverridden || it->second.second & osg::StateAttribute::PROTECTED)
@@ -305,8 +291,9 @@ namespace Shader
                     alphaSettingsChanged = true;
                 }
             }
+            // Eventually, move alpha testing to discard in shader adn remove deprecated state here
         }
-        // we don't need to check for glEnable/glDisable of blending and testing as we always set it at the same time
+        // we don't need to check for glEnable/glDisable of blending as we always set it at the same time
         if (alphaSettingsChanged)
         {
             if (!writableStateSet)
diff --git a/components/shader/shadervisitor.hpp b/components/shader/shadervisitor.hpp
index 311f6213f..8e35f1d9c 100644
--- a/components/shader/shadervisitor.hpp
+++ b/components/shader/shadervisitor.hpp
@@ -75,7 +75,6 @@ namespace Shader
             int mColorMode;
             
             bool mMaterialOverridden;
-            bool mAlphaFuncOverridden;
             bool mBlendFuncOverridden;
 
             bool mNormalHeight; // true if normal map has height info in alpha channel
diff --git a/files/shaders/shadowcasting_fragment.glsl b/files/shaders/shadowcasting_fragment.glsl
index 47323cc6a..a5410d008 100644
--- a/files/shaders/shadowcasting_fragment.glsl
+++ b/files/shaders/shadowcasting_fragment.glsl
@@ -11,18 +11,12 @@ uniform bool alphaTestShadows;
 void main()
 {
     gl_FragData[0].rgb = vec3(1.0);
-    if (!alphaTestShadows)
-    {
-        gl_FragData[0].a = 1.0;
-        return;
-    }
-
     if (useDiffuseMapForShadowAlpha)
         gl_FragData[0].a = texture2D(diffuseMap, diffuseMapUV).a * alphaPassthrough;
     else
         gl_FragData[0].a = alphaPassthrough;
 
-    // Prevent translucent things casting shadow (including the player using an invisibility effect)
-    if (gl_FragData[0].a <= 0.5)
+    // Prevent translucent things casting shadow (including the player using an invisibility effect). For now, rely on the deprecated FF test for non-blended stuff.
+    if (alphaTestShadows && gl_FragData[0].a <= 0.5)
         discard;
 }
diff --git a/files/shaders/shadowcasting_vertex.glsl b/files/shaders/shadowcasting_vertex.glsl
index e19b587e5..e36f21a4d 100644
--- a/files/shaders/shadowcasting_vertex.glsl
+++ b/files/shaders/shadowcasting_vertex.glsl
@@ -5,8 +5,8 @@ varying vec2 diffuseMapUV;
 varying float alphaPassthrough;
 
 uniform int colorMode;
-uniform bool useDiffuseMapForShadowAlpha;
-uniform bool alphaTestShadows;
+uniform bool useDiffuseMapForShadowAlpha = true;
+uniform bool alphaTestShadows = true;
 
 void main(void)
 {
@@ -15,13 +15,11 @@ void main(void)
     vec4 viewPos = (gl_ModelViewMatrix * gl_Vertex);
     gl_ClipVertex = viewPos;
 
-    if (alphaTestShadows && useDiffuseMapForShadowAlpha)
+    if (useDiffuseMapForShadowAlpha)
         diffuseMapUV = (gl_TextureMatrix[0] * gl_MultiTexCoord0).xy;
     else
         diffuseMapUV = vec2(0.0); // Avoid undefined behaviour if running on hardware predating the concept of dynamically uniform expressions
-    if (!alphaTestShadows)
-        alphaPassthrough = 1.0;
-    else if (colorMode == 2)
+    if (colorMode == 2)
         alphaPassthrough = gl_Color.a;
     else
         // This is uniform, so if it's too low, we might be able to put the position/clip vertex outside the view frustum and skip the fragment shader and rasteriser