Switch shadow map rendering to a specialised, simplified shader.

apps/openmw/mwrender/renderingmanager.cpp

@@ -239,7 +239,7 @@ namespace MWRender
         if (Settings::Manager::getBool("object shadows", "Shadows"))
             shadowCastingTraversalMask |= Mask_Object;
         
-        mShadowManager.reset(new SceneUtil::ShadowManager(sceneRoot, mRootNode, shadowCastingTraversalMask, indoorShadowCastingTraversalMask));
+        mShadowManager.reset(new SceneUtil::ShadowManager(sceneRoot, mRootNode, shadowCastingTraversalMask, indoorShadowCastingTraversalMask, mResourceSystem->getSceneManager()->getShaderManager()));
 
         Shader::ShaderManager::DefineMap shadowDefines = mShadowManager->getShadowDefines();
         Shader::ShaderManager::DefineMap globalDefines = mResourceSystem->getSceneManager()->getShaderManager().getGlobalDefines();

components/nifosg/nifloader.cpp

@@ -1249,6 +1249,7 @@ namespace NifOsg
                 boundTextures.clear();
             }
 
+            // If this loop is changed such that the base texture isn't guaranteed to end up in texture unit 0, the shadow casting shader will need to be updated accordingly.
             for (int i=0; i<Nif::NiTexturingProperty::NumTextures; ++i)
             {
                 if (texprop->textures[i].inUse)

components/sceneutil/mwshadowtechnique.cpp

@@ -820,6 +820,16 @@ void SceneUtil::MWShadowTechnique::setSplitPointDeltaBias(double bias)
     _splitPointDeltaBias = bias;
 }
 
+void SceneUtil::MWShadowTechnique::setupCastingShader(Shader::ShaderManager & shaderManager)
+{
+    // This can't be part of the constructor as OSG mandates that there be a trivial constructor available
+    
+    _castingProgram = new osg::Program();
+
+    _castingProgram->addShader(shaderManager.getShader("shadowcasting_vertex.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::VERTEX));
+    _castingProgram->addShader(shaderManager.getShader("shadowcasting_fragment.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::FRAGMENT));
+}
+
 MWShadowTechnique::ViewDependentData* MWShadowTechnique::createViewDependentData(osgUtil::CullVisitor* /*cv*/)
 {
     return new ViewDependentData(this);
@@ -1480,6 +1490,14 @@ void MWShadowTechnique::createShaders()
         _fallbackShadowMapTexture->setFilter(osg::Texture2D::MAG_FILTER,osg::Texture2D::NEAREST);
 
     }
+
+    if (!_castingProgram)
+        OSG_NOTICE << "Shadow casting shader has not been set up. Remember to call setupCastingShader(Shader::ShaderManager &)" << std::endl;
+
+    _shadowCastingStateSet->setAttributeAndModes(_castingProgram, osg::StateAttribute::ON | osg::StateAttribute::OVERRIDE);
+    // The casting program uses a sampler, so to avoid undefined behaviour, we must bind a dummy texture in case no other is supplied
+    _shadowCastingStateSet->setTextureAttributeAndModes(0, _fallbackBaseTexture.get(), osg::StateAttribute::ON);
+    _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
 }
 
 osg::Polytope MWShadowTechnique::computeLightViewFrustumPolytope(Frustum& frustum, LightData& positionedLight)

components/sceneutil/mwshadowtechnique.hpp

@@ -27,6 +27,7 @@
 
 #include <osgShadow/ShadowTechnique>
 
+#include <components/shader/shadermanager.hpp>
 #include <components/terrain/quadtreeworld.hpp>
 
 namespace SceneUtil {
@@ -73,6 +74,8 @@ namespace SceneUtil {
 
         virtual void setSplitPointDeltaBias(double bias);
 
+        virtual void setupCastingShader(Shader::ShaderManager &shaderManager);
+
         class ComputeLightSpaceBounds : public osg::NodeVisitor, public osg::CullStack
         {
         public:
@@ -265,6 +268,7 @@ namespace SceneUtil {
         };
 
         osg::ref_ptr<DebugHUD>                  _debugHud;
+        osg::ref_ptr<osg::Program>              _castingProgram;
     };
 
 }

components/sceneutil/shadow.cpp

@@ -68,7 +68,7 @@ namespace SceneUtil
             stateset->setTextureAttributeAndModes(i, fakeShadowMapTexture, osg::StateAttribute::ON | osg::StateAttribute::OVERRIDE | osg::StateAttribute::PROTECTED);
     }
 
-    ShadowManager::ShadowManager(osg::ref_ptr<osg::Group> sceneRoot, osg::ref_ptr<osg::Group> rootNode, unsigned int outdoorShadowCastingMask, unsigned int indoorShadowCastingMask) : mShadowedScene(new osgShadow::ShadowedScene),
+    ShadowManager::ShadowManager(osg::ref_ptr<osg::Group> sceneRoot, osg::ref_ptr<osg::Group> rootNode, unsigned int outdoorShadowCastingMask, unsigned int indoorShadowCastingMask, Shader::ShaderManager &shaderManager) : mShadowedScene(new osgShadow::ShadowedScene),
         mShadowTechnique(new MWShadowTechnique),
         mOutdoorShadowCastingMask(outdoorShadowCastingMask),
         mIndoorShadowCastingMask(indoorShadowCastingMask)
@@ -81,6 +81,8 @@ namespace SceneUtil
         mShadowSettings = mShadowedScene->getShadowSettings();
         setupShadowSettings();
 
+        mShadowTechnique->setupCastingShader(shaderManager);
+
         enableOutdoorMode();
     }
 

components/sceneutil/shadow.hpp

@@ -16,7 +16,7 @@ namespace SceneUtil
     public:
         static void disableShadowsForStateSet(osg::ref_ptr<osg::StateSet> stateSet);
 
-        ShadowManager(osg::ref_ptr<osg::Group> sceneRoot, osg::ref_ptr<osg::Group> rootNode, unsigned int outdoorShadowCastingMask, unsigned int indoorShadowCastingMask);
+        ShadowManager(osg::ref_ptr<osg::Group> sceneRoot, osg::ref_ptr<osg::Group> rootNode, unsigned int outdoorShadowCastingMask, unsigned int indoorShadowCastingMask, Shader::ShaderManager &shaderManager);
 
         virtual ~ShadowManager() = default;
 

components/shader/shadervisitor.cpp

@@ -215,6 +215,13 @@ namespace Shader
                     mRequirements.back().mShaderRequired = true;
                 }
             }
+
+            if (diffuseMap)
+            {
+                if (!writableStateSet)
+                    writableStateSet = getWritableStateSet(node);
+                writableStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", true));
+            }
         }
 
         const osg::StateSet::AttributeList& attributes = stateset->getAttributeList();

files/shaders/CMakeLists.txt

@@ -20,6 +20,8 @@ set(SHADER_FILES
     s360_vertex.glsl
     shadows_vertex.glsl
     shadows_fragment.glsl
+    shadowcasting_vertex.glsl
+    shadowcasting_fragment.glsl
 )
 
 copy_all_resource_files(${CMAKE_CURRENT_SOURCE_DIR} ${OPENMW_SHADERS_ROOT} ${DDIRRELATIVE} "${SHADER_FILES}")

files/shaders/shadowcasting_fragment.glsl

@@ -0,0 +1,17 @@
+#version 120
+
+uniform sampler2D diffuseMap;
+varying vec2 diffuseMapUV;
+
+varying float alphaPassthrough;
+
+uniform bool useDiffuseMapForShadowAlpha;
+
+void main()
+{
+    gl_FragData[0].rgb = vec3(1.0);
+    if (useDiffuseMapForShadowAlpha)
+        gl_FragData[0].a = texture2D(diffuseMap, diffuseMapUV).a * alphaPassthrough;
+    else
+        gl_FragData[0].a = alphaPassthrough;
+}

files/shaders/shadowcasting_vertex.glsl

@@ -0,0 +1,27 @@
+#version 120
+
+varying vec2 diffuseMapUV;
+
+varying float alphaPassthrough;
+
+uniform int colorMode;
+uniform bool useDiffuseMapForShadowAlpha;
+
+void main(void)
+{
+    gl_Position = gl_ModelViewProjectionMatrix * gl_Vertex;
+
+    vec4 viewPos = (gl_ModelViewMatrix * gl_Vertex);
+    gl_ClipVertex = viewPos;
+
+    if (useDiffuseMapForShadowAlpha)
+        diffuseMapUV = (gl_TextureMatrix[0] * gl_MultiTexCoord0).xy;
+    else
+        diffuseMapUV = vec2(0.0); // Avoid undefined behaviour if running on hardware predating the concept of dynamically uniform expressions
+
+    if (colorMode == 2)
+        alphaPassthrough = gl_Color.a;
+    else
+        // This is uniform, so if it's too low, we might be able to put the position/clip vertex outside the view frustum and skip the fragment shader and rasteriser
+        alphaPassthrough = gl_FrontMaterial.diffuse.a;
+}