Merge branch 'alpha-meddling' into 'master'

Replace deprecated alpha test in shader visitor

Closes #4899

See merge request OpenMW/openmw!473

CHANGELOG.md

@@ -116,6 +116,7 @@
     Feature #2686: Timestamps in openmw.log
     Feature #3171: OpenMW-CS: Instance drag selection
     Feature #4894: Consider actors as obstacles for pathfinding
+    Feature #4899: Alpha-To-Coverage Anti-Aliasing for alpha testing
     Feature #4977: Use the "default icon.tga" when an item's icon is not found
     Feature #5043: Head Bobbing
     Feature #5199: OpenMW-CS: Improve scene view colors

apps/openmw/mwrender/characterpreview.cpp

@@ -21,6 +21,7 @@
 #include <components/resource/resourcesystem.hpp>
 #include <components/sceneutil/lightmanager.hpp>
 #include <components/sceneutil/shadow.hpp>
+#include <components/settings/settings.hpp>
 
 #include "../mwbase/environment.hpp"
 #include "../mwbase/world.hpp"
@@ -166,7 +167,7 @@ namespace MWRender
         mCamera->setProjectionMatrixAsPerspective(fovYDegrees, sizeX/static_cast<float>(sizeY), 0.1f, 10000.f); // zNear and zFar are autocomputed
         mCamera->setViewport(0, 0, sizeX, sizeY);
         mCamera->setRenderOrder(osg::Camera::PRE_RENDER);
-        mCamera->attach(osg::Camera::COLOR_BUFFER, mTexture);
+        mCamera->attach(osg::Camera::COLOR_BUFFER, mTexture, 0, 0, false, Settings::Manager::getInt("antialiasing", "Video"));
         mCamera->setName("CharacterPreview");
         mCamera->setComputeNearFarMode(osg::Camera::COMPUTE_NEAR_FAR_USING_BOUNDING_VOLUMES);
         mCamera->setCullMask(~(Mask_UpdateVisitor));

apps/openmw/mwrender/groundcover.cpp

@@ -1,5 +1,6 @@
 #include "groundcover.hpp"
 
+#include <osg/AlphaFunc>
 #include <osg/Geometry>
 #include <osg/VertexAttribDivisor>
 
@@ -258,11 +259,16 @@ namespace MWRender
             // Keep link to original mesh to keep it in cache
             group->getOrCreateUserDataContainer()->addUserObject(new Resource::TemplateRef(temp));
 
+            mSceneManager->reinstateRemovedState(node);
+
             InstancingVisitor visitor(pair.second, worldCenter);
             node->accept(visitor);
             group->addChild(node);
         }
 
+        // Force a unified alpha handling instead of data from meshes
+        osg::ref_ptr<osg::AlphaFunc> alpha = new osg::AlphaFunc(osg::AlphaFunc::GEQUAL, 128.f / 255.f);
+        group->getOrCreateStateSet()->setAttributeAndModes(alpha.get(), osg::StateAttribute::ON);
         group->getBound();
         group->setNodeMask(Mask_Groundcover);
         mSceneManager->recreateShaders(group, "groundcover", false, true);

apps/openmw/mwrender/localmap.cpp

@@ -18,6 +18,7 @@
 #include <components/settings/settings.hpp>
 #include <components/sceneutil/visitor.hpp>
 #include <components/sceneutil/shadow.hpp>
+#include <components/sceneutil/util.hpp>
 #include <components/files/memorystream.hpp>
 
 #include "../mwbase/environment.hpp"
@@ -237,7 +238,7 @@ void LocalMap::setupRenderToTexture(osg::ref_ptr<osg::Camera> camera, int x, int
     texture->setWrap(osg::Texture::WRAP_S, osg::Texture::CLAMP_TO_EDGE);
     texture->setWrap(osg::Texture::WRAP_T, osg::Texture::CLAMP_TO_EDGE);
 
-    camera->attach(osg::Camera::COLOR_BUFFER, texture);
+    SceneUtil::attachAlphaToCoverageFriendlyFramebufferToCamera(camera, osg::Camera::COLOR_BUFFER, texture);
 
     camera->addChild(mSceneRoot);
     mRoot->addChild(camera);

apps/openmw/mwrender/renderingmanager.cpp

@@ -3,7 +3,6 @@
 #include <limits>
 #include <cstdlib>
 
-#include <osg/AlphaFunc>
 #include <osg/Light>
 #include <osg/LightModel>
 #include <osg/Fog>
@@ -26,6 +25,7 @@
 #include <components/resource/scenemanager.hpp>
 #include <components/resource/keyframemanager.hpp>
 
+#include <components/shader/removedalphafunc.hpp>
 #include <components/shader/shadermanager.hpp>
 
 #include <components/settings/settings.hpp>
@@ -212,6 +212,7 @@ namespace MWRender
         resourceSystem->getSceneManager()->setAutoUseSpecularMaps(Settings::Manager::getBool("auto use object specular maps", "Shaders"));
         resourceSystem->getSceneManager()->setSpecularMapPattern(Settings::Manager::getString("specular map pattern", "Shaders"));
         resourceSystem->getSceneManager()->setApplyLightingToEnvMaps(Settings::Manager::getBool("apply lighting to environment maps", "Shaders"));
+        resourceSystem->getSceneManager()->setConvertAlphaTestToAlphaToCoverage(Settings::Manager::getBool("antialias alpha test", "Shaders") && Settings::Manager::getInt("antialiasing", "Video") > 1);
 
         osg::ref_ptr<SceneUtil::LightManager> sceneRoot = new SceneUtil::LightManager;
         sceneRoot->setLightingMask(Mask_Lighting);
@@ -244,6 +245,7 @@ namespace MWRender
         globalDefines["clamp"] = Settings::Manager::getBool("clamp lighting", "Shaders") ? "1" : "0";
         globalDefines["preLightEnv"] = Settings::Manager::getBool("apply lighting to environment maps", "Shaders") ? "1" : "0";
         globalDefines["radialFog"] = Settings::Manager::getBool("radial fog", "Shaders") ? "1" : "0";
+        globalDefines["useGPUShader4"] = "0";
 
         float groundcoverDistance = (Constants::CellSizeInUnits * std::max(1, Settings::Manager::getInt("distance", "Groundcover")) - 1024) * 0.93;
         globalDefines["groundcoverFadeStart"] = std::to_string(groundcoverDistance * 0.9f);
@@ -310,10 +312,6 @@ namespace MWRender
             groundcoverRoot->setName("Groundcover Root");
             sceneRoot->addChild(groundcoverRoot);
 
-            // Force a unified alpha handling instead of data from meshes
-            osg::ref_ptr<osg::AlphaFunc> alpha = new osg::AlphaFunc(osg::AlphaFunc::GEQUAL, 128.f/255.f);
-            groundcoverRoot->getOrCreateStateSet()->setAttributeAndModes(alpha.get(), osg::StateAttribute::ON);
-
             mGroundcoverUpdater = new GroundcoverUpdater;
             groundcoverRoot->addUpdateCallback(mGroundcoverUpdater);
 
@@ -408,6 +406,11 @@ namespace MWRender
         mRootNode->getOrCreateStateSet()->addUniform(new osg::Uniform("far", mViewDistance));
         mRootNode->getOrCreateStateSet()->addUniform(new osg::Uniform("simpleWater", false));
 
+        // Hopefully, anything genuinely requiring the default alpha func of GL_ALWAYS explicitly sets it
+        mRootNode->getOrCreateStateSet()->setAttribute(Shader::RemovedAlphaFunc::getInstance(GL_ALWAYS));
+        // The transparent renderbin sets alpha testing on because that was faster on old GPUs. It's now slower and breaks things.
+        mRootNode->getOrCreateStateSet()->setMode(GL_ALPHA_TEST, osg::StateAttribute::OFF);
+
         mUniformNear = mRootNode->getOrCreateStateSet()->getUniform("near");
         mUniformFar = mRootNode->getOrCreateStateSet()->getUniform("far");
         updateProjectionMatrix();

apps/openmw/mwrender/water.cpp

@@ -26,6 +26,7 @@
 #include <components/resource/scenemanager.hpp>
 
 #include <components/sceneutil/shadow.hpp>
+#include <components/sceneutil/util.hpp>
 #include <components/sceneutil/waterutil.hpp>
 
 #include <components/misc/constants.hpp>
@@ -302,7 +303,7 @@ public:
         mRefractionTexture->setFilter(osg::Texture::MIN_FILTER, osg::Texture::LINEAR);
         mRefractionTexture->setFilter(osg::Texture::MAG_FILTER, osg::Texture::LINEAR);
 
-        attach(osg::Camera::COLOR_BUFFER, mRefractionTexture);
+        SceneUtil::attachAlphaToCoverageFriendlyFramebufferToCamera(this, osg::Camera::COLOR_BUFFER, mRefractionTexture);
 
         mRefractionDepthTexture = new osg::Texture2D;
         mRefractionDepthTexture->setTextureSize(rttSize, rttSize);
@@ -387,7 +388,7 @@ public:
         mReflectionTexture->setWrap(osg::Texture::WRAP_S, osg::Texture::CLAMP_TO_EDGE);
         mReflectionTexture->setWrap(osg::Texture::WRAP_T, osg::Texture::CLAMP_TO_EDGE);
 
-        attach(osg::Camera::COLOR_BUFFER, mReflectionTexture);
+        SceneUtil::attachAlphaToCoverageFriendlyFramebufferToCamera(this, osg::Camera::COLOR_BUFFER, mReflectionTexture);
 
         // XXX: should really flip the FrontFace on each renderable instead of forcing clockwise.
         osg::ref_ptr<osg::FrontFace> frontFace (new osg::FrontFace);

components/CMakeLists.txt

@@ -46,7 +46,7 @@ add_component_dir (resource
     )
 
 add_component_dir (shader
-    shadermanager shadervisitor
+    shadermanager shadervisitor removedalphafunc
     )
 
 add_component_dir (sceneutil

components/debug/gldebug.cpp

@@ -32,12 +32,16 @@ either expressed or implied, of the FreeBSD Project.
 #include "gldebug.hpp"
 
 #include <cstdlib>
+#include <memory>
 
 #include <components/debug/debuglog.hpp>
 
 // OpenGL constants not provided by OSG:
 #include <SDL_opengl_glext.h>
 
+namespace Debug
+{
+
     void debugCallback(GLenum source, GLenum type, GLuint id, GLenum severity, GLsizei length, const GLchar *message, const void *userParam)
     {
 #ifdef GL_DEBUG_OUTPUT
@@ -69,12 +73,12 @@ void debugCallback(GLenum source, GLenum type, GLuint id, GLenum severity, GLsiz
 
         std::string typeStr;
 
-    Debug::Level logSeverity = Debug::Warning;
+        Level logSeverity = Warning;
         switch (type)
         {
         case GL_DEBUG_TYPE_ERROR:
             typeStr = "ERROR";
-        logSeverity = Debug::Error;
+            logSeverity = Error;
             break;
         case GL_DEBUG_TYPE_DEPRECATED_BEHAVIOR:
             typeStr = "DEPRECATED_BEHAVIOR";
@@ -100,9 +104,34 @@ void debugCallback(GLenum source, GLenum type, GLuint id, GLenum severity, GLsiz
 #endif
     }
 
-void enableGLDebugExtension(unsigned int contextID)
+    class PushDebugGroup
+    {
+    public:
+        static std::unique_ptr<PushDebugGroup> sInstance;
+
+        void (GL_APIENTRY * glPushDebugGroup) (GLenum source, GLuint id, GLsizei length, const GLchar * message);
+
+        void (GL_APIENTRY * glPopDebugGroup) (void);
+
+        bool valid()
+        {
+            return glPushDebugGroup && glPopDebugGroup;
+        }
+    };
+
+    std::unique_ptr<PushDebugGroup> PushDebugGroup::sInstance{ std::make_unique<PushDebugGroup>() };
+
+    EnableGLDebugOperation::EnableGLDebugOperation() : osg::GraphicsOperation("EnableGLDebugOperation", false)
+    {
+    }
+
+    void EnableGLDebugOperation::operator()(osg::GraphicsContext* graphicsContext)
     {
 #ifdef GL_DEBUG_OUTPUT
+        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mMutex);
+
+        unsigned int contextID = graphicsContext->getState()->getContextID();
+
         typedef void (GL_APIENTRY *DEBUGPROC)(GLenum source, GLenum type, GLuint id, GLenum severity, GLsizei length, const GLchar *message, const void *userParam);
         typedef void (GL_APIENTRY *GLDebugMessageControlFunction)(GLenum source, GLenum type, GLenum severity, GLsizei count, const GLuint *ids, GLboolean enabled);
         typedef void (GL_APIENTRY *GLDebugMessageCallbackFunction)(DEBUGPROC, const void* userParam);
@@ -114,6 +143,8 @@ void enableGLDebugExtension(unsigned int contextID)
         {
             osg::setGLExtensionFuncPtr(glDebugMessageCallback, "glDebugMessageCallback");
             osg::setGLExtensionFuncPtr(glDebugMessageControl, "glDebugMessageControl");
+            osg::setGLExtensionFuncPtr(PushDebugGroup::sInstance->glPushDebugGroup, "glPushDebugGroup");
+            osg::setGLExtensionFuncPtr(PushDebugGroup::sInstance->glPopDebugGroup, "glPopDebugGroup");
         }
         else if (osg::isGLExtensionSupported(contextID, "GL_ARB_debug_output"))
         {
@@ -132,26 +163,14 @@ void enableGLDebugExtension(unsigned int contextID)
             glDebugMessageControl(GL_DONT_CARE, GL_DEBUG_TYPE_ERROR, GL_DEBUG_SEVERITY_MEDIUM, 0, nullptr, true);
             glDebugMessageCallback(debugCallback, nullptr);
 
-        Log(Debug::Info) << "OpenGL debug callback attached.";
+            Log(Info) << "OpenGL debug callback attached.";
         }
         else
 #endif
-        Log(Debug::Error) << "Unable to attach OpenGL debug callback.";
+            Log(Error) << "Unable to attach OpenGL debug callback.";
     }
 
-Debug::EnableGLDebugOperation::EnableGLDebugOperation() : osg::GraphicsOperation("EnableGLDebugOperation", false)
+    bool shouldDebugOpenGL()
-{
-}
-
-void Debug::EnableGLDebugOperation::operator()(osg::GraphicsContext* graphicsContext)
-{
-    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mMutex);
-
-    unsigned int contextID = graphicsContext->getState()->getContextID();
-    enableGLDebugExtension(contextID);
-}
-
-bool Debug::shouldDebugOpenGL()
     {
         const char* env = std::getenv("OPENMW_DEBUG_OPENGL");
         if (!env)
@@ -162,3 +181,93 @@ bool Debug::shouldDebugOpenGL()
 
         return str.find("OFF") == std::string::npos && str.find("0") == std::string::npos && str.find("NO") == std::string::npos;
     }
+
+    DebugGroup::DebugGroup(const std::string & message, GLuint id)
+        : mSource(GL_DEBUG_SOURCE_APPLICATION)
+        , mId(id)
+        , mMessage(message)
+    {
+    }
+
+    DebugGroup::DebugGroup(const DebugGroup & debugGroup, const osg::CopyOp & copyop)
+        : osg::StateAttribute(debugGroup, copyop)
+        , mSource(debugGroup.mSource)
+        , mId(debugGroup.mId)
+        , mMessage(debugGroup.mMessage)
+    {
+    }
+
+    void DebugGroup::apply(osg::State & state) const
+    {
+        if (!PushDebugGroup::sInstance->valid())
+        {
+            Log(Error) << "OpenGL debug groups not supported on this system, or OPENMW_DEBUG_OPENGL environment variable not set.";
+            return;
+        }
+
+        auto& attributeVec = state.getAttributeVec(this);
+        auto& lastAppliedStack = sLastAppliedStack[state.getContextID()];
+
+        size_t firstNonMatch = 0;
+        while (firstNonMatch < lastAppliedStack.size()
+               && ((firstNonMatch < attributeVec.size() && lastAppliedStack[firstNonMatch] == attributeVec[firstNonMatch].first)
+                   || lastAppliedStack[firstNonMatch] == this))
+            firstNonMatch++;
+
+        for (size_t i = lastAppliedStack.size(); i > firstNonMatch; --i)
+            lastAppliedStack[i - 1]->pop(state);
+        lastAppliedStack.resize(firstNonMatch);
+
+        lastAppliedStack.reserve(attributeVec.size());
+        for (size_t i = firstNonMatch; i < attributeVec.size(); ++i)
+        {
+            const DebugGroup* group = static_cast<const DebugGroup*>(attributeVec[i].first);
+            group->push(state);
+            lastAppliedStack.push_back(group);
+        }
+        if (!(lastAppliedStack.back() == this))
+        {
+            push(state);
+            lastAppliedStack.push_back(this);
+        }
+    }
+
+    int DebugGroup::compare(const StateAttribute & sa) const
+    {
+        COMPARE_StateAttribute_Types(DebugGroup, sa);
+
+        COMPARE_StateAttribute_Parameter(mSource);
+        COMPARE_StateAttribute_Parameter(mId);
+        COMPARE_StateAttribute_Parameter(mMessage);
+
+        return 0;
+    }
+
+    void DebugGroup::releaseGLObjects(osg::State * state) const
+    {
+        if (state)
+            sLastAppliedStack.erase(state->getContextID());
+        else
+            sLastAppliedStack.clear();
+    }
+
+    bool DebugGroup::isValid() const
+    {
+        return mSource || mId || mMessage.length();
+    }
+
+    void DebugGroup::push(osg::State & state) const
+    {
+        if (isValid())
+            PushDebugGroup::sInstance->glPushDebugGroup(mSource, mId, mMessage.size(), mMessage.c_str());
+    }
+
+    void DebugGroup::pop(osg::State & state) const
+    {
+        if (isValid())
+            PushDebugGroup::sInstance->glPopDebugGroup();
+    }
+
+    std::map<unsigned int, std::vector<const DebugGroup *>> DebugGroup::sLastAppliedStack{};
+
+}

components/debug/gldebug.hpp

@@ -17,5 +17,65 @@ namespace Debug
     };
 
     bool shouldDebugOpenGL();
+
+
+    /*
+        Debug groups allow rendering to be annotated, making debugging via APITrace/CodeXL/NSight etc. much clearer.
+
+        Because I've not thought of a quick and clean way of doing it without incurring a small performance cost,
+        there are no uses of this class checked in. For now, add annotations locally when you need them.
+
+        To use this class, add it to a StateSet just like any other StateAttribute. Prefer the string-only constructor.
+        You'll need OPENMW_DEBUG_OPENGL set to true, or shouldDebugOpenGL() redefined to just return true as otherwise
+        the extension function pointers won't get set up. That can maybe be cleaned up in the future.
+
+        Beware that consecutive identical debug groups (i.e. pointers match) won't always get applied due to OSG thinking
+        it's already applied them. Either avoid nesting the same object, add dummy groups so they're not consecutive, or
+        ensure the leaf group isn't identical to its parent.
+    */
+    class DebugGroup : public osg::StateAttribute
+    {
+    public:
+        DebugGroup()
+            : mSource(0)
+            , mId(0)
+            , mMessage("")
+        {}
+
+        DebugGroup(GLenum source, GLuint id, const std::string& message)
+            : mSource(source)
+            , mId(id)
+            , mMessage(message)
+        {}
+
+        DebugGroup(const std::string& message, GLuint id = 0);
+
+        DebugGroup(const DebugGroup& debugGroup, const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY);
+
+        META_StateAttribute(Debug, DebugGroup, osg::StateAttribute::Type(101));
+
+        void apply(osg::State& state) const override;
+
+        int compare(const StateAttribute& sa) const override;
+
+        void releaseGLObjects(osg::State* state = nullptr) const override;
+
+        virtual bool isValid() const;
+
+    protected:
+        virtual ~DebugGroup() = default;
+
+        virtual void push(osg::State& state) const;
+
+        virtual void pop(osg::State& state) const;
+
+        GLenum mSource;
+        GLuint mId;
+        std::string mMessage;
+
+        static std::map<unsigned int, std::vector<const DebugGroup *>> sLastAppliedStack;
+
+        friend EnableGLDebugOperation;
+    };
 }
 #endif

components/resource/scenemanager.cpp

@@ -257,6 +257,12 @@ namespace Resource
         node->accept(*shaderVisitor);
     }
 
+    void SceneManager::reinstateRemovedState(osg::ref_ptr<osg::Node> node)
+    {
+        osg::ref_ptr<Shader::ReinstateRemovedStateVisitor> reinstateRemovedStateVisitor = new Shader::ReinstateRemovedStateVisitor(false);
+        node->accept(*reinstateRemovedStateVisitor);
+    }
+
     void SceneManager::setClampLighting(bool clamp)
     {
         mClampLighting = clamp;
@@ -297,6 +303,11 @@ namespace Resource
         mApplyLightingToEnvMaps = apply;
     }
 
+    void SceneManager::setConvertAlphaTestToAlphaToCoverage(bool convert)
+    {
+        mConvertAlphaTestToAlphaToCoverage = convert;
+    }
+
     SceneManager::~SceneManager()
     {
         // this has to be defined in the .cpp file as we can't delete incomplete types
@@ -771,6 +782,7 @@ namespace Resource
         shaderVisitor->setAutoUseSpecularMaps(mAutoUseSpecularMaps);
         shaderVisitor->setSpecularMapPattern(mSpecularMapPattern);
         shaderVisitor->setApplyLightingToEnvMaps(mApplyLightingToEnvMaps);
+        shaderVisitor->setConvertAlphaTestToAlphaToCoverage(mConvertAlphaTestToAlphaToCoverage);
         shaderVisitor->setTranslucentFramebuffer(translucentFramebuffer);
         return shaderVisitor;
     }

components/resource/scenemanager.hpp

@@ -75,9 +75,14 @@ namespace Resource
 
         Shader::ShaderManager& getShaderManager();
 
-        /// Re-create shaders for this node, need to call this if texture stages or vertex color mode have changed.
+        /// Re-create shaders for this node, need to call this if alpha testing, texture stages or vertex color mode have changed.
         void recreateShaders(osg::ref_ptr<osg::Node> node, const std::string& shaderPrefix = "objects", bool translucentFramebuffer = false, bool forceShadersForNode = false);
 
+        /// Applying shaders to a node may replace some fixed-function state.
+        /// This restores it.
+        /// When editing such state, it should be reinstated before the edits, and shaders should be recreated afterwards.
+        void reinstateRemovedState(osg::ref_ptr<osg::Node> node);
+
         /// @see ShaderVisitor::setForceShaders
         void setForceShaders(bool force);
         bool getForceShaders() const;
@@ -100,6 +105,8 @@ namespace Resource
 
         void setApplyLightingToEnvMaps(bool apply);
 
+        void setConvertAlphaTestToAlphaToCoverage(bool convert);
+
         void setShaderPath(const std::string& path);
 
         /// Check if a given scene is loaded and if so, update its usage timestamp to prevent it from being unloaded
@@ -184,6 +191,7 @@ namespace Resource
         bool mAutoUseSpecularMaps;
         std::string mSpecularMapPattern;
         bool mApplyLightingToEnvMaps;
+        bool mConvertAlphaTestToAlphaToCoverage;
 
         osg::ref_ptr<MultiObjectCache> mInstanceCache;
 

components/sceneutil/mwshadowtechnique.cpp

@@ -278,7 +278,7 @@ void VDSMCameraCullCallback::operator()(osg::Node* node, osg::NodeVisitor* nv)
     static osg::ref_ptr<osg::StateSet> ss;
     if (!ss)
     {
-        ShadowsBinAdder adder("ShadowsBin");
+        ShadowsBinAdder adder("ShadowsBin", _vdsm->getCastingPrograms());
         ss = new osg::StateSet;
         ss->setRenderBinDetails(osg::StateSet::OPAQUE_BIN, "ShadowsBin", osg::StateSet::OVERRIDE_PROTECTED_RENDERBIN_DETAILS);
     }
@@ -782,7 +782,8 @@ void MWShadowTechnique::ViewDependentData::releaseGLObjects(osg::State* state) c
 MWShadowTechnique::MWShadowTechnique():
     ShadowTechnique(),
     _enableShadows(false),
-    _debugHud(nullptr)
+    _debugHud(nullptr),
+    _castingPrograms{ nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr }
 {
     _shadowRecievingPlaceholderStateSet = new osg::StateSet;
     mSetDummyStateWhenDisabled = false;
@@ -790,6 +791,7 @@ MWShadowTechnique::MWShadowTechnique():
 
 MWShadowTechnique::MWShadowTechnique(const MWShadowTechnique& vdsm, const osg::CopyOp& copyop):
     ShadowTechnique(vdsm,copyop)
+    , _castingPrograms(vdsm._castingPrograms)
 {
     _shadowRecievingPlaceholderStateSet = new osg::StateSet;
     _enableShadows = vdsm._enableShadows;
@@ -870,7 +872,10 @@ void SceneUtil::MWShadowTechnique::enableFrontFaceCulling()
     _useFrontFaceCulling = true;
 
     if (_shadowCastingStateSet)
+    {
         _shadowCastingStateSet->setAttribute(new osg::CullFace(osg::CullFace::FRONT), osg::StateAttribute::ON | osg::StateAttribute::OVERRIDE);
+        _shadowCastingStateSet->setMode(GL_CULL_FACE, osg::StateAttribute::OFF);
+    }
 }
 
 void SceneUtil::MWShadowTechnique::disableFrontFaceCulling()
@@ -878,17 +883,29 @@ void SceneUtil::MWShadowTechnique::disableFrontFaceCulling()
     _useFrontFaceCulling = false;
 
     if (_shadowCastingStateSet)
+    {
+        _shadowCastingStateSet->removeAttribute(osg::StateAttribute::CULLFACE);
         _shadowCastingStateSet->setMode(GL_CULL_FACE, osg::StateAttribute::OFF | osg::StateAttribute::OVERRIDE);
     }
+}
 
 void SceneUtil::MWShadowTechnique::setupCastingShader(Shader::ShaderManager & shaderManager)
 {
     // This can't be part of the constructor as OSG mandates that there be a trivial constructor available
 
-    _castingProgram = new osg::Program();
+    osg::ref_ptr<osg::Shader> castingVertexShader = shaderManager.getShader("shadowcasting_vertex.glsl", {}, osg::Shader::VERTEX);
-
+    osg::ref_ptr<osg::GLExtensions> exts = osg::GLExtensions::Get(0, false);
-    _castingProgram->addShader(shaderManager.getShader("shadowcasting_vertex.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::VERTEX));
+    std::string useGPUShader4 = exts && exts->isGpuShader4Supported ? "1" : "0";
-    _castingProgram->addShader(shaderManager.getShader("shadowcasting_fragment.glsl", Shader::ShaderManager::DefineMap(), osg::Shader::FRAGMENT));
+    for (int alphaFunc = GL_NEVER; alphaFunc <= GL_ALWAYS; ++alphaFunc)
+    {
+        auto& program = _castingPrograms[alphaFunc - GL_NEVER];
+        program = new osg::Program();
+        program->addShader(castingVertexShader);
+        program->addShader(shaderManager.getShader("shadowcasting_fragment.glsl", { {"alphaFunc", std::to_string(alphaFunc)},
+                                                                                    {"alphaToCoverage", "0"},
+                                                                                    {"useGPUShader4", useGPUShader4}
+                                                                                  }, osg::Shader::FRAGMENT));
+    }
 }
 
 MWShadowTechnique::ViewDependentData* MWShadowTechnique::createViewDependentData(osgUtil::CullVisitor* /*cv*/)
@@ -1606,10 +1623,11 @@ void MWShadowTechnique::createShaders()
 
     }
 
-    if (!_castingProgram)
+    if (!_castingPrograms[GL_ALWAYS - GL_NEVER])
         OSG_NOTICE << "Shadow casting shader has not been set up. Remember to call setupCastingShader(Shader::ShaderManager &)" << std::endl;
 
-    _shadowCastingStateSet->setAttributeAndModes(_castingProgram, osg::StateAttribute::ON | osg::StateAttribute::OVERRIDE);
+    // Always use the GL_ALWAYS shader as the shadows bin will change it if necessary
+    _shadowCastingStateSet->setAttributeAndModes(_castingPrograms[GL_ALWAYS - GL_NEVER], osg::StateAttribute::ON | osg::StateAttribute::OVERRIDE);
     // The casting program uses a sampler, so to avoid undefined behaviour, we must bind a dummy texture in case no other is supplied
     _shadowCastingStateSet->setTextureAttributeAndModes(0, _fallbackBaseTexture.get(), osg::StateAttribute::ON);
     _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", true));

components/sceneutil/mwshadowtechnique.hpp

@@ -215,6 +215,8 @@ namespace SceneUtil {
 
         virtual void createShaders();
 
+        virtual std::array<osg::ref_ptr<osg::Program>, GL_ALWAYS - GL_NEVER + 1> getCastingPrograms() const { return _castingPrograms; }
+
         virtual bool selectActiveLights(osgUtil::CullVisitor* cv, ViewDependentData* vdd) const;
 
         virtual osg::Polytope computeLightViewFrustumPolytope(Frustum& frustum, LightData& positionedLight);
@@ -288,7 +290,7 @@ namespace SceneUtil {
         };
 
         osg::ref_ptr<DebugHUD>                  _debugHud;
-        osg::ref_ptr<osg::Program>              _castingProgram;
+        std::array<osg::ref_ptr<osg::Program>, GL_ALWAYS - GL_NEVER + 1> _castingPrograms;
     };
 
 }

components/sceneutil/shadowsbin.cpp

@@ -1,7 +1,9 @@
 #include "shadowsbin.hpp"
 #include <unordered_set>
 #include <osg/StateSet>
+#include <osg/AlphaFunc>
 #include <osg/Material>
+#include <osg/Program>
 #include <osgUtil/StateGraph>
 
 using namespace osgUtil;
@@ -25,9 +27,9 @@ namespace
         osg::StateSet::ModeList::const_iterator mf = l.find(mode);
         if (mf == l.end())
             return;
-        int flags = mf->second;
+        unsigned int flags = mf->second;
         bool newValue = flags & osg::StateAttribute::ON;
-        accumulateState(currentValue, newValue, isOverride, ss->getMode(mode));
+        accumulateState(currentValue, newValue, isOverride, flags);
     }
 
     inline bool materialNeedShadows(osg::Material* m)
@@ -40,6 +42,10 @@ namespace
 namespace SceneUtil
 {
 
+std::array<osg::ref_ptr<osg::Program>, GL_ALWAYS - GL_NEVER + 1> ShadowsBin::sCastingPrograms = {
+    nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr
+};
+
 ShadowsBin::ShadowsBin()
 {
     mNoTestStateSet = new osg::StateSet;
@@ -48,10 +54,16 @@ ShadowsBin::ShadowsBin()
 
     mShaderAlphaTestStateSet = new osg::StateSet;
     mShaderAlphaTestStateSet->addUniform(new osg::Uniform("alphaTestShadows", true));
-    mShaderAlphaTestStateSet->setMode(GL_BLEND, osg::StateAttribute::OFF | osg::StateAttribute::OVERRIDE);
+    mShaderAlphaTestStateSet->setMode(GL_BLEND, osg::StateAttribute::OFF | osg::StateAttribute::PROTECTED | osg::StateAttribute::OVERRIDE);
+
+    for (size_t i = 0; i < sCastingPrograms.size(); ++i)
+    {
+        mAlphaFuncShaders[i] = new osg::StateSet;
+        mAlphaFuncShaders[i]->setAttribute(sCastingPrograms[i], osg::StateAttribute::ON | osg::StateAttribute::PROTECTED | osg::StateAttribute::OVERRIDE);
+    }
 }
 
-StateGraph* ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::unordered_set<StateGraph*>& uninterestingCache)
+StateGraph* ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::unordered_set<StateGraph*>& uninterestingCache, bool cullFaceOverridden)
 {
     std::vector<StateGraph*> return_path;
     State state;
@@ -71,7 +83,6 @@ StateGraph* ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::un
             continue;
 
         accumulateModeState(ss, state.mAlphaBlend, state.mAlphaBlendOverride, GL_BLEND);
-        accumulateModeState(ss, state.mAlphaTest, state.mAlphaTestOverride, GL_ALPHA_TEST);
 
         const osg::StateSet::AttributeList& attributes = ss->getAttributeList();
         osg::StateSet::AttributeList::const_iterator found = attributes.find(std::make_pair(osg::StateAttribute::MATERIAL, 0));
@@ -83,10 +94,21 @@ StateGraph* ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::un
                 state.mMaterial = nullptr;
         }
 
-        // osg::FrontFace specifies triangle winding, not front-face culling. We can't safely reparent anything under it.
+        found = attributes.find(std::make_pair(osg::StateAttribute::ALPHAFUNC, 0));
+        if (found != attributes.end())
+        {
+            // As force shaders is on, we know this is really a RemovedAlphaFunc
+            const osg::StateSet::RefAttributePair& rap = found->second;
+            accumulateState(state.mAlphaFunc, static_cast<osg::AlphaFunc*>(rap.first.get()), state.mAlphaFuncOverride, rap.second);
+        }
+
+        if (!cullFaceOverridden)
+        {
+            // osg::FrontFace specifies triangle winding, not front-face culling. We can't safely reparent anything under it unless GL_CULL_FACE is off or we flip face culling.
             found = attributes.find(std::make_pair(osg::StateAttribute::FRONTFACE, 0));
             if (found != attributes.end())
                 state.mImportantState = true;
+        }
 
         if ((*itr) != sg && !state.interesting())
             uninterestingCache.insert(*itr);
@@ -108,21 +130,45 @@ StateGraph* ShadowsBin::cullStateGraph(StateGraph* sg, StateGraph* root, std::un
     if (state.mAlphaBlend)
     {
         sg_new = sg->find_or_insert(mShaderAlphaTestStateSet);
-        for (RenderLeaf* leaf : sg->_leaves)
+        sg_new->_leaves = std::move(sg->_leaves);
-        {
+        for (RenderLeaf* leaf : sg_new->_leaves)
             leaf->_parent = sg_new;
-            sg_new->_leaves.push_back(leaf);
+        sg = sg_new;
     }
-        return sg_new;
+
+    // GL_ALWAYS is set by default by mwshadowtechnique
+    if (state.mAlphaFunc && state.mAlphaFunc->getFunction() != GL_ALWAYS)
+    {
+        sg_new = sg->find_or_insert(mAlphaFuncShaders[state.mAlphaFunc->getFunction() - GL_NEVER]);
+        sg_new->_leaves = std::move(sg->_leaves);
+        for (RenderLeaf* leaf : sg_new->_leaves)
+            leaf->_parent = sg_new;
+        sg = sg_new;
     }
+
     return sg;
 }
 
+void ShadowsBin::addPrototype(const std::string & name, const std::array<osg::ref_ptr<osg::Program>, GL_ALWAYS - GL_NEVER + 1>& castingPrograms)
+{
+    sCastingPrograms = castingPrograms;
+    osg::ref_ptr<osgUtil::RenderBin> bin(new ShadowsBin);
+    osgUtil::RenderBin::addRenderBinPrototype(name, bin);
+}
+
+inline bool ShadowsBin::State::needTexture() const
+{
+    return mAlphaBlend || (mAlphaFunc && mAlphaFunc->getFunction() != GL_ALWAYS);
+}
+
 bool ShadowsBin::State::needShadows() const
 {
-    if (!mMaterial)
+    if (mAlphaFunc && mAlphaFunc->getFunction() == GL_NEVER)
-        return true;
+        return false;
+    // other alpha func + material combinations might be skippable
+    if (mAlphaBlend && mMaterial)
         return materialNeedShadows(mMaterial);
+    return true;
 }
 
 void ShadowsBin::sortImplementation()
@@ -139,13 +185,27 @@ void ShadowsBin::sortImplementation()
         root = root->_parent;
         const osg::StateSet* ss = root->getStateSet();
         if (ss->getMode(GL_NORMALIZE) & osg::StateAttribute::ON // that is root stategraph of renderingmanager cpp
-           || ss->getAttribute(osg::StateAttribute::VIEWPORT)) // fallback to rendertargets sg just in case
+           || ss->getAttribute(osg::StateAttribute::VIEWPORT)) // fallback to rendertarget's sg just in case
             break;
         if (!root->_parent)
             return;
     }
     StateGraph* noTestRoot = root->find_or_insert(mNoTestStateSet.get());
-    // root is now a stategraph with useDiffuseMapForShadowAlpha disabled but minimal other state
+    // noTestRoot is now a stategraph with useDiffuseMapForShadowAlpha disabled but minimal other state
+
+    bool cullFaceOverridden = false;
+    while ((root = root->_parent))
+    {
+        if (!root->getStateSet())
+            continue;
+        unsigned int cullFaceFlags = root->getStateSet()->getMode(GL_CULL_FACE);
+        if (cullFaceFlags & osg::StateAttribute::OVERRIDE && !(cullFaceFlags & osg::StateAttribute::ON))
+        {
+            cullFaceOverridden = true;
+            break;
+        }
+    }
+
     noTestRoot->_leaves.reserve(_stateGraphList.size());
     StateGraphList newList;
     std::unordered_set<StateGraph*> uninterestingCache;
@@ -154,7 +214,7 @@ void ShadowsBin::sortImplementation()
         // Render leaves which shouldn't use the diffuse map for shadow alpha but do cast shadows become children of root, so graph is now empty. Don't add to newList.
         // Graphs containing just render leaves which don't cast shadows are discarded. Don't add to newList.
         // Graphs containing other leaves need to be in newList.
-        StateGraph* graphToAdd = cullStateGraph(graph, noTestRoot, uninterestingCache);
+        StateGraph* graphToAdd = cullStateGraph(graph, noTestRoot, uninterestingCache, cullFaceOverridden);
         if (graphToAdd)
             newList.push_back(graphToAdd);
     }

components/sceneutil/shadowsbin.hpp

@@ -1,11 +1,13 @@
 #ifndef OPENMW_COMPONENTS_SCENEUTIL_SHADOWBIN_H
 #define OPENMW_COMPONENTS_SCENEUTIL_SHADOWBIN_H
+#include <array>
 #include <unordered_set>
 #include <osgUtil/RenderBin>
 
 namespace osg
 {
     class Material;
+    class AlphaFunc;
 }
 
 namespace SceneUtil
@@ -15,8 +17,12 @@ namespace SceneUtil
     class ShadowsBin : public osgUtil::RenderBin
     {
     private:
+        static std::array<osg::ref_ptr<osg::Program>, GL_ALWAYS - GL_NEVER + 1> sCastingPrograms;
+
         osg::ref_ptr<osg::StateSet> mNoTestStateSet;
         osg::ref_ptr<osg::StateSet> mShaderAlphaTestStateSet;
+
+        std::array<osg::ref_ptr<osg::StateSet>, GL_ALWAYS - GL_NEVER + 1> mAlphaFuncShaders;
     public:
         META_Object(SceneUtil, ShadowsBin)
         ShadowsBin();
@@ -24,6 +30,7 @@ namespace SceneUtil
             : osgUtil::RenderBin(rhs, copyop)
             , mNoTestStateSet(rhs.mNoTestStateSet)
             , mShaderAlphaTestStateSet(rhs.mShaderAlphaTestStateSet)
+            , mAlphaFuncShaders(rhs.mAlphaFuncShaders)
             {}
 
         void sortImplementation() override;
@@ -33,8 +40,8 @@ namespace SceneUtil
             State()
                 : mAlphaBlend(false)
                 , mAlphaBlendOverride(false)
-                , mAlphaTest(false)
+                , mAlphaFunc(nullptr)
-                , mAlphaTestOverride(false)
+                , mAlphaFuncOverride(false)
                 , mMaterial(nullptr)
                 , mMaterialOverride(false)
                 , mImportantState(false)
@@ -42,33 +49,29 @@ namespace SceneUtil
 
             bool mAlphaBlend;
             bool mAlphaBlendOverride;
-            bool mAlphaTest;
+            osg::AlphaFunc* mAlphaFunc;
-            bool mAlphaTestOverride;
+            bool mAlphaFuncOverride;
             osg::Material* mMaterial;
             bool mMaterialOverride;
             bool mImportantState;
-            bool needTexture() const { return mAlphaBlend || mAlphaTest; }
+            bool needTexture() const;
             bool needShadows() const;
             // A state is interesting if there's anything about it that might affect whether we can optimise child state
             bool interesting() const
             {
-                return !needShadows() || needTexture() || mAlphaBlendOverride || mAlphaTestOverride || mMaterialOverride || mImportantState;
+                return !needShadows() || needTexture() || mAlphaBlendOverride || mAlphaFuncOverride || mMaterialOverride || mImportantState;
             }
         };
 
-        osgUtil::StateGraph* cullStateGraph(osgUtil::StateGraph* sg, osgUtil::StateGraph* root, std::unordered_set<osgUtil::StateGraph*>& uninteresting);
+        osgUtil::StateGraph* cullStateGraph(osgUtil::StateGraph* sg, osgUtil::StateGraph* root, std::unordered_set<osgUtil::StateGraph*>& uninteresting, bool cullFaceOverridden);
 
-        static void addPrototype(const std::string& name)
+        static void addPrototype(const std::string& name, const std::array<osg::ref_ptr<osg::Program>, GL_ALWAYS - GL_NEVER + 1>& castingPrograms);
-        {
-            osg::ref_ptr<osgUtil::RenderBin> bin (new ShadowsBin);
-            osgUtil::RenderBin::addRenderBinPrototype(name, bin);
-        }
     };
 
     class ShadowsBinAdder
     {
         public:
-        ShadowsBinAdder(const std::string& name){ ShadowsBin::addPrototype(name); }
+        ShadowsBinAdder(const std::string& name, const std::array<osg::ref_ptr<osg::Program>, GL_ALWAYS - GL_NEVER + 1>& castingPrograms){ ShadowsBin::addPrototype(name, castingPrograms); }
     };
 
 }

components/sceneutil/util.cpp

@@ -11,6 +11,7 @@
 
 #include <components/resource/imagemanager.hpp>
 #include <components/resource/scenemanager.hpp>
+#include <components/settings/settings.hpp>
 
 namespace SceneUtil
 {
@@ -260,4 +261,21 @@ osg::ref_ptr<GlowUpdater> addEnchantedGlow(osg::ref_ptr<osg::Node> node, Resourc
     return glowUpdater;
 }
 
+bool attachAlphaToCoverageFriendlyFramebufferToCamera(osg::Camera* camera, osg::Camera::BufferComponent buffer, osg::Texture * texture, unsigned int level, unsigned int face, bool mipMapGeneration)
+{
+    unsigned int samples = 0;
+    unsigned int colourSamples = 0;
+    bool addMSAAIntermediateTarget = Settings::Manager::getBool("antialias alpha test", "Shaders") && Settings::Manager::getInt("antialiasing", "Video") > 1;
+    if (addMSAAIntermediateTarget)
+    {
+        // Alpha-to-coverage requires a multisampled framebuffer.
+        // OSG will set that up automatically and resolve it to the specified single-sample texture for us.
+        // For some reason, two samples are needed, at least with some drivers.
+        samples = 2;
+        colourSamples = 1;
+    }
+    camera->attach(buffer, texture, level, face, mipMapGeneration, samples, colourSamples);
+    return addMSAAIntermediateTarget;
+}
+
 }

components/sceneutil/util.hpp

@@ -3,6 +3,7 @@
 
 #include <osg/Matrix>
 #include <osg/BoundingSphere>
+#include <osg/Camera>
 #include <osg/NodeCallback>
 #include <osg/Texture2D>
 #include <osg/Vec4f>
@@ -60,6 +61,9 @@ namespace SceneUtil
     bool hasUserDescription(const osg::Node* node, const std::string pattern);
 
     osg::ref_ptr<GlowUpdater> addEnchantedGlow(osg::ref_ptr<osg::Node> node, Resource::ResourceSystem* resourceSystem, osg::Vec4f glowColor, float glowDuration=-1);
+
+    // Alpha-to-coverage requires a multisampled framebuffer, so we need to set that up for RTTs
+    bool attachAlphaToCoverageFriendlyFramebufferToCamera(osg::Camera* camera, osg::Camera::BufferComponent buffer, osg::Texture* texture, unsigned int level = 0, unsigned int face = 0, bool mipMapGeneration = false);
 }
 
 #endif

components/shader/removedalphafunc.cpp

@@ -0,0 +1,20 @@
+#include "removedalphafunc.hpp"
+
+#include <cassert>
+
+#include <osg/State>
+
+namespace Shader
+{
+    std::array<osg::ref_ptr<RemovedAlphaFunc>, GL_ALWAYS - GL_NEVER + 1> RemovedAlphaFunc::sInstances{
+        nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr
+    };
+
+    osg::ref_ptr<RemovedAlphaFunc> RemovedAlphaFunc::getInstance(GLenum func)
+    {
+        assert(func >= GL_NEVER && func <= GL_ALWAYS);
+        if (!sInstances[func - GL_NEVER])
+            sInstances[func - GL_NEVER] = new RemovedAlphaFunc(static_cast<osg::AlphaFunc::ComparisonFunction>(func), 1.0);
+        return sInstances[func - GL_NEVER];
+    }
+}

components/shader/removedalphafunc.hpp

@@ -0,0 +1,40 @@
+#ifndef OPENMW_COMPONENTS_REMOVEDALPHAFUNC_H
+#define OPENMW_COMPONENTS_REMOVEDALPHAFUNC_H
+
+#include <array>
+
+#include <osg/AlphaFunc>
+
+namespace Shader
+{
+    // State attribute used when shader visitor replaces the deprecated alpha function with a shader
+    // Prevents redundant glAlphaFunc calls and lets the shadowsbin know the stateset had alpha testing
+    class RemovedAlphaFunc : public osg::AlphaFunc
+    {
+    public:
+        // Get a singleton-like instance with the right func (but a default threshold)
+        static osg::ref_ptr<RemovedAlphaFunc> getInstance(GLenum func);
+
+        RemovedAlphaFunc()
+            : osg::AlphaFunc()
+        {}
+
+        RemovedAlphaFunc(ComparisonFunction func, float ref)
+            : osg::AlphaFunc(func, ref)
+        {}
+
+        RemovedAlphaFunc(const RemovedAlphaFunc& raf, const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY)
+            : osg::AlphaFunc(raf, copyop)
+        {}
+
+        META_StateAttribute(Shader, RemovedAlphaFunc, ALPHAFUNC);
+
+        void apply(osg::State& state) const override {}
+
+    protected:
+        virtual ~RemovedAlphaFunc() = default;
+
+        static std::array<osg::ref_ptr<RemovedAlphaFunc>, GL_ALWAYS - GL_NEVER + 1> sInstances;
+    };
+}
+#endif //OPENMW_COMPONENTS_REMOVEDALPHAFUNC_H

components/shader/shadervisitor.cpp

@@ -1,7 +1,10 @@
 #include "shadervisitor.hpp"
 
+#include <osg/AlphaFunc>
 #include <osg/Geometry>
+#include <osg/GLExtensions>
 #include <osg/Material>
+#include <osg/Multisample>
 #include <osg/Texture>
 
 #include <osgUtil/TangentSpaceGenerator>
@@ -13,6 +16,7 @@
 #include <components/sceneutil/riggeometry.hpp>
 #include <components/sceneutil/morphgeometry.hpp>
 
+#include "removedalphafunc.hpp"
 #include "shadermanager.hpp"
 
 namespace Shader
@@ -22,6 +26,11 @@ namespace Shader
         : mShaderRequired(false)
         , mColorMode(0)
         , mMaterialOverridden(false)
+        , mAlphaTestOverridden(false)
+        , mAlphaBlendOverridden(false)
+        , mAlphaFunc(GL_ALWAYS)
+        , mAlphaRef(1.0)
+        , mAlphaBlend(false)
         , mNormalHeight(false)
         , mTexStageRequiringTangents(-1)
         , mNode(nullptr)
@@ -77,6 +86,34 @@ namespace Shader
         return newStateSet.get();
     }
 
+    osg::UserDataContainer* getWritableUserDataContainer(osg::Object& object)
+    {
+        if (!object.getUserDataContainer())
+            return object.getOrCreateUserDataContainer();
+
+        osg::ref_ptr<osg::UserDataContainer> newUserData = static_cast<osg::UserDataContainer *>(object.getUserDataContainer()->clone(osg::CopyOp::SHALLOW_COPY));
+        object.setUserDataContainer(newUserData);
+        return newUserData.get();
+    }
+
+    osg::StateSet* getRemovedState(osg::StateSet& stateSet)
+    {
+        if (!stateSet.getUserDataContainer())
+            return nullptr;
+
+        return static_cast<osg::StateSet *>(stateSet.getUserDataContainer()->getUserObject("removedState"));
+    }
+
+    void updateRemovedState(osg::UserDataContainer& userData, osg::StateSet* stateSet)
+    {
+        unsigned int index = userData.getUserObjectIndex("removedState");
+        if (index < userData.getNumUserObjects())
+            userData.setUserObject(index, stateSet);
+        else
+            userData.addUserObject(stateSet);
+        stateSet->setName("removedState");
+    }
+
     const char* defaultTextures[] = { "diffuseMap", "normalMap", "emissiveMap", "darkMap", "detailMap", "envMap", "specularMap", "decalMap", "bumpMap" };
     bool isTextureNameRecognized(const std::string& name)
     {
@@ -235,7 +272,13 @@ namespace Shader
         }
 
         const osg::StateSet::AttributeList& attributes = stateset->getAttributeList();
-        for (osg::StateSet::AttributeList::const_iterator it = attributes.begin(); it != attributes.end(); ++it)
+        osg::StateSet::AttributeList removedAttributes;
+        osg::ref_ptr<osg::StateSet> removedState;
+        if (removedState = getRemovedState(*stateset))
+            removedAttributes = removedState->getAttributeList();
+        for (const auto& attributeMap : { attributes, removedAttributes })
+        {
+            for (osg::StateSet::AttributeList::const_iterator it = attributeMap.begin(); it != attributeMap.end(); ++it)
             {
                 if (it->first.first == osg::StateAttribute::MATERIAL)
                 {
@@ -277,7 +320,28 @@ namespace Shader
                         mRequirements.back().mColorMode = colorMode;
                     }
                 }
-            // Eventually, move alpha testing to discard in shader adn remove deprecated state here
+                else if (it->first.first == osg::StateAttribute::ALPHAFUNC)
+                {
+                    if (!mRequirements.back().mAlphaTestOverridden || it->second.second & osg::StateAttribute::PROTECTED)
+                    {
+                        if (it->second.second & osg::StateAttribute::OVERRIDE)
+                            mRequirements.back().mAlphaTestOverridden = true;
+
+                        const osg::AlphaFunc* alpha = static_cast<const osg::AlphaFunc*>(it->second.first.get());
+                        mRequirements.back().mAlphaFunc = alpha->getFunction();
+                        mRequirements.back().mAlphaRef = alpha->getReferenceValue();
+                    }
+                }
+            }
+        }
+
+        unsigned int alphaBlend = stateset->getMode(GL_BLEND);
+        if (alphaBlend != osg::StateAttribute::INHERIT && (!mRequirements.back().mAlphaBlendOverridden || alphaBlend & osg::StateAttribute::PROTECTED))
+        {
+            if (alphaBlend & osg::StateAttribute::OVERRIDE)
+                mRequirements.back().mAlphaBlendOverridden = true;
+
+            mRequirements.back().mAlphaBlend = alphaBlend & osg::StateAttribute::ON;
         }
     }
 
@@ -323,6 +387,57 @@ namespace Shader
 
         writableStateSet->addUniform(new osg::Uniform("colorMode", reqs.mColorMode));
 
+        defineMap["alphaFunc"] = std::to_string(reqs.mAlphaFunc);
+
+        // back up removed state in case recreateShaders gets rid of the shader later
+        osg::ref_ptr<osg::StateSet> removedState;
+        if ((removedState = getRemovedState(*writableStateSet)) && !mAllowedToModifyStateSets)
+            removedState = new osg::StateSet(*removedState, osg::CopyOp::SHALLOW_COPY);
+        if (!removedState)
+            removedState = new osg::StateSet();
+
+        defineMap["alphaToCoverage"] = "0";
+        if (reqs.mAlphaFunc != osg::AlphaFunc::ALWAYS)
+        {
+            writableStateSet->addUniform(new osg::Uniform("alphaRef", reqs.mAlphaRef));
+
+            const auto* alphaFunc = writableStateSet->getAttributePair(osg::StateAttribute::ALPHAFUNC);
+            if (alphaFunc)
+                removedState->setAttribute(alphaFunc->first, alphaFunc->second);
+            // This prevents redundant glAlphaFunc calls while letting the shadows bin still see the test
+            writableStateSet->setAttribute(RemovedAlphaFunc::getInstance(reqs.mAlphaFunc), osg::StateAttribute::ON | osg::StateAttribute::OVERRIDE);
+
+            // Blending won't work with A2C as we use the alpha channel for coverage. gl_SampleCoverage from ARB_sample_shading would save the day, but requires GLSL 130
+            if (mConvertAlphaTestToAlphaToCoverage && !reqs.mAlphaBlend)
+            {
+                writableStateSet->setMode(GL_SAMPLE_ALPHA_TO_COVERAGE_ARB, osg::StateAttribute::ON);
+                defineMap["alphaToCoverage"] = "1";
+            }
+
+            // Preventing alpha tested stuff shrinking as lower mip levels are used requires knowing the texture size
+            osg::ref_ptr<osg::GLExtensions> exts = osg::GLExtensions::Get(0, false);
+            if (exts && exts->isGpuShader4Supported)
+                defineMap["useGPUShader4"] = "1";
+            // We could fall back to a texture size uniform if EXT_gpu_shader4 is missing
+        }
+
+        if (writableStateSet->getMode(GL_ALPHA_TEST) != osg::StateAttribute::INHERIT)
+            removedState->setMode(GL_ALPHA_TEST, writableStateSet->getMode(GL_ALPHA_TEST));
+        // This disables the deprecated fixed-function alpha test
+        writableStateSet->setMode(GL_ALPHA_TEST, osg::StateAttribute::OFF | osg::StateAttribute::PROTECTED);
+
+        if (!removedState->getModeList().empty() || !removedState->getAttributeList().empty())
+        {
+            // user data is normally shallow copied so shared with the original stateset
+            osg::ref_ptr<osg::UserDataContainer> writableUserData;
+            if (mAllowedToModifyStateSets)
+                writableUserData = writableStateSet->getOrCreateUserDataContainer();
+            else
+                writableUserData = getWritableUserDataContainer(*writableStateSet);
+
+            updateRemovedState(*writableUserData, removedState);
+        }
+
         defineMap["translucentFramebuffer"] = mTranslucentFramebuffer ? "1" : "0";
 
         osg::ref_ptr<osg::Shader> vertexShader (mShaderManager.getShader(mDefaultVsTemplate, defineMap, osg::Shader::VERTEX));
@@ -350,6 +465,25 @@ namespace Shader
             writableStateSet = getWritableStateSet(node);
 
         writableStateSet->removeAttribute(osg::StateAttribute::PROGRAM);
+
+        osg::ref_ptr<osg::StateSet> removedState;
+        if (removedState = getRemovedState(*writableStateSet))
+        {
+            // user data is normally shallow copied so shared with the original stateset
+            osg::ref_ptr<osg::UserDataContainer> writableUserData;
+            if (mAllowedToModifyStateSets)
+                writableUserData = writableStateSet->getUserDataContainer();
+            else
+                writableUserData = getWritableUserDataContainer(*writableStateSet);
+            unsigned int index = writableUserData->getUserObjectIndex("removedState");
+            writableUserData->removeUserObject(index);
+
+            for (const auto& [mode, value] : removedState->getModeList())
+                writableStateSet->setMode(mode, value);
+
+            for (const auto& attribute : removedState->getAttributeList())
+                writableStateSet->setAttribute(attribute.second.first, attribute.second.second);
+        }
     }
 
     bool ShaderVisitor::adjustGeometry(osg::Geometry& sourceGeometry, const ShaderRequirements& reqs)
@@ -477,9 +611,53 @@ namespace Shader
         mApplyLightingToEnvMaps = apply;
     }
 
+    void ShaderVisitor::setConvertAlphaTestToAlphaToCoverage(bool convert)
+    {
+        mConvertAlphaTestToAlphaToCoverage = convert;
+    }
+
     void ShaderVisitor::setTranslucentFramebuffer(bool translucent)
     {
         mTranslucentFramebuffer = translucent;
     }
 
+    ReinstateRemovedStateVisitor::ReinstateRemovedStateVisitor(bool allowedToModifyStateSets)
+        : osg::NodeVisitor(TRAVERSE_ALL_CHILDREN)
+        , mAllowedToModifyStateSets(allowedToModifyStateSets)
+    {
+    }
+
+    void ReinstateRemovedStateVisitor::apply(osg::Node& node)
+    {
+        if (node.getStateSet())
+        {
+            osg::ref_ptr<osg::StateSet> removedState = getRemovedState(*node.getStateSet());
+            if (removedState)
+            {
+                osg::ref_ptr<osg::StateSet> writableStateSet;
+                if (mAllowedToModifyStateSets)
+                    writableStateSet = node.getStateSet();
+                else
+                    writableStateSet = getWritableStateSet(node);
+
+                // user data is normally shallow copied so shared with the original stateset
+                osg::ref_ptr<osg::UserDataContainer> writableUserData;
+                if (mAllowedToModifyStateSets)
+                    writableUserData = writableStateSet->getUserDataContainer();
+                else
+                    writableUserData = getWritableUserDataContainer(*writableStateSet);
+                unsigned int index = writableUserData->getUserObjectIndex("removedState");
+                writableUserData->removeUserObject(index);
+
+                for (const auto&[mode, value] : removedState->getModeList())
+                    writableStateSet->setMode(mode, value);
+
+                for (const auto& attribute : removedState->getAttributeList())
+                    writableStateSet->setAttribute(attribute.second.first, attribute.second.second);
+            }
+        }
+
+        traverse(node);
+    }
+
 }

components/shader/shadervisitor.hpp

@@ -40,6 +40,8 @@ namespace Shader
 
         void setApplyLightingToEnvMaps(bool apply);
 
+        void setConvertAlphaTestToAlphaToCoverage(bool convert);
+
         void setTranslucentFramebuffer(bool translucent);
 
         void apply(osg::Node& node) override;
@@ -65,6 +67,8 @@ namespace Shader
 
         bool mApplyLightingToEnvMaps;
 
+        bool mConvertAlphaTestToAlphaToCoverage;
+
         bool mTranslucentFramebuffer;
 
         ShaderManager& mShaderManager;
@@ -83,6 +87,12 @@ namespace Shader
             int mColorMode;
             
             bool mMaterialOverridden;
+            bool mAlphaTestOverridden;
+            bool mAlphaBlendOverridden;
+
+            GLenum mAlphaFunc;
+            float mAlphaRef;
+            bool mAlphaBlend;
 
             bool mNormalHeight; // true if normal map has height info in alpha channel
 
@@ -102,6 +112,17 @@ namespace Shader
         bool adjustGeometry(osg::Geometry& sourceGeometry, const ShaderRequirements& reqs);
     };
 
+    class ReinstateRemovedStateVisitor : public osg::NodeVisitor
+    {
+    public:
+        ReinstateRemovedStateVisitor(bool allowedToModifyStateSets);
+
+        void apply(osg::Node& node) override;
+
+    private:
+        bool mAllowedToModifyStateSets;
+    };
+
 }
 
 #endif

docs/source/reference/modding/settings/shaders.rst

@@ -147,3 +147,14 @@ radial fog
 By default, the fog becomes thicker proportionally to your distance from the clipping plane set at the clipping distance, which causes distortion at the edges of the screen.
 This setting makes the fog use the actual eye point distance (or so called Euclidean distance) to calculate the fog, which makes the fog look less artificial, especially if you have a wide FOV.
 Note that the rendering will act as if you have 'force shaders' option enabled with this on, which means that shaders will be used to render all objects and the terrain.
+
+antialias alpha test
+---------------------------------------
+
+:Type:		boolean
+:Range:		True/False
+:Default:	False
+
+Convert the alpha test (cutout/punchthrough alpha) to alpha-to-coverage when :ref:`antialiasing` is on.
+This allows MSAA to work with alpha-tested meshes, producing better-looking edges without pixelation.
+When MSAA is off, this setting will have no visible effect, but might have a performance cost.

files/settings-default.cfg

@@ -442,6 +442,11 @@ apply lighting to environment maps = false
 # This makes fogging independent from the viewing angle. Shaders will be used to render all objects.
 radial fog = false
 
+# Convert the alpha test (cutout/punchthrough alpha) to alpha-to-coverage.
+# This allows MSAA to work with alpha-tested meshes, producing better-looking edges without pixelation.
+# When MSAA is off, this setting will have no visible effect, but might have a performance cost.
+antialias alpha test = false
+
 [Input]
 
 # Capture control of the cursor prevent movement outside the window.

files/shaders/CMakeLists.txt

@@ -12,6 +12,7 @@ set(SHADER_FILES
     water_vertex.glsl
     water_fragment.glsl
     water_nm.png
+    alpha.glsl
     objects_vertex.glsl
     objects_fragment.glsl
     terrain_vertex.glsl

files/shaders/alpha.glsl

@@ -0,0 +1,85 @@
+
+#define FUNC_NEVER                          512 // 0x0200
+#define FUNC_LESS                           513 // 0x0201
+#define FUNC_EQUAL                          514 // 0x0202
+#define FUNC_LEQUAL                         515 // 0x0203
+#define FUNC_GREATER                        516 // 0x0204
+#define FUNC_NOTEQUAL                       517 // 0x0205
+#define FUNC_GEQUAL                         518 // 0x0206
+#define FUNC_ALWAYS                         519 // 0x0207
+
+#if @alphaFunc != FUNC_ALWAYS && @alphaFunc != FUNC_NEVER
+uniform float alphaRef;
+#endif
+
+float mipmapLevel(vec2 scaleduv)
+{
+    vec2 dUVdx = dFdx(scaleduv);
+    vec2 dUVdy = dFdy(scaleduv);
+    float maxDUVSquared = max(dot(dUVdx, dUVdx), dot(dUVdy, dUVdy));
+    return max(0.0, 0.5 * log2(maxDUVSquared));
+}
+
+float coveragePreservingAlphaScale(sampler2D diffuseMap, vec2 uv)
+{
+    #if @alphaFunc != FUNC_ALWAYS && @alphaFunc != FUNC_NEVER
+        vec2 textureSize;
+        #if @useGPUShader4
+            textureSize = textureSize2D(diffuseMap, 0);
+        #else
+            textureSize = vec2(256.0);
+        #endif
+            return 1.0 + mipmapLevel(uv * textureSize) * 0.25;
+    #else
+        return 1.0;
+    #endif
+}
+
+void alphaTest()
+{
+    #if @alphaToCoverage 
+        float coverageAlpha = (gl_FragData[0].a - clamp(alphaRef, 0.0001, 0.9999)) / max(fwidth(gl_FragData[0].a), 0.0001) + 0.5;
+
+        // Some functions don't make sense with A2C or are a pain to think about and no meshes use them anyway
+        // Use regular alpha testing in such cases until someone complains.
+        #if @alphaFunc == FUNC_NEVER
+            discard;
+        #elif @alphaFunc == FUNC_LESS
+            gl_FragData[0].a = 1.0 - coverageAlpha;
+        #elif @alphaFunc == FUNC_EQUAL
+            if (gl_FragData[0].a != alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_LEQUAL
+            gl_FragData[0].a = 1.0 - coverageAlpha;
+        #elif @alphaFunc == FUNC_GREATER
+            gl_FragData[0].a = coverageAlpha;
+        #elif @alphaFunc == FUNC_NOTEQUAL
+            if (gl_FragData[0].a == alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_GEQUAL
+            gl_FragData[0].a = coverageAlpha;
+        #endif
+    #else
+        #if @alphaFunc == FUNC_NEVER
+            discard;
+        #elif @alphaFunc == FUNC_LESS
+            if (gl_FragData[0].a >= alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_EQUAL
+            if (gl_FragData[0].a != alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_LEQUAL
+            if (gl_FragData[0].a > alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_GREATER
+            if (gl_FragData[0].a <= alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_NOTEQUAL
+            if (gl_FragData[0].a == alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_GEQUAL
+            if (gl_FragData[0].a < alphaRef)
+                discard;
+        #endif
+    #endif
+}

files/shaders/groundcover_fragment.glsl

@@ -1,5 +1,9 @@
 #version 120
 
+#if @useGPUShader4
+    #extension EXT_gpu_shader4: require
+#endif
+
 #define GROUNDCOVER
 
 #if @diffuseMap
@@ -30,11 +34,7 @@ centroid varying vec3 shadowDiffuseLighting;
 
 #include "shadows_fragment.glsl"
 #include "lighting.glsl"
-
+#include "alpha.glsl"
-float calc_coverage(float a, float alpha_ref, float falloff_rate)
-{
-    return clamp(falloff_rate * (a - alpha_ref) + alpha_ref, 0.0, 1.0);
-}
 
 void main()
 {
@@ -55,12 +55,13 @@ void main()
     gl_FragData[0] = vec4(1.0);
 #endif
 
-    gl_FragData[0].a = calc_coverage(gl_FragData[0].a, 128.0/255.0, 4.0);
-
-    float shadowing = unshadowedLightRatio(linearDepth);
     if (euclideanDepth > @groundcoverFadeStart)
         gl_FragData[0].a *= 1.0-smoothstep(@groundcoverFadeStart, @groundcoverFadeEnd, euclideanDepth);
 
+    alphaTest();
+
+    float shadowing = unshadowedLightRatio(linearDepth);
+
     vec3 lighting;
 #if !PER_PIXEL_LIGHTING
     lighting = passLighting + shadowDiffuseLighting * shadowing;

files/shaders/objects_fragment.glsl

@@ -1,5 +1,9 @@
 #version 120
 
+#if @useGPUShader4
+    #extension GL_EXT_gpu_shader4: require
+#endif
+
 #if @diffuseMap
 uniform sampler2D diffuseMap;
 varying vec2 diffuseMapUV;
@@ -68,6 +72,7 @@ varying vec3 passNormal;
 #include "shadows_fragment.glsl"
 #include "lighting.glsl"
 #include "parallax.glsl"
+#include "alpha.glsl"
 
 void main()
 {
@@ -109,10 +114,15 @@ void main()
 
 #if @diffuseMap
     gl_FragData[0] = texture2D(diffuseMap, adjustedDiffuseUV);
+    gl_FragData[0].a *= coveragePreservingAlphaScale(diffuseMap, adjustedDiffuseUV);
 #else
     gl_FragData[0] = vec4(1.0);
 #endif
 
+    vec4 diffuseColor = getDiffuseColor();
+    gl_FragData[0].a *= diffuseColor.a;
+    alphaTest();
+
 #if @detailMap
     gl_FragData[0].xyz *= texture2D(detailMap, detailMapUV).xyz * 2.0;
 #endif
@@ -151,9 +161,6 @@ void main()
 
 #endif
 
-    vec4 diffuseColor = getDiffuseColor();
-    gl_FragData[0].a *= diffuseColor.a;
-
     float shadowing = unshadowedLightRatio(linearDepth);
     vec3 lighting;
 #if !PER_PIXEL_LIGHTING

files/shaders/shadowcasting_fragment.glsl

@@ -1,5 +1,9 @@
 #version 120
 
+#if @useGPUShader4
+    #extension EXT_gpu_shader4: require
+#endif
+
 uniform sampler2D diffuseMap;
 varying vec2 diffuseMapUV;
 
@@ -8,6 +12,8 @@ varying float alphaPassthrough;
 uniform bool useDiffuseMapForShadowAlpha;
 uniform bool alphaTestShadows;
 
+#include "alpha.glsl"
+
 void main()
 {
     gl_FragData[0].rgb = vec3(1.0);
@@ -16,7 +22,10 @@ void main()
     else
         gl_FragData[0].a = alphaPassthrough;
 
-    // Prevent translucent things casting shadow (including the player using an invisibility effect). For now, rely on the deprecated FF test for non-blended stuff.
+    alphaTest();
+
+    // Prevent translucent things casting shadow (including the player using an invisibility effect).
+    // This replaces alpha blending, which obviously doesn't work with depth buffers
     if (alphaTestShadows && gl_FragData[0].a <= 0.5)
         discard;
 }