Initial A2C implementation

apps/openmw/mwrender/renderingmanager.cpp

@@ -219,6 +219,7 @@ namespace MWRender
         resourceSystem->getSceneManager()->setAutoUseSpecularMaps(Settings::Manager::getBool("auto use object specular maps", "Shaders"));
         resourceSystem->getSceneManager()->setSpecularMapPattern(Settings::Manager::getString("specular map pattern", "Shaders"));
         resourceSystem->getSceneManager()->setApplyLightingToEnvMaps(Settings::Manager::getBool("apply lighting to environment maps", "Shaders"));
+        resourceSystem->getSceneManager()->setConvertAlphaTestToAlphaToCoverage(Settings::Manager::getBool("convert alpha test to alpha-to-coverage", "Shaders"));
 
         osg::ref_ptr<SceneUtil::LightManager> sceneRoot = new SceneUtil::LightManager;
         sceneRoot->setLightingMask(Mask_Lighting);

components/resource/scenemanager.cpp

@@ -290,6 +290,11 @@ namespace Resource
         mApplyLightingToEnvMaps = apply;
     }
 
+    void SceneManager::setConvertAlphaTestToAlphaToCoverage(bool convert)
+    {
+        mConvertAlphaTestToAlphaToCoverage = convert;
+    }
+
     SceneManager::~SceneManager()
     {
         // this has to be defined in the .cpp file as we can't delete incomplete types
@@ -769,6 +774,7 @@ namespace Resource
         shaderVisitor->setAutoUseSpecularMaps(mAutoUseSpecularMaps);
         shaderVisitor->setSpecularMapPattern(mSpecularMapPattern);
         shaderVisitor->setApplyLightingToEnvMaps(mApplyLightingToEnvMaps);
+        shaderVisitor->setConvertAlphaTestToAlphaToCoverage(mConvertAlphaTestToAlphaToCoverage);
         return shaderVisitor;
     }
 

components/resource/scenemanager.hpp

@@ -75,6 +75,8 @@ namespace Resource
 
         void setApplyLightingToEnvMaps(bool apply);
 
+        void setConvertAlphaTestToAlphaToCoverage(bool convert);
+
         void setShaderPath(const std::string& path);
 
         /// Check if a given scene is loaded and if so, update its usage timestamp to prevent it from being unloaded
@@ -159,6 +161,7 @@ namespace Resource
         bool mAutoUseSpecularMaps;
         std::string mSpecularMapPattern;
         bool mApplyLightingToEnvMaps;
+        bool mConvertAlphaTestToAlphaToCoverage;
 
         osg::ref_ptr<MultiObjectCache> mInstanceCache;
 

components/sceneutil/mwshadowtechnique.cpp

@@ -897,7 +897,7 @@ void SceneUtil::MWShadowTechnique::setupCastingShader(Shader::ShaderManager & sh
         auto& program = _castingPrograms[alphaFunc - GL_NEVER];
         program = new osg::Program();
         program->addShader(castingVertexShader);
-        program->addShader(shaderManager.getShader("shadowcasting_fragment.glsl", { {"alphaFunc", std::to_string(alphaFunc)} }, osg::Shader::FRAGMENT));
+        program->addShader(shaderManager.getShader("shadowcasting_fragment.glsl", { {"alphaFunc", std::to_string(alphaFunc)}, {"alphaToCoverage", "0"} }, osg::Shader::FRAGMENT));
     }
 }
 

components/shader/shadervisitor.cpp

@@ -3,6 +3,7 @@
 #include <osg/AlphaFunc>
 #include <osg/Geometry>
 #include <osg/Material>
+#include <osg/Multisample>
 #include <osg/Texture>
 
 #include <osgUtil/TangentSpaceGenerator>
@@ -382,6 +383,7 @@ namespace Shader
         if (!removedState)
             removedState = new osg::StateSet();
 
+        defineMap["alphaToCoverage"] = "0";
         if (reqs.mAlphaFunc != osg::AlphaFunc::ALWAYS)
         {
             writableStateSet->addUniform(new osg::Uniform("alphaRef", reqs.mAlphaRef));
@@ -391,6 +393,12 @@ namespace Shader
                 removedState->setAttribute(alphaFunc->first, alphaFunc->second);
             // This prevents redundant glAlphaFunc calls while letting the shadows bin still see the test
             writableStateSet->setAttribute(RemovedAlphaFunc::getInstance(reqs.mAlphaFunc), osg::StateAttribute::ON | osg::StateAttribute::OVERRIDE);
+
+            if (mConvertAlphaTestToAlphaToCoverage)
+            {
+                writableStateSet->setMode(GL_SAMPLE_ALPHA_TO_COVERAGE_ARB, osg::StateAttribute::ON);
+                defineMap["alphaToCoverage"] = "1";
+            }
         }
 
         if (writableStateSet->getMode(GL_ALPHA_TEST) != osg::StateAttribute::INHERIT)
@@ -581,4 +589,9 @@ namespace Shader
         mApplyLightingToEnvMaps = apply;
     }
 
+    void ShaderVisitor::setConvertAlphaTestToAlphaToCoverage(bool convert)
+    {
+        mConvertAlphaTestToAlphaToCoverage = convert;
+    }
+
 }

components/shader/shadervisitor.hpp

@@ -40,6 +40,8 @@ namespace Shader
 
         void setApplyLightingToEnvMaps(bool apply);
 
+        void setConvertAlphaTestToAlphaToCoverage(bool convert);
+
         void apply(osg::Node& node) override;
 
         void apply(osg::Drawable& drawable) override;
@@ -63,6 +65,8 @@ namespace Shader
 
         bool mApplyLightingToEnvMaps;
 
+        bool mConvertAlphaTestToAlphaToCoverage;
+
         ShaderManager& mShaderManager;
         Resource::ImageManager& mImageManager;
 

docs/source/reference/modding/settings/shaders.rst

@@ -146,3 +146,14 @@ radial fog
 By default, the fog becomes thicker proportionally to your distance from the clipping plane set at the clipping distance, which causes distortion at the edges of the screen.
 This setting makes the fog use the actual eye point distance (or so called Euclidean distance) to calculate the fog, which makes the fog look less artificial, especially if you have a wide FOV.
 Note that the rendering will act as if you have 'force shaders' option enabled with this on, which means that shaders will be used to render all objects and the terrain.
+
+convert alpha test to alpha-to-coverage
+---------------------------------------
+
+:Type:		boolean
+:Range:		True/False
+:Default:	False
+
+Convert the alpha test (cutout/punchthrough alpha) to alpha-to-coverage.
+This allows MSAA to work with alpha-tested meshes, producing better-looking edges without pixelation.
+When MSAA is off, this setting will have no visible effect, but might have a performance cost.

files/settings-default.cfg

@@ -442,6 +442,11 @@ apply lighting to environment maps = false
 # This makes fogging independent from the viewing angle. Shaders will be used to render all objects.
 radial fog = false
 
+# Convert the alpha test (cutout/punchthrough alpha) to alpha-to-coverage.
+# This allows MSAA to work with alpha-tested meshes, producing better-looking edges without pixelation.
+# When MSAA is off, this setting will have no visible effect, but might have a performance cost.
+convert alpha test to alpha-to-coverage = false
+
 [Input]
 
 # Capture control of the cursor prevent movement outside the window.

files/shaders/alpha.glsl

@@ -14,25 +14,49 @@ uniform float alphaRef;
 
 void alphaTest()
 {
-    #if @alphaFunc == FUNC_NEVER
-        discard;
-    #elif @alphaFunc == FUNC_LESS
-        if (gl_FragData[0].a > alphaRef)
-            discard;
-    #elif @alphaFunc == FUNC_EQUAL
-        if (gl_FragData[0].a != alphaRef)
-            discard;
-    #elif @alphaFunc == FUNC_LEQUAL
-        if (gl_FragData[0].a >= alphaRef)
-            discard;
-    #elif @alphaFunc == FUNC_GREATER
-        if (gl_FragData[0].a < alphaRef)
-            discard;
-    #elif @alphaFunc == FUNC_NOTEQUAL
-        if (gl_FragData[0].a == alphaRef)
+    #if @alphaToCoverage 
+        float coverageAlpha = (gl_FragData[0].a - alphaRef) / max(fwidth(gl_FragData[0].a), 0.0001) + 0.5;
+
+        // Some functions don't make sense with A2C or are a pain to think about and no meshes use them anyway
+        // Use regular alpha testing in such cases until someone complains.
+        #if @alphaFunc == FUNC_NEVER
             discard;
-    #elif @alphaFunc == FUNC_GEQUAL
-        if (gl_FragData[0].a <= alphaRef)
+        #elif @alphaFunc == FUNC_LESS
+            gl_FragData[0].a = 1.0 - coverageAlpha;
+        #elif @alphaFunc == FUNC_EQUAL
+            if (gl_FragData[0].a != alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_LEQUAL
+            gl_FragData[0].a = 1.0 - coverageAlpha;
+        #elif @alphaFunc == FUNC_GREATER
+            gl_FragData[0].a = coverageAlpha;
+        #elif @alphaFunc == FUNC_NOTEQUAL
+            if (gl_FragData[0].a == alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_GEQUAL
+            gl_FragData[0].a = coverageAlpha;
+        #endif
+    #else
+        #if @alphaFunc == FUNC_NEVER
             discard;
+        #elif @alphaFunc == FUNC_LESS
+            if (gl_FragData[0].a > alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_EQUAL
+            if (gl_FragData[0].a != alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_LEQUAL
+            if (gl_FragData[0].a >= alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_GREATER
+            if (gl_FragData[0].a < alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_NOTEQUAL
+            if (gl_FragData[0].a == alphaRef)
+                discard;
+        #elif @alphaFunc == FUNC_GEQUAL
+            if (gl_FragData[0].a <= alphaRef)
+                discard;
+        #endif
     #endif
 }