Merge branch 'now-it-blends' into 'master'

Convert blending factors properly for the character preview

See merge request OpenMW/openmw!644

apps/openmw/mwrender/characterpreview.cpp

@@ -84,7 +84,8 @@ namespace MWRender
     };
 
 
-    // Set up alpha blending to Additive mode to avoid issues caused by transparent objects writing onto the alpha value of the FBO
+    // Set up alpha blending mode to avoid issues caused by transparent objects writing onto the alpha value of the FBO
+    // This makes the RTT have premultiplied alpha, though, so the source blend factor must be GL_ONE when it's applied
     class SetUpBlendVisitor : public osg::NodeVisitor
     {
     public:
@@ -94,22 +95,40 @@ namespace MWRender
 
         void apply(osg::Node& node) override
         {
-            if (osg::StateSet* stateset = node.getStateSet())
+            if (osg::ref_ptr<osg::StateSet> stateset = node.getStateSet())
             {
+                osg::ref_ptr<osg::StateSet> newStateSet;
                 if (stateset->getAttribute(osg::StateAttribute::BLENDFUNC) || stateset->getBinNumber() == osg::StateSet::TRANSPARENT_BIN)
                 {
-                    osg::ref_ptr<osg::StateSet> newStateSet = new osg::StateSet(*stateset, osg::CopyOp::SHALLOW_COPY);
                     osg::BlendFunc* blendFunc = static_cast<osg::BlendFunc*>(stateset->getAttribute(osg::StateAttribute::BLENDFUNC));
-                    osg::ref_ptr<osg::BlendFunc> newBlendFunc = blendFunc ? new osg::BlendFunc(*blendFunc) : new osg::BlendFunc;
-                    newBlendFunc->setDestinationAlpha(osg::BlendFunc::ONE);
-                    newStateSet->setAttribute(newBlendFunc, osg::StateAttribute::ON);
-                    node.setStateSet(newStateSet);
+
+                    if (blendFunc)
+                    {
+                        newStateSet = new osg::StateSet(*stateset, osg::CopyOp::SHALLOW_COPY);
+                        node.setStateSet(newStateSet);
+                        osg::ref_ptr<osg::BlendFunc> newBlendFunc = new osg::BlendFunc(*blendFunc);
+                        newStateSet->setAttribute(newBlendFunc, osg::StateAttribute::ON);
+                        // I *think* (based on some by-hand maths) that the RGB and dest alpha factors are unchanged, and only dest determines source alpha factor
+                        // This has the benefit of being idempotent if we assume nothing used glBlendFuncSeparate before we touched it
+                        if (blendFunc->getDestination() == osg::BlendFunc::ONE_MINUS_SRC_ALPHA)
+                            newBlendFunc->setSourceAlpha(osg::BlendFunc::ONE);
+                        else if (blendFunc->getDestination() == osg::BlendFunc::ONE)
+                            newBlendFunc->setSourceAlpha(osg::BlendFunc::ZERO);
+                        // Other setups barely exist in the wild and aren't worth supporting as they're not equippable gear
+                        else
+                            Log(Debug::Info) << "Unable to adjust blend mode for character preview. Source factor 0x" << std::hex << blendFunc->getSource() << ", destination factor 0x" << blendFunc->getDestination() << std::dec;
+                    }
                 }
                 if (stateset->getMode(GL_BLEND) & osg::StateAttribute::ON)
                 {
+                    if (!newStateSet)
+                    {
+                        newStateSet = new osg::StateSet(*stateset, osg::CopyOp::SHALLOW_COPY);
+                        node.setStateSet(newStateSet);
+                    }
                     // Disable noBlendAlphaEnv
-                    stateset->setTextureMode(7, GL_TEXTURE_2D, osg::StateAttribute::OFF);
-                    stateset->addUniform(mNoAlphaUniform);
+                    newStateSet->setTextureMode(7, GL_TEXTURE_2D, osg::StateAttribute::OFF);
+                    newStateSet->addUniform(mNoAlphaUniform);
                 }
             }
             traverse(node);
@@ -134,6 +153,7 @@ namespace MWRender
         mTexture->setInternalFormat(GL_RGBA);
         mTexture->setFilter(osg::Texture::MIN_FILTER, osg::Texture::LINEAR);
         mTexture->setFilter(osg::Texture::MAG_FILTER, osg::Texture::LINEAR);
+        mTexture->setUserValue("premultiplied alpha", true);
 
         mCamera = new osg::Camera;
         // hints that the camera is not relative to the master camera

components/myguiplatform/myguirendermanager.cpp

@@ -14,6 +14,8 @@
 
 #include <components/resource/imagemanager.hpp>
 
+#include <components/debug/debuglog.hpp>
+
 #include "myguicompat.h"
 #include "myguitexture.hpp"
 
@@ -439,14 +441,23 @@ void RenderManager::doRender(MyGUI::IVertexBuffer *buffer, MyGUI::ITexture *text
     batch.mVertexBuffer = static_cast<OSGVertexBuffer*>(buffer)->getVertexBuffer();
     batch.mArray = static_cast<OSGVertexBuffer*>(buffer)->getVertexArray();
     static_cast<OSGVertexBuffer*>(buffer)->markUsed();
+    bool premultipliedAlpha = false;
     if (texture)
     {
         batch.mTexture = static_cast<OSGTexture*>(texture)->getTexture();
         if (batch.mTexture->getDataVariance() == osg::Object::DYNAMIC)
             mDrawable->setDataVariance(osg::Object::DYNAMIC); // only for this frame, reset in begin()
+        batch.mTexture->getUserValue("premultiplied alpha", premultipliedAlpha);
     }
     if (mInjectState)
         batch.mStateSet = mInjectState;
+    else if (premultipliedAlpha)
+    {
+        // This is hacky, but MyGUI made it impossible to use a custom layer for a nested node, so state couldn't be injected 'properly'
+        osg::ref_ptr<osg::StateSet> stateSet = new osg::StateSet();
+        stateSet->setAttribute(new osg::BlendFunc(osg::BlendFunc::ONE, osg::BlendFunc::ONE_MINUS_SRC_ALPHA));
+        batch.mStateSet = stateSet;
+    }
 
     mDrawable->addBatch(batch);
 }