Deduplicate decal and alpha handling in NifLoader

components/nifosg/nifloader.cpp

@@ -2209,36 +2209,79 @@ namespace NifOsg
             handleDepthFlags(stateset, material->mDepthTest, material->mDepthWrite);
         }
 
-        void handleShaderMaterialDrawableProperties(
+        void handleDecal(bool hasSortAlpha, osg::ref_ptr<osg::StateSet> stateset)
-            Bgsm::MaterialFilePtr shaderMat, osg::ref_ptr<osg::Material> mat, osg::Node& node, bool& hasSortAlpha)
         {
-            mat->setAlpha(osg::Material::FRONT_AND_BACK, shaderMat->mTransparency);
+            osg::ref_ptr<osg::PolygonOffset> polygonOffset(new osg::PolygonOffset);
-            if (shaderMat->mAlphaTest)
+            polygonOffset->setUnits(SceneUtil::AutoDepth::isReversed() ? 1.f : -1.f);
+            polygonOffset->setFactor(SceneUtil::AutoDepth::isReversed() ? 0.65f : -0.65f);
+            polygonOffset = shareAttribute(polygonOffset);
+            stateset->setAttributeAndModes(polygonOffset, osg::StateAttribute::ON);
+            if (!mPushedSorter && !hasSortAlpha)
+                stateset->setRenderBinDetails(1, "SORT_BACK_TO_FRONT");
+        }
+
+        void handleAlphaTesting(
+            bool enabled, osg::AlphaFunc::ComparisonFunction function, int threshold, osg::Node& node)
+        {
+            if (enabled)
             {
-                osg::StateSet* stateset = node.getOrCreateStateSet();
+                osg::ref_ptr<osg::AlphaFunc> alphaFunc(new osg::AlphaFunc(function, threshold / 255.f));
-                osg::ref_ptr<osg::AlphaFunc> alphaFunc(
-                    new osg::AlphaFunc(osg::AlphaFunc::GREATER, shaderMat->mAlphaTestThreshold / 255.f));
                 alphaFunc = shareAttribute(alphaFunc);
-                stateset->setAttributeAndModes(alphaFunc, osg::StateAttribute::ON);
+                node.getOrCreateStateSet()->setAttributeAndModes(alphaFunc, osg::StateAttribute::ON);
             }
-            if (shaderMat->mAlphaBlend)
+            else if (osg::StateSet* stateset = node.getStateSet())
             {
-                osg::StateSet* stateset = node.getOrCreateStateSet();
+                stateset->removeAttribute(osg::StateAttribute::ALPHAFUNC);
-                osg::ref_ptr<osg::BlendFunc> blendFunc(new osg::BlendFunc(
+                stateset->removeMode(GL_ALPHA_TEST);
-                    getBlendMode(shaderMat->mSourceBlendMode), getBlendMode(shaderMat->mDestinationBlendMode)));
+            }
+        }
+
+        void handleAlphaBlending(
+            bool enabled, int sourceMode, int destMode, bool sort, bool& hasSortAlpha, osg::Node& node)
+        {
+            if (enabled)
+            {
+                osg::ref_ptr<osg::StateSet> stateset = node.getOrCreateStateSet();
+                osg::ref_ptr<osg::BlendFunc> blendFunc(
+                    new osg::BlendFunc(getBlendMode(sourceMode), getBlendMode(destMode)));
+                // on AMD hardware, alpha still seems to be stored with an RGBA framebuffer with OpenGL.
+                // This might be mandated by the OpenGL 2.1 specification section 2.14.9, or might be a bug.
+                // Either way, D3D8.1 doesn't do that, so adapt the destination factor.
+                if (blendFunc->getDestination() == GL_DST_ALPHA)
+                    blendFunc->setDestination(GL_ONE);
                 blendFunc = shareAttribute(blendFunc);
                 stateset->setAttributeAndModes(blendFunc, osg::StateAttribute::ON);
-                hasSortAlpha = true;
+
+                if (sort)
+                {
+                    hasSortAlpha = true;
+                    if (!mPushedSorter)
+                        stateset->setRenderingHint(osg::StateSet::TRANSPARENT_BIN);
+                }
+                else if (!mPushedSorter)
+                {
+                    stateset->setRenderBinToInherit();
+                }
             }
+            else if (osg::ref_ptr<osg::StateSet> stateset = node.getStateSet())
+            {
+                stateset->removeAttribute(osg::StateAttribute::BLENDFUNC);
+                stateset->removeMode(GL_BLEND);
+                if (!mPushedSorter)
+                    stateset->setRenderBinToInherit();
+            }
+        }
+
+        void handleShaderMaterialDrawableProperties(
+            Bgsm::MaterialFilePtr shaderMat, osg::ref_ptr<osg::Material> mat, osg::Node& node, bool& hasSortAlpha)
+        {
+            mat->setAlpha(osg::Material::FRONT_AND_BACK, shaderMat->mTransparency);
+            handleAlphaTesting(shaderMat->mAlphaTest, osg::AlphaFunc::GREATER, shaderMat->mAlphaTestThreshold, node);
+            handleAlphaBlending(shaderMat->mAlphaBlend, shaderMat->mSourceBlendMode, shaderMat->mDestinationBlendMode,
+                true, hasSortAlpha, node);
             if (shaderMat->mDecal)
             {
-                osg::StateSet* stateset = node.getOrCreateStateSet();
+                handleDecal(hasSortAlpha, node.getOrCreateStateSet());
-                if (!mPushedSorter && !hasSortAlpha)
-                    stateset->setRenderBinDetails(1, "SORT_BACK_TO_FRONT");
-                osg::ref_ptr<osg::PolygonOffset> polygonOffset(new osg::PolygonOffset);
-                polygonOffset->setUnits(SceneUtil::AutoDepth::isReversed() ? 1.f : -1.f);
-                polygonOffset->setFactor(SceneUtil::AutoDepth::isReversed() ? 0.65f : -0.65f);
-                stateset->setAttributeAndModes(polygonOffset, osg::StateAttribute::ON);
             }
             if (shaderMat->mShaderType == Bgsm::ShaderType::Lighting)
             {
@@ -2627,12 +2670,9 @@ namespace NifOsg
 
             bool hasMatCtrl = false;
             bool hasSortAlpha = false;
-            osg::StateSet* blendFuncStateSet = nullptr;
 
-            auto setBin_Transparent = [](osg::StateSet* ss) { ss->setRenderingHint(osg::StateSet::TRANSPARENT_BIN); };
             auto setBin_BackToFront = [](osg::StateSet* ss) { ss->setRenderBinDetails(0, "SORT_BACK_TO_FRONT"); };
             auto setBin_Traversal = [](osg::StateSet* ss) { ss->setRenderBinDetails(2, "TraversalOrderBin"); };
-            auto setBin_Inherit = [](osg::StateSet* ss) { ss->setRenderBinToInherit(); };
 
             auto lightmode = Nif::NiVertexColorProperty::LightMode::LightMode_EmiAmbDif;
             float emissiveMult = 1.f;
@@ -2718,52 +2758,10 @@ namespace NifOsg
                     case Nif::RC_NiAlphaProperty:
                     {
                         const Nif::NiAlphaProperty* alphaprop = static_cast<const Nif::NiAlphaProperty*>(property);
-                        if (alphaprop->useAlphaBlending())
+                        handleAlphaBlending(alphaprop->useAlphaBlending(), alphaprop->sourceBlendMode(),
-                        {
+                            alphaprop->destinationBlendMode(), !alphaprop->noSorter(), hasSortAlpha, *node);
-                            osg::ref_ptr<osg::BlendFunc> blendFunc(
+                        handleAlphaTesting(alphaprop->useAlphaTesting(), getTestMode(alphaprop->alphaTestMode()),
-                                new osg::BlendFunc(getBlendMode(alphaprop->sourceBlendMode()),
+                            alphaprop->mThreshold, *node);
-                                    getBlendMode(alphaprop->destinationBlendMode())));
-                            // on AMD hardware, alpha still seems to be stored with an RGBA framebuffer with OpenGL.
-                            // This might be mandated by the OpenGL 2.1 specification section 2.14.9, or might be a bug.
-                            // Either way, D3D8.1 doesn't do that, so adapt the destination factor.
-                            if (blendFunc->getDestination() == GL_DST_ALPHA)
-                                blendFunc->setDestination(GL_ONE);
-                            blendFunc = shareAttribute(blendFunc);
-                            node->getOrCreateStateSet()->setAttributeAndModes(blendFunc, osg::StateAttribute::ON);
-
-                            if (!alphaprop->noSorter())
-                            {
-                                hasSortAlpha = true;
-                                if (!mPushedSorter)
-                                    setBin_Transparent(node->getStateSet());
-                            }
-                            else
-                            {
-                                if (!mPushedSorter)
-                                    setBin_Inherit(node->getStateSet());
-                            }
-                        }
-                        else if (osg::StateSet* stateset = node->getStateSet())
-                        {
-                            stateset->removeAttribute(osg::StateAttribute::BLENDFUNC);
-                            stateset->removeMode(GL_BLEND);
-                            blendFuncStateSet = stateset;
-                            if (!mPushedSorter)
-                                blendFuncStateSet->setRenderBinToInherit();
-                        }
-
-                        if (alphaprop->useAlphaTesting())
-                        {
-                            osg::ref_ptr<osg::AlphaFunc> alphaFunc(new osg::AlphaFunc(
-                                getTestMode(alphaprop->alphaTestMode()), alphaprop->mThreshold / 255.f));
-                            alphaFunc = shareAttribute(alphaFunc);
-                            node->getOrCreateStateSet()->setAttributeAndModes(alphaFunc, osg::StateAttribute::ON);
-                        }
-                        else if (osg::StateSet* stateset = node->getStateSet())
-                        {
-                            stateset->removeAttribute(osg::StateAttribute::ALPHAFUNC);
-                            stateset->removeMode(GL_ALPHA_TEST);
-                        }
                         break;
                     }
                     case Nif::RC_BSShaderPPLightingProperty:
@@ -2778,8 +2776,6 @@ namespace NifOsg
                         if (Bgsm::MaterialFilePtr shaderMat = getShaderMaterial(shaderprop->mName))
                         {
                             handleShaderMaterialDrawableProperties(shaderMat, mat, *node, hasSortAlpha);
-                            if (shaderMat->mAlphaBlend && !mPushedSorter)
-                                setBin_Transparent(node->getOrCreateStateSet());
                             if (shaderMat->mShaderType == Bgsm::ShaderType::Lighting)
                             {
                                 auto bgsm = static_cast<const Bgsm::BGSMFile*>(shaderMat.get());
@@ -2799,15 +2795,7 @@ namespace NifOsg
                         specStrength = shaderprop->mSpecStrength;
                         specEnabled = shaderprop->specular();
                         if (shaderprop->decal())
-                        {
+                            handleDecal(hasSortAlpha, node->getOrCreateStateSet());
-                            osg::StateSet* stateset = node->getOrCreateStateSet();
-                            if (!mPushedSorter && !hasSortAlpha)
-                                stateset->setRenderBinDetails(1, "SORT_BACK_TO_FRONT");
-                            osg::ref_ptr<osg::PolygonOffset> polygonOffset(new osg::PolygonOffset);
-                            polygonOffset->setUnits(SceneUtil::AutoDepth::isReversed() ? 1.f : -1.f);
-                            polygonOffset->setFactor(SceneUtil::AutoDepth::isReversed() ? 0.65f : -0.65f);
-                            stateset->setAttributeAndModes(polygonOffset, osg::StateAttribute::ON);
-                        }
                         break;
                     }
                     case Nif::RC_BSEffectShaderProperty:
@@ -2816,20 +2804,10 @@ namespace NifOsg
                         if (Bgsm::MaterialFilePtr shaderMat = getShaderMaterial(shaderprop->mName))
                         {
                             handleShaderMaterialDrawableProperties(shaderMat, mat, *node, hasSortAlpha);
-                            if (shaderMat->mAlphaBlend && !mPushedSorter)
-                                setBin_Transparent(node->getOrCreateStateSet());
                             break;
                         }
                         if (shaderprop->decal())
-                        {
+                            handleDecal(hasSortAlpha, node->getOrCreateStateSet());
-                            osg::StateSet* stateset = node->getOrCreateStateSet();
-                            if (!mPushedSorter && !hasSortAlpha)
-                                stateset->setRenderBinDetails(1, "SORT_BACK_TO_FRONT");
-                            osg::ref_ptr<osg::PolygonOffset> polygonOffset(new osg::PolygonOffset);
-                            polygonOffset->setUnits(SceneUtil::AutoDepth::isReversed() ? 1.f : -1.f);
-                            polygonOffset->setFactor(SceneUtil::AutoDepth::isReversed() ? 0.65f : -0.65f);
-                            stateset->setAttributeAndModes(polygonOffset, osg::StateAttribute::ON);
-                        }
                         if (shaderprop->softEffect())
                             SceneUtil::setupSoftEffect(
                                 *node, shaderprop->mFalloffDepth, true, shaderprop->mFalloffDepth);