Merge branch 'pythagorastrousers' into 'master'

Support red-green normal maps (#7932)

Closes #7932

See merge request OpenMW/openmw!3983

CHANGELOG.md

@@ -225,6 +225,7 @@
     Feature #7875: Disable MyGUI windows snapping
     Feature #7914: Do not allow to move GUI windows out of screen
     Feature #7923: Don't show non-existent higher ranks for factions with fewer than 9 ranks
+    Feature #7932: Support two-channel normal maps
     Task #5896: Do not use deprecated MyGUI properties
     Task #6085: Replace boost::filesystem with std::filesystem
     Task #6149: Dehardcode Lua API_REVISION

components/resource/imagemanager.cpp

@@ -78,8 +78,7 @@ namespace Resource
                 }
                 break;
             }
-            // not bothering with checks for other compression formats right now, we are unlikely to ever use those
-            // anyway
+            // not bothering with checks for other compression formats right now
             default:
                 return true;
         }

components/sceneutil/util.cpp

@@ -286,4 +286,125 @@ namespace SceneUtil
         mOperationQueue->add(operation);
     }
 
+    GLenum computeUnsizedPixelFormat(GLenum format)
+    {
+        switch (format)
+        {
+            // Try compressed formats first, they're more likely to be used
+
+            // Generic
+            case GL_COMPRESSED_ALPHA_ARB:
+                return GL_ALPHA;
+            case GL_COMPRESSED_INTENSITY_ARB:
+                return GL_INTENSITY;
+            case GL_COMPRESSED_LUMINANCE_ALPHA_ARB:
+                return GL_LUMINANCE_ALPHA;
+            case GL_COMPRESSED_LUMINANCE_ARB:
+                return GL_LUMINANCE;
+            case GL_COMPRESSED_RGB_ARB:
+                return GL_RGB;
+            case GL_COMPRESSED_RGBA_ARB:
+                return GL_RGBA;
+
+            // S3TC
+            case GL_COMPRESSED_RGB_S3TC_DXT1_EXT:
+            case GL_COMPRESSED_SRGB_S3TC_DXT1_EXT:
+                return GL_RGB;
+            case GL_COMPRESSED_RGBA_S3TC_DXT1_EXT:
+            case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT:
+            case GL_COMPRESSED_RGBA_S3TC_DXT3_EXT:
+            case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT:
+            case GL_COMPRESSED_RGBA_S3TC_DXT5_EXT:
+            case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT:
+                return GL_RGBA;
+
+            // RGTC
+            case GL_COMPRESSED_RED_RGTC1_EXT:
+            case GL_COMPRESSED_SIGNED_RED_RGTC1_EXT:
+                return GL_RED;
+            case GL_COMPRESSED_RED_GREEN_RGTC2_EXT:
+            case GL_COMPRESSED_SIGNED_RED_GREEN_RGTC2_EXT:
+                return GL_RG;
+
+            // PVRTC
+            case GL_COMPRESSED_RGB_PVRTC_4BPPV1_IMG:
+            case GL_COMPRESSED_RGB_PVRTC_2BPPV1_IMG:
+                return GL_RGB;
+            case GL_COMPRESSED_RGBA_PVRTC_4BPPV1_IMG:
+            case GL_COMPRESSED_RGBA_PVRTC_2BPPV1_IMG:
+                return GL_RGBA;
+
+            // ETC
+            case GL_COMPRESSED_R11_EAC:
+            case GL_COMPRESSED_SIGNED_R11_EAC:
+                return GL_RED;
+            case GL_COMPRESSED_RG11_EAC:
+            case GL_COMPRESSED_SIGNED_RG11_EAC:
+                return GL_RG;
+            case GL_ETC1_RGB8_OES:
+            case GL_COMPRESSED_RGB8_ETC2:
+            case GL_COMPRESSED_SRGB8_ETC2:
+                return GL_RGB;
+            case GL_COMPRESSED_RGB8_PUNCHTHROUGH_ALPHA1_ETC2:
+            case GL_COMPRESSED_SRGB8_PUNCHTHROUGH_ALPHA1_ETC2:
+            case GL_COMPRESSED_RGBA8_ETC2_EAC:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ETC2_EAC:
+                return GL_RGBA;
+
+            // ASTC
+            case GL_COMPRESSED_RGBA_ASTC_4x4_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_5x4_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_5x5_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_6x5_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_6x6_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_8x5_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_8x6_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_8x8_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_10x5_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_10x6_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_10x8_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_10x10_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_12x10_KHR:
+            case GL_COMPRESSED_RGBA_ASTC_12x12_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_4x4_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_5x4_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_5x5_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_6x5_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_6x6_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_8x5_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_8x6_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_8x8_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_10x5_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_10x6_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_10x8_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_10x10_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_12x10_KHR:
+            case GL_COMPRESSED_SRGB8_ALPHA8_ASTC_12x12_KHR:
+                return GL_RGBA;
+
+            // Plug in some holes computePixelFormat has, you never know when these could come in handy
+            case GL_INTENSITY4:
+            case GL_INTENSITY8:
+            case GL_INTENSITY12:
+            case GL_INTENSITY16:
+                return GL_INTENSITY;
+
+            case GL_LUMINANCE4:
+            case GL_LUMINANCE8:
+            case GL_LUMINANCE12:
+            case GL_LUMINANCE16:
+                return GL_LUMINANCE;
+
+            case GL_LUMINANCE4_ALPHA4:
+            case GL_LUMINANCE6_ALPHA2:
+            case GL_LUMINANCE8_ALPHA8:
+            case GL_LUMINANCE12_ALPHA4:
+            case GL_LUMINANCE12_ALPHA12:
+            case GL_LUMINANCE16_ALPHA16:
+                return GL_LUMINANCE_ALPHA;
+        }
+
+        return osg::Image::computePixelFormat(format);
+    }
+
 }

components/sceneutil/util.hpp

@@ -112,6 +112,10 @@ namespace SceneUtil
     protected:
         osg::ref_ptr<osg::OperationQueue> mOperationQueue;
     };
+
+    // Compute the unsized format equivalent to the given pixel format
+    // Unlike osg::Image::computePixelFormat, this also covers compressed formats
+    GLenum computeUnsizedPixelFormat(GLenum format);
 }
 
 #endif

components/shader/shadervisitor.cpp

@@ -26,6 +26,7 @@
 #include <components/sceneutil/morphgeometry.hpp>
 #include <components/sceneutil/riggeometry.hpp>
 #include <components/sceneutil/riggeometryosgaextension.hpp>
+#include <components/sceneutil/util.hpp>
 #include <components/settings/settings.hpp>
 #include <components/stereo/stereomanager.hpp>
 #include <components/vfs/manager.hpp>
@@ -184,6 +185,7 @@ namespace Shader
         , mAdditiveBlending(false)
         , mDiffuseHeight(false)
         , mNormalHeight(false)
+        , mReconstructNormalZ(false)
         , mTexStageRequiringTangents(-1)
         , mSoftParticles(false)
         , mNode(nullptr)
@@ -429,6 +431,7 @@ namespace Shader
                     normalMapTex->setFilter(osg::Texture::MAG_FILTER, diffuseMap->getFilter(osg::Texture::MAG_FILTER));
                     normalMapTex->setMaxAnisotropy(diffuseMap->getMaxAnisotropy());
                     normalMapTex->setName("normalMap");
+                    normalMap = normalMapTex;
 
                     int unit = texAttributes.size();
                     if (!writableStateSet)
@@ -440,6 +443,21 @@ namespace Shader
                     mRequirements.back().mNormalHeight = normalHeight;
                 }
             }
+
+            if (normalMap != nullptr && normalMap->getImage(0))
+            {
+                // Special handling for red-green normal maps (e.g. BC5 or R8G8)
+                switch (SceneUtil::computeUnsizedPixelFormat(normalMap->getImage(0)->getPixelFormat()))
+                {
+                    case GL_RG:
+                    case GL_RG_INTEGER:
+                    {
+                        mRequirements.back().mReconstructNormalZ = true;
+                        mRequirements.back().mNormalHeight = false;
+                    }
+                }
+            }
+
             if (mAutoUseSpecularMaps && diffuseMap != nullptr && specularMap == nullptr && diffuseMap->getImage(0))
             {
                 std::string specularMapFileName = diffuseMap->getImage(0)->getFileName();
@@ -629,6 +647,7 @@ namespace Shader
 
         defineMap["diffuseParallax"] = reqs.mDiffuseHeight ? "1" : "0";
         defineMap["parallax"] = reqs.mNormalHeight ? "1" : "0";
+        defineMap["reconstructNormalZ"] = reqs.mReconstructNormalZ ? "1" : "0";
 
         writableStateSet->addUniform(new osg::Uniform("colorMode", reqs.mColorMode));
         addedState->addUniform("colorMode");

components/shader/shadervisitor.hpp

@@ -110,6 +110,7 @@ namespace Shader
 
             bool mDiffuseHeight; // true if diffuse map has height info in alpha channel
             bool mNormalHeight; // true if normal map has height info in alpha channel
+            bool mReconstructNormalZ; // used for red-green normal maps (e.g. BC5)
 
             // -1 == no tangents required
             int mTexStageRequiringTangents;

components/terrain/material.cpp

@@ -10,6 +10,7 @@
 
 #include <components/resource/scenemanager.hpp>
 #include <components/sceneutil/depth.hpp>
+#include <components/sceneutil/util.hpp>
 #include <components/shader/shadermanager.hpp>
 #include <components/stereo/stereomanager.hpp>
 
@@ -271,18 +272,37 @@ namespace Terrain
                     stateset->addUniform(UniformCollection::value().mBlendMap);
                 }
 
+                bool parallax = it->mNormalMap && it->mParallax;
+                bool reconstructNormalZ = false;
+
                 if (it->mNormalMap)
                 {
                     stateset->setTextureAttributeAndModes(2, it->mNormalMap);
                     stateset->addUniform(UniformCollection::value().mNormalMap);
+
+                    // Special handling for red-green normal maps (e.g. BC5 or R8G8).
+                    const osg::Image* image = it->mNormalMap->getImage(0);
+                    if (image)
+                    {
+                        switch (SceneUtil::computeUnsizedPixelFormat(image->getPixelFormat()))
+                        {
+                            case GL_RG:
+                            case GL_RG_INTEGER:
+                            {
+                                reconstructNormalZ = true;
+                                parallax = false;
+                            }
+                        }
+                    }
                 }
 
                 Shader::ShaderManager::DefineMap defineMap;
                 defineMap["normalMap"] = (it->mNormalMap) ? "1" : "0";
                 defineMap["blendMap"] = (!blendmaps.empty()) ? "1" : "0";
                 defineMap["specularMap"] = it->mSpecular ? "1" : "0";
-                defineMap["parallax"] = (it->mNormalMap && it->mParallax) ? "1" : "0";
+                defineMap["parallax"] = parallax ? "1" : "0";
                 defineMap["writeNormals"] = (it == layers.end() - 1) ? "1" : "0";
+                defineMap["reconstructNormalZ"] = reconstructNormalZ ? "1" : "0";
                 Stereo::shaderStereoDefines(defineMap);
 
                 stateset->setAttributeAndModes(shaderManager.getProgram("terrain", defineMap));

docs/source/reference/modding/texture-modding/texture-basics.rst

@@ -25,6 +25,19 @@ Content creators need to know that OpenMW uses the DX format for normal maps, an
 
 See the section `Automatic use`_ further down below for detailed information.
 
+The RGB channels of the normal map are used to store XYZ components of tangent space normals and the alpha channel of the normal map may be used to store a height map used for parallax.
+
+This is different from the setup used in Bethesda games that use the traditional pipeline, which may store specular information in the alpha channel.
+
+Special pixel formats that only store two color channels exist and are used by Bethesda games that employ a PBR-based pipeline. Compressed red-green formats are optimized for use with normal maps and suffer from far less quality degradation than S3TC-compressed normal maps of equivalent size.
+
+OpenMW supports the use of such pixel formats. When a red-green normal map is provided, the Z component of the normal will be reconstructed based on XY components it stores.
+Naturally, since these formats cannot provide an alpha channel, they do not support parallax.
+
+Keep in mind, however, that while the necessary hardware support is widespread for compressed red-green formats, it is less ubiquitous than the support for S3TC family of compressed formats.
+Should you run into the consequences of this, you might want to convert such textures into an uncompressed red-green format such as R8G8.
+Be careful not to try and convert such textures into a full-color format as the previously non-existent blue channel would then be used.
+
 Specular Mapping
 ################
 

files/shaders/compatibility/bs/default.frag

@@ -77,6 +77,9 @@ void main()
     vec3 specularColor = getSpecularColor().xyz;
 #if @normalMap
     vec4 normalTex = texture2D(normalMap, normalMapUV);
+#if @reconstructNormalZ
+    normalTex.z = sqrt(1.0 - dot(normalTex.xy, normalTex.xy));
+#endif
     vec3 viewNormal = normalToView(normalTex.xyz * 2.0 - 1.0);
     specularColor *= normalTex.a;
 #else

files/shaders/compatibility/groundcover.frag

@@ -59,7 +59,11 @@ void main()
     gl_FragData[0].a = alphaTest(gl_FragData[0].a, alphaRef);
 
 #if @normalMap
-    vec3 viewNormal = normalToView(texture2D(normalMap, normalMapUV).xyz * 2.0 - 1.0);
+    vec4 normalTex = texture2D(normalMap, normalMapUV);
+#if @reconstructNormalZ
+    normalTex.z = sqrt(1.0 - dot(normalTex.xy, normalTex.xy));
+#endif
+    vec3 viewNormal = normalToView(normalTex.xyz * 2.0 - 1.0);
 #else
     vec3 viewNormal = normalToView(normalize(passNormal));
 #endif

files/shaders/compatibility/objects.frag

@@ -167,7 +167,11 @@ vec2 screenCoords = gl_FragCoord.xy / screenRes;
     gl_FragData[0].a = alphaTest(gl_FragData[0].a, alphaRef);
 
 #if @normalMap
-    vec3 viewNormal = normalToView(texture2D(normalMap, normalMapUV + offset).xyz * 2.0 - 1.0);
+    vec4 normalTex = texture2D(normalMap, normalMapUV + offset);
+#if @reconstructNormalZ
+    normalTex.z = sqrt(1.0 - dot(normalTex.xy, normalTex.xy));
+#endif
+    vec3 viewNormal = normalToView(normalTex.xyz * 2.0 - 1.0);
 #else
     vec3 viewNormal = normalize(gl_NormalMatrix * passNormal);
 #endif

files/shaders/compatibility/terrain.frag

@@ -63,7 +63,11 @@ void main()
 #endif
 
 #if @normalMap
-    vec3 viewNormal = normalToView(texture2D(normalMap, adjustedUV).xyz * 2.0 - 1.0);
+    vec4 normalTex = texture2D(normalMap, adjustedUV);
+#if @reconstructNormalZ
+    normalTex.z = sqrt(1.0 - dot(normalTex.xy, normalTex.xy));
+#endif
+    vec3 viewNormal = normalToView(normalTex.xyz * 2.0 - 1.0);
 #else
     vec3 viewNormal = normalize(gl_NormalMatrix * passNormal);
 #endif