Untangle normals and parallax handling

Move tangent space generation to the vertex shaders
Support diffuse parallax when no normal map is present
Don't use diffuse parallax if there's no diffuse map
Generalize normal-to-view conversion
Rewrite parallax

components/shader/shadervisitor.cpp

@@ -304,14 +304,6 @@ namespace Shader
         if (node.getUserValue(Misc::OsgUserValues::sXSoftEffect, softEffect) && softEffect)
             mRequirements.back().mSoftParticles = true;
 
-        int applyMode;
-        // Oblivion parallax
-        if (node.getUserValue("applyMode", applyMode) && applyMode == 4)
-        {
-            mRequirements.back().mShaderRequired = true;
-            mRequirements.back().mDiffuseHeight = true;
-        }
-
         // Make sure to disregard any state that came from a previous call to createProgram
         osg::ref_ptr<AddedState> addedState = getAddedState(*stateset);
 
@@ -359,7 +351,17 @@ namespace Shader
                                 normalMap = texture;
                             }
                             else if (texName == "diffuseMap")
+                            {
+                                int applyMode;
+                                // Oblivion parallax
+                                if (node.getUserValue("applyMode", applyMode) && applyMode == 4)
+                                {
+                                    mRequirements.back().mShaderRequired = true;
+                                    mRequirements.back().mDiffuseHeight = true;
+                                    mRequirements.back().mTexStageRequiringTangents = unit;
+                                }
                                 diffuseMap = texture;
+                            }
                             else if (texName == "specularMap")
                                 specularMap = texture;
                             else if (texName == "bumpMap")

files/shaders/CMakeLists.txt

@@ -45,6 +45,7 @@ set(SHADER_FILES
     compatibility/shadowcasting.vert
     compatibility/shadowcasting.frag
     compatibility/vertexcolors.glsl
+    compatibility/normals.glsl
     compatibility/multiview_resolve.vert
     compatibility/multiview_resolve.frag
     compatibility/depthclipped.vert

files/shaders/compatibility/bs/default.frag

@@ -24,7 +24,6 @@ varying vec2 emissiveMapUV;
 #if @normalMap
 uniform sampler2D normalMap;
 varying vec2 normalMapUV;
-varying vec4 passTangent;
 #endif
 
 varying float euclideanDepth;
@@ -46,11 +45,10 @@ uniform bool useTreeAnim;
 #include "compatibility/vertexcolors.glsl"
 #include "compatibility/shadows_fragment.glsl"
 #include "compatibility/fog.glsl"
+#include "compatibility/normals.glsl"
 
 void main()
 {
-    vec3 normal = normalize(passNormal);
-
 #if @diffuseMap
     gl_FragData[0] = texture2D(diffuseMap, diffuseMapUV);
     gl_FragData[0].a *= coveragePreservingAlphaScale(diffuseMap, diffuseMapUV);
@@ -65,15 +63,10 @@ void main()
 
 #if @normalMap
     vec4 normalTex = texture2D(normalMap, normalMapUV);
-
-    vec3 normalizedNormal = normal;
-    vec3 normalizedTangent = normalize(passTangent.xyz);
-    vec3 binormal = cross(normalizedTangent, normalizedNormal) * passTangent.w;
-    mat3 tbnTranspose = mat3(normalizedTangent, binormal, normalizedNormal);
-
-    normal = normalize(tbnTranspose * (normalTex.xyz * 2.0 - 1.0));
+    vec3 viewNormal = normalToView(normalTex.xyz * 2.0 - 1.0);
+#else
+    vec3 viewNormal = normalToView(normalize(passNormal));
 #endif
-    vec3 viewNormal = normalize(gl_NormalMatrix * normal);
 
     float shadowing = unshadowedLightRatio(linearDepth);
     vec3 diffuseLight, ambientLight;

files/shaders/compatibility/bs/default.vert

@@ -36,6 +36,7 @@ varying vec3 passNormal;
 
 #include "compatibility/vertexcolors.glsl"
 #include "compatibility/shadows_vertex.glsl"
+#include "compatibility/normals.glsl"
 
 void main(void)
 {
@@ -45,6 +46,14 @@ void main(void)
     gl_ClipVertex = viewPos;
     euclideanDepth = length(viewPos.xyz);
     linearDepth = getLinearDepth(gl_Position.z, viewPos.z);
+    passColor = gl_Color;
+    passViewPos = viewPos.xyz;
+    passNormal = gl_Normal.xyz;
+    normalToViewMatrix = gl_NormalMatrix;
+
+#if @normalMap
+    normalToViewMatrix *= generateTangentSpace(gl_MultiTexCoord7.xyzw, passNormal);
+#endif
 
 #if @diffuseMap
     diffuseMapUV = (gl_TextureMatrix[@diffuseMapUV] * gl_MultiTexCoord@diffuseMapUV).xy;
@@ -56,15 +65,11 @@ void main(void)
 
 #if @normalMap
     normalMapUV = (gl_TextureMatrix[@normalMapUV] * gl_MultiTexCoord@normalMapUV).xy;
-    passTangent = gl_MultiTexCoord7.xyzw;
 #endif
 
-    passColor = gl_Color;
-    passViewPos = viewPos.xyz;
-    passNormal = gl_Normal.xyz;
 
 #if @shadows_enabled
-    vec3 viewNormal = normalize((gl_NormalMatrix * gl_Normal).xyz);
+    vec3 viewNormal = normalToView(passNormal);
     setupShadowCoords(viewPos, viewNormal);
 #endif
 }

files/shaders/compatibility/groundcover.frag

@@ -18,7 +18,6 @@ varying vec2 diffuseMapUV;
 #if @normalMap
 uniform sampler2D normalMap;
 varying vec2 normalMapUV;
-varying vec4 passTangent;
 #endif
 
 // Other shaders respect forcePPL, but legacy groundcover mods were designed to work with vertex lighting.
@@ -44,23 +43,10 @@ varying vec3 passNormal;
 #include "lib/light/lighting.glsl"
 #include "lib/material/alpha.glsl"
 #include "fog.glsl"
+#include "compatibility/normals.glsl"
 
 void main()
 {
-    vec3 normal = normalize(passNormal);
-
-#if @normalMap
-    vec4 normalTex = texture2D(normalMap, normalMapUV);
-
-    vec3 normalizedNormal = normal;
-    vec3 normalizedTangent = normalize(passTangent.xyz);
-    vec3 binormal = cross(normalizedTangent, normalizedNormal) * passTangent.w;
-    mat3 tbnTranspose = mat3(normalizedTangent, binormal, normalizedNormal);
-
-    normal = normalize(tbnTranspose * (normalTex.xyz * 2.0 - 1.0));
-#endif
-    vec3 viewNormal = normalize(gl_NormalMatrix * normal);
-
 #if @diffuseMap
     gl_FragData[0] = texture2D(diffuseMap, diffuseMapUV);
 #else
@@ -72,6 +58,12 @@ void main()
 
     gl_FragData[0].a = alphaTest(gl_FragData[0].a, alphaRef);
 
+#if @normalMap
+    vec3 viewNormal = normalToView(texture2D(normalMap, normalMapUV).xyz * 2.0 - 1.0);
+#else
+    vec3 viewNormal = normalToView(normalize(passNormal));
+#endif
+
     float shadowing = unshadowedLightRatio(linearDepth);
 
     vec3 lighting;

files/shaders/compatibility/groundcover.vert

@@ -21,7 +21,6 @@ varying vec2 diffuseMapUV;
 
 #if @normalMap
 varying vec2 normalMapUV;
-varying vec4 passTangent;
 #endif
 
 // Other shaders respect forcePPL, but legacy groundcover mods were designed to work with vertex lighting.
@@ -41,6 +40,7 @@ centroid varying vec3 shadowDiffuseLighting;
 varying vec3 passNormal;
 
 #include "shadows_vertex.glsl"
+#include "compatibility/normals.glsl"
 #include "lib/light/lighting.glsl"
 #include "lib/view/depth.glsl"
 
@@ -149,8 +149,14 @@ void main(void)
 
     linearDepth = getLinearDepth(gl_Position.z, viewPos.z);
 
+    passNormal = rotation3(rotation) * gl_Normal.xyz;
+    normalToViewMatrix = gl_NormalMatrix;
+#if @normalMap
+    normalToViewMatrix *= generateTangentSpace(gl_MultiTexCoord7.xyzw * rotation, passNormal);
+#endif
+
 #if (!PER_PIXEL_LIGHTING || @shadows_enabled)
-    vec3 viewNormal = normalize((gl_NormalMatrix * rotation3(rotation) * gl_Normal).xyz);
+    vec3 viewNormal = normalToView(passNormal);
 #endif
 
 #if @diffuseMap
@@ -159,10 +165,8 @@ void main(void)
 
 #if @normalMap
     normalMapUV = (gl_TextureMatrix[@normalMapUV] * gl_MultiTexCoord@normalMapUV).xy;
-    passTangent = gl_MultiTexCoord7.xyzw * rotation;
 #endif
 
-    passNormal = rotation3(rotation) * gl_Normal.xyz;
 #if PER_PIXEL_LIGHTING
     passViewPos = viewPos.xyz;
 #else

files/shaders/compatibility/normals.glsl

@@ -0,0 +1,14 @@
+varying mat3 normalToViewMatrix;
+
+mat3 generateTangentSpace(vec4 tangent, vec3 normal)
+{
+    vec3 normalizedNormal = normalize(normal);
+    vec3 normalizedTangent = normalize(tangent.xyz);
+    vec3 binormal = cross(normalizedTangent, normalizedNormal) * tangent.w;
+    return mat3(normalizedTangent, binormal, normalizedNormal);
+}
+
+vec3 normalToView(vec3 normal)
+{
+    return normalize(normalToViewMatrix * normal);
+}

files/shaders/compatibility/objects.frag

@@ -37,7 +37,6 @@ varying vec2 emissiveMapUV;
 #if @normalMap
 uniform sampler2D normalMap;
 varying vec2 normalMapUV;
-varying vec4 passTangent;
 #endif
 
 #if @envMap
@@ -79,6 +78,9 @@ uniform float emissiveMult;
 uniform float specStrength;
 varying vec3 passViewPos;
 varying vec3 passNormal;
+#if @normalMap || @diffuseParallax
+varying vec4 passTangent;
+#endif
 
 #if @additiveBlending
 #define ADDITIVE_BLENDING
@@ -91,6 +93,7 @@ varying vec3 passNormal;
 #include "fog.glsl"
 #include "vertexcolors.glsl"
 #include "shadows_fragment.glsl"
+#include "compatibility/normals.glsl"
 
 #if @softParticles
 #include "lib/particle/soft.glsl"
@@ -113,62 +116,32 @@ void main()
     applyOcclusionDiscard(orthoDepthMapCoord, texture2D(orthoDepthMap, orthoDepthMapCoord.xy * 0.5 + 0.5).r);
 #endif
 
-    vec3 normal = normalize(passNormal);
-    vec3 viewVec = normalize(passViewPos.xyz);
+    // only offset diffuse and normal maps for now, other textures are more likely to be using a completely different UV set
+    vec2 offset = vec2(0.0);
 
-#if @normalMap
-    vec4 normalTex = texture2D(normalMap, normalMapUV);
-
-    vec3 normalizedNormal = normal;
-    vec3 normalizedTangent = normalize(passTangent.xyz);
-    vec3 binormal = cross(normalizedTangent, normalizedNormal) * passTangent.w;
-    mat3 tbnTranspose = mat3(normalizedTangent, binormal, normalizedNormal);
-
-    normal = normalize(tbnTranspose * (normalTex.xyz * 2.0 - 1.0));
-#endif
-
-#if !@diffuseMap
-    gl_FragData[0] = vec4(1.0);
-#else
-    vec2 adjustedDiffuseUV = diffuseMapUV;
-
-#if @normalMap && (@parallax || @diffuseParallax)
-    vec3 cameraPos = (gl_ModelViewMatrixInverse * vec4(0,0,0,1)).xyz;
-    vec3 objectPos = (gl_ModelViewMatrixInverse * vec4(passViewPos, 1)).xyz;
-    vec3 eyeDir = normalize(cameraPos - objectPos);
+#if @parallax || @diffuseParallax
 #if @parallax
-    float height = normalTex.a;
+    float height = texture2D(normalMap, normalMapUV).a;
     float flipY = (passTangent.w > 0.0) ? -1.f : 1.f;
 #else
     float height = texture2D(diffuseMap, diffuseMapUV).a;
     // FIXME: shouldn't be necessary, but in this path false-positives are common
     float flipY = -1.f;
 #endif
-    vec2 offset = getParallaxOffset(eyeDir, tbnTranspose, height, flipY);
-    adjustedDiffuseUV += offset; // only offset diffuse for now, other textures are more likely to be using a completely different UV set
-
-    // TODO: check not working as the same UV buffer is being bound to different targets
-    // if diffuseMapUV == normalMapUV
-#if 1
-    // fetch a new normal using updated coordinates
-    normalTex = texture2D(normalMap, adjustedDiffuseUV);
-
-    normal = normalize(tbnTranspose * (normalTex.xyz * 2.0 - 1.0));
-#endif
-
+    offset = getParallaxOffset(transpose(normalToViewMatrix) * normalize(-passViewPos), height, flipY);
 #endif
 
-    vec4 diffuseTex = texture2D(diffuseMap, adjustedDiffuseUV);
-    gl_FragData[0].xyz = diffuseTex.xyz;
-#if !@diffuseParallax
-    gl_FragData[0].a = diffuseTex.a * coveragePreservingAlphaScale(diffuseMap, adjustedDiffuseUV);
-#else
+#if @diffuseMap
+    gl_FragData[0] = texture2D(diffuseMap, diffuseMapUV + offset);
+#if @diffuseParallax
     gl_FragData[0].a = 1.0;
+#else
+    gl_FragData[0].a *= coveragePreservingAlphaScale(diffuseMap, diffuseMapUV + offset);
 #endif
+#else
+    gl_FragData[0] = vec4(1.0);
 #endif
 
-    vec3 viewNormal = normalize(gl_NormalMatrix * normal);
-
     vec4 diffuseColor = getDiffuseColor();
     gl_FragData[0].a *= diffuseColor.a;
 
@@ -179,6 +152,14 @@ void main()
 
     gl_FragData[0].a = alphaTest(gl_FragData[0].a, alphaRef);
 
+#if @normalMap
+    vec3 viewNormal = normalToView(texture2D(normalMap, normalMapUV + offset).xyz * 2.0 - 1.0);
+#else
+    vec3 viewNormal = normalToView(normalize(passNormal));
+#endif
+
+    vec3 viewVec = normalize(passViewPos);
+
 #if @detailMap
     gl_FragData[0].xyz *= texture2D(detailMap, detailMapUV).xyz * 2.0;
 #endif

files/shaders/compatibility/objects.vert

@@ -31,7 +31,6 @@ varying vec2 emissiveMapUV;
 
 #if @normalMap
 varying vec2 normalMapUV;
-varying vec4 passTangent;
 #endif
 
 #if @envMap
@@ -59,9 +58,13 @@ uniform float emissiveMult;
 #endif
 varying vec3 passViewPos;
 varying vec3 passNormal;
+#if @normalMap || @diffuseParallax
+varying vec4 passTangent;
+#endif
 
 #include "vertexcolors.glsl"
 #include "shadows_vertex.glsl"
+#include "compatibility/normals.glsl"
 
 #include "lib/light/lighting.glsl"
 #include "lib/view/depth.glsl"
@@ -84,9 +87,18 @@ void main(void)
 
     vec4 viewPos = modelToView(gl_Vertex);
     gl_ClipVertex = viewPos;
+    passColor = gl_Color;
+    passViewPos = viewPos.xyz;
+    passNormal = gl_Normal.xyz;
+    normalToViewMatrix = gl_NormalMatrix;
+
+#if @normalMap || @diffuseParallax
+    passTangent = gl_MultiTexCoord7.xyzw;
+    normalToViewMatrix *= generateTangentSpace(passTangent, passNormal);
+#endif
 
-#if (@envMap || !PER_PIXEL_LIGHTING || @shadows_enabled)
-    vec3 viewNormal = normalize((gl_NormalMatrix * gl_Normal).xyz);
+#if @envMap || !PER_PIXEL_LIGHTING || @shadows_enabled
+    vec3 viewNormal = normalToView(passNormal);
 #endif
 
 #if @envMap
@@ -118,7 +130,6 @@ void main(void)
 
 #if @normalMap
     normalMapUV = (gl_TextureMatrix[@normalMapUV] * gl_MultiTexCoord@normalMapUV).xy;
-    passTangent = gl_MultiTexCoord7.xyzw;
 #endif
 
 #if @bumpMap
@@ -133,10 +144,6 @@ void main(void)
     glossMapUV = (gl_TextureMatrix[@glossMapUV] * gl_MultiTexCoord@glossMapUV).xy;
 #endif
 
-    passColor = gl_Color;
-    passViewPos = viewPos.xyz;
-    passNormal = gl_Normal.xyz;
-
 #if !PER_PIXEL_LIGHTING
     vec3 diffuseLight, ambientLight;
     doLighting(viewPos.xyz, viewNormal, diffuseLight, ambientLight, shadowDiffuseLighting);

files/shaders/compatibility/terrain.frag

@@ -40,49 +40,31 @@ uniform float far;
 #include "lib/light/lighting.glsl"
 #include "lib/material/parallax.glsl"
 #include "fog.glsl"
+#include "compatibility/normals.glsl"
 
 void main()
 {
     vec2 adjustedUV = (gl_TextureMatrix[0] * vec4(uv, 0.0, 1.0)).xy;
 
-    vec3 normal = normalize(passNormal);
-
-#if @normalMap
-    vec4 normalTex = texture2D(normalMap, adjustedUV);
-
-    vec3 normalizedNormal = normal;
-    vec3 tangent = vec3(1.0, 0.0, 0.0);
-    vec3 binormal = normalize(cross(tangent, normalizedNormal));
-    tangent = normalize(cross(normalizedNormal, binormal)); // note, now we need to re-cross to derive tangent again because it wasn't orthonormal
-    mat3 tbnTranspose = mat3(tangent, binormal, normalizedNormal);
-
-    normal = tbnTranspose * (normalTex.xyz * 2.0 - 1.0);
-#endif
-
 #if @parallax
-    vec3 cameraPos = (gl_ModelViewMatrixInverse * vec4(0,0,0,1)).xyz;
-    vec3 objectPos = (gl_ModelViewMatrixInverse * vec4(passViewPos, 1)).xyz;
-    vec3 eyeDir = normalize(cameraPos - objectPos);
-    adjustedUV += getParallaxOffset(eyeDir, tbnTranspose, normalTex.a, 1.f);
-
-    // update normal using new coordinates
-    normalTex = texture2D(normalMap, adjustedUV);
-
-    normal = tbnTranspose * (normalTex.xyz * 2.0 - 1.0);
+    adjustedUV += getParallaxOffset(transpose(normalToViewMatrix) * normalize(-passViewPos), texture2D(normalMap, adjustedUV).a, 1.f);
 #endif
-
-    vec3 viewNormal = normalize(gl_NormalMatrix * normal);
-
     vec4 diffuseTex = texture2D(diffuseMap, adjustedUV);
     gl_FragData[0] = vec4(diffuseTex.xyz, 1.0);
 
+    vec4 diffuseColor = getDiffuseColor();
+    gl_FragData[0].a *= diffuseColor.a;
+
 #if @blendMap
     vec2 blendMapUV = (gl_TextureMatrix[1] * vec4(uv, 0.0, 1.0)).xy;
     gl_FragData[0].a *= texture2D(blendMap, blendMapUV).a;
 #endif
 
-    vec4 diffuseColor = getDiffuseColor();
-    gl_FragData[0].a *= diffuseColor.a;
+#if @normalMap
+    vec3 viewNormal = normalToView(texture2D(normalMap, adjustedUV).xyz * 2.0 - 1.0);
+#else
+    vec3 viewNormal = normalToView(normalize(passNormal));
+#endif
 
     float shadowing = unshadowedLightRatio(linearDepth);
     vec3 lighting;

files/shaders/compatibility/terrain.vert

@@ -24,6 +24,7 @@ varying vec3 passNormal;
 
 #include "vertexcolors.glsl"
 #include "shadows_vertex.glsl"
+#include "compatibility/normals.glsl"
 
 #include "lib/light/lighting.glsl"
 #include "lib/view/depth.glsl"
@@ -37,13 +38,20 @@ void main(void)
     euclideanDepth = length(viewPos.xyz);
     linearDepth = getLinearDepth(gl_Position.z, viewPos.z);
 
-#if (!PER_PIXEL_LIGHTING || @shadows_enabled)
-    vec3 viewNormal = normalize((gl_NormalMatrix * gl_Normal).xyz);
-#endif
-
     passColor = gl_Color;
     passNormal = gl_Normal.xyz;
     passViewPos = viewPos.xyz;
+    normalToViewMatrix = gl_NormalMatrix;
+
+#if @normalMap
+    mat3 tbnMatrix = generateTangentSpace(vec4(1.0, 0.0, 0.0, 1.0), passNormal);
+    tbnMatrix[0] = normalize(cross(tbnMatrix[2], tbnMatrix[1])); // note, now we need to re-cross to derive tangent again because it wasn't orthonormal
+    normalToViewMatrix *= tbnMatrix;
+#endif
+
+#if !PER_PIXEL_LIGHTING || @shadows_enabled
+    vec3 viewNormal = normalToView(passNormal);
+#endif
 
 #if !PER_PIXEL_LIGHTING
     vec3 diffuseLight, ambientLight;

files/shaders/lib/material/parallax.glsl

@@ -4,10 +4,9 @@
 #define PARALLAX_SCALE 0.04
 #define PARALLAX_BIAS -0.02
 
-vec2 getParallaxOffset(vec3 eyeDir, mat3 tbnTranspose, float height, float flipY)
+vec2 getParallaxOffset(vec3 eyeDir, float height, float flipY)
 {
-    vec3 TSeyeDir = normalize(eyeDir * tbnTranspose);
-    return vec2(TSeyeDir.x, TSeyeDir.y * flipY) * ( height * PARALLAX_SCALE + PARALLAX_BIAS );
+    return vec2(eyeDir.x, eyeDir.y * flipY) * ( height * PARALLAX_SCALE + PARALLAX_BIAS );
 }
 
-#endif
+#endif