From 09759c66208d61d2bad0e85d7a0438fe3767a93a Mon Sep 17 00:00:00 2001
From: Alexei Dobrohotov <alexdobrohotov@yandex.ru>
Date: Sat, 19 Dec 2020 20:17:42 +0300
Subject: [PATCH] Clean up shader lighting

---
 files/shaders/CMakeLists.txt        |  1 +
 files/shaders/lighting.glsl         | 83 +++++++----------------------
 files/shaders/objects_fragment.glsl | 29 +++++-----
 files/shaders/objects_vertex.glsl   | 14 +++--
 files/shaders/terrain_fragment.glsl | 29 +++++-----
 files/shaders/terrain_vertex.glsl   | 14 +++--
 files/shaders/vertexcolors.glsl     | 38 +++++++++++++
 7 files changed, 107 insertions(+), 101 deletions(-)
 create mode 100644 files/shaders/vertexcolors.glsl

diff --git a/files/shaders/CMakeLists.txt b/files/shaders/CMakeLists.txt
index 8012c2bc1..47670e7a0 100644
--- a/files/shaders/CMakeLists.txt
+++ b/files/shaders/CMakeLists.txt
@@ -22,6 +22,7 @@ set(SHADER_FILES
     shadows_fragment.glsl
     shadowcasting_vertex.glsl
     shadowcasting_fragment.glsl
+    vertexcolors.glsl
 )
 
 copy_all_resource_files(${CMAKE_CURRENT_SOURCE_DIR} ${OPENMW_SHADERS_ROOT} ${DDIRRELATIVE} "${SHADER_FILES}")
diff --git a/files/shaders/lighting.glsl b/files/shaders/lighting.glsl
index 1ed162eea..930f4de26 100644
--- a/files/shaders/lighting.glsl
+++ b/files/shaders/lighting.glsl
@@ -1,85 +1,38 @@
 #define MAX_LIGHTS 8
 
-uniform int colorMode;
-
-const int ColorMode_None = 0;
-const int ColorMode_Emission = 1;
-const int ColorMode_AmbientAndDiffuse = 2;
-const int ColorMode_Ambient = 3;
-const int ColorMode_Diffuse = 4;
-const int ColorMode_Specular = 5;
-
-void perLight(out vec3 ambientOut, out vec3 diffuseOut, int lightIndex, vec3 viewPos, vec3 viewNormal, vec4 diffuse, vec3 ambient)
+void perLight(out vec3 ambientOut, out vec3 diffuseOut, int lightIndex, vec3 viewPos, vec3 viewNormal)
 {
-    vec3 lightDir;
-    float lightDistance;
-
-    lightDir = gl_LightSource[lightIndex].position.xyz - (viewPos.xyz * gl_LightSource[lightIndex].position.w);
-    lightDistance = length(lightDir);
+    vec3 lightDir = gl_LightSource[lightIndex].position.xyz - viewPos * gl_LightSource[lightIndex].position.w;
+    float lightDistance = length(lightDir);
     lightDir = normalize(lightDir);
     float illumination = clamp(1.0 / (gl_LightSource[lightIndex].constantAttenuation + gl_LightSource[lightIndex].linearAttenuation * lightDistance + gl_LightSource[lightIndex].quadraticAttenuation * lightDistance * lightDistance), 0.0, 1.0);
 
-    ambientOut = ambient * gl_LightSource[lightIndex].ambient.xyz * illumination;
-    diffuseOut = diffuse.xyz * gl_LightSource[lightIndex].diffuse.xyz * max(dot(viewNormal.xyz, lightDir), 0.0) * illumination;
+    ambientOut = gl_LightSource[lightIndex].ambient.xyz * illumination;
+    diffuseOut = gl_LightSource[lightIndex].diffuse.xyz * max(dot(viewNormal, lightDir), 0.0) * illumination;
 }
 
 #if PER_PIXEL_LIGHTING
-vec4 doLighting(vec3 viewPos, vec3 viewNormal, vec4 vertexColor, float shadowing)
+void doLighting(vec3 viewPos, vec3 viewNormal, float shadowing, out vec3 diffuseLight, out vec3 ambientLight)
 #else
-vec4 doLighting(vec3 viewPos, vec3 viewNormal, vec4 vertexColor, out vec3 shadowDiffuse)
+void doLighting(vec3 viewPos, vec3 viewNormal, out vec3 diffuseLight, out vec3 ambientLight, out vec3 shadowDiffuse)
 #endif
 {
-    vec4 diffuse;
-    vec3 ambient;
-    if (colorMode == ColorMode_AmbientAndDiffuse)
-    {
-        diffuse = vertexColor;
-        ambient = vertexColor.xyz;
-    }
-    else if (colorMode == ColorMode_Diffuse)
-    {
-        diffuse = vertexColor;
-        ambient = gl_FrontMaterial.ambient.xyz;
-    }
-    else if (colorMode == ColorMode_Ambient)
-    {
-        diffuse = gl_FrontMaterial.diffuse;
-        ambient = vertexColor.xyz;
-    }
-    else
-    {
-        diffuse = gl_FrontMaterial.diffuse;
-        ambient = gl_FrontMaterial.ambient.xyz;
-    }
-    vec4 lightResult = vec4(0.0, 0.0, 0.0, diffuse.a);
-
-    vec3 diffuseLight, ambientLight;
-    perLight(ambientLight, diffuseLight, 0, viewPos, viewNormal, diffuse, ambient);
+    vec3 ambientOut, diffuseOut;
+    // This light gets added a second time in the loop to fix Mesa users' slowdown, so we need to negate its contribution here.
+    perLight(ambientOut, diffuseOut, 0, viewPos, viewNormal);
 #if PER_PIXEL_LIGHTING
-    lightResult.xyz += diffuseLight * shadowing - diffuseLight; // This light gets added a second time in the loop to fix Mesa users' slowdown, so we need to negate its contribution here.
+    diffuseLight = diffuseOut * shadowing - diffuseOut;
 #else
-    shadowDiffuse = diffuseLight;
-    lightResult.xyz -= shadowDiffuse; // This light gets added a second time in the loop to fix Mesa users' slowdown, so we need to negate its contribution here.
+    shadowDiffuse = diffuseOut;
+    diffuseLight = -diffuseOut;
 #endif
+    ambientLight = gl_LightModel.ambient.xyz;
     for (int i=0; i<MAX_LIGHTS; ++i)
     {
-        perLight(ambientLight, diffuseLight, i, viewPos, viewNormal, diffuse, ambient);
-        lightResult.xyz += ambientLight + diffuseLight;
+        perLight(ambientOut, diffuseOut, i, viewPos, viewNormal);
+        ambientLight += ambientOut;
+        diffuseLight += diffuseOut;
     }
-
-    lightResult.xyz += gl_LightModel.ambient.xyz * ambient;
-
-    if (colorMode == ColorMode_Emission)
-        lightResult.xyz += vertexColor.xyz;
-    else
-        lightResult.xyz += gl_FrontMaterial.emission.xyz;
-
-#if @clamp
-    lightResult = clamp(lightResult, vec4(0.0), vec4(1.0));
-#else
-    lightResult = max(lightResult, 0.0);
-#endif
-    return lightResult;
 }
 
 
@@ -88,7 +41,7 @@ vec3 getSpecular(vec3 viewNormal, vec3 viewDirection, float shininess, vec3 matS
     vec3 lightDir = normalize(gl_LightSource[0].position.xyz);
     float NdotL = dot(viewNormal, lightDir);
     if (NdotL <= 0.0)
-        return vec3(0.,0.,0.);
+        return vec3(0.0);
     vec3 halfVec = normalize(lightDir - viewDirection);
     float NdotH = dot(viewNormal, halfVec);
     return pow(max(NdotH, 0.0), max(1e-4, shininess)) * gl_LightSource[0].specular.xyz * matSpec;
diff --git a/files/shaders/objects_fragment.glsl b/files/shaders/objects_fragment.glsl
index d5716c378..78660685f 100644
--- a/files/shaders/objects_fragment.glsl
+++ b/files/shaders/objects_fragment.glsl
@@ -57,13 +57,13 @@ varying float linearDepth;
 #define PER_PIXEL_LIGHTING (@normalMap || @forcePPL)
 
 #if !PER_PIXEL_LIGHTING
-centroid varying vec4 lighting;
+centroid varying vec3 passLighting;
 centroid varying vec3 shadowDiffuseLighting;
 #endif
-centroid varying vec4 passColor;
 varying vec3 passViewPos;
 varying vec3 passNormal;
 
+#include "vertexcolors.glsl"
 #include "shadows_fragment.glsl"
 #include "lighting.glsl"
 #include "parallax.glsl"
@@ -150,20 +150,27 @@ void main()
 
 #endif
 
-    float shadowing = unshadowedLightRatio(linearDepth);
+    vec4 diffuseColor = getDiffuseColor();
+    gl_FragData[0].a *= diffuseColor.a;
 
+    float shadowing = unshadowedLightRatio(linearDepth);
+    vec3 lighting;
 #if !PER_PIXEL_LIGHTING
-
-#if @clamp
-    gl_FragData[0] *= clamp(lighting + vec4(shadowDiffuseLighting * shadowing, 0), vec4(0.0), vec4(1.0));
+    lighting = passLighting + shadowDiffuseLighting * shadowing;
 #else
-    gl_FragData[0] *= lighting + vec4(shadowDiffuseLighting * shadowing, 0);
+    vec3 diffuseLight, ambientLight;
+    doLighting(passViewPos, normalize(viewNormal), shadowing, diffuseLight, ambientLight);
+    lighting = diffuseColor.xyz * diffuseLight + getAmbientColor().xyz * ambientLight + getEmissionColor().xyz;
 #endif
 
+#if @clamp
+    lighting = clamp(lighting, vec3(0.0), vec3(1.0));
 #else
-    gl_FragData[0] *= doLighting(passViewPos, normalize(viewNormal), passColor, shadowing);
+    lighting = max(lighting, 0.0);
 #endif
 
+    gl_FragData[0].xyz *= lighting;
+
 #if @envMap && !@preLightEnv
     gl_FragData[0].xyz += texture2D(envMap, envTexCoordGen).xyz * envMapColor.xyz * envLuma;
 #endif
@@ -178,11 +185,7 @@ void main()
     vec3 matSpec = specTex.xyz;
 #else
     float shininess = gl_FrontMaterial.shininess;
-    vec3 matSpec;
-    if (colorMode == ColorMode_Specular)
-        matSpec = passColor.xyz;
-    else
-        matSpec = gl_FrontMaterial.specular.xyz;
+    vec3 matSpec = getSpecularColor().xyz;
 #endif
 
     if (matSpec != vec3(0.0))
diff --git a/files/shaders/objects_vertex.glsl b/files/shaders/objects_vertex.glsl
index 40c448de9..15467933b 100644
--- a/files/shaders/objects_vertex.glsl
+++ b/files/shaders/objects_vertex.glsl
@@ -43,13 +43,13 @@ varying float linearDepth;
 #define PER_PIXEL_LIGHTING (@normalMap || @forcePPL)
 
 #if !PER_PIXEL_LIGHTING
-centroid varying vec4 lighting;
+centroid varying vec3 passLighting;
 centroid varying vec3 shadowDiffuseLighting;
 #endif
-centroid varying vec4 passColor;
 varying vec3 passViewPos;
 varying vec3 passNormal;
 
+#include "vertexcolors.glsl"
 #include "shadows_vertex.glsl"
 
 #include "lighting.glsl"
@@ -107,13 +107,17 @@ void main(void)
     specularMapUV = (gl_TextureMatrix[@specularMapUV] * gl_MultiTexCoord@specularMapUV).xy;
 #endif
 
-#if !PER_PIXEL_LIGHTING
-    lighting = doLighting(viewPos.xyz, viewNormal, gl_Color, shadowDiffuseLighting);
-#endif
     passColor = gl_Color;
     passViewPos = viewPos.xyz;
     passNormal = gl_Normal.xyz;
 
+#if !PER_PIXEL_LIGHTING
+    vec3 diffuseLight, ambientLight;
+    doLighting(viewPos.xyz, viewNormal, diffuseLight, ambientLight, shadowDiffuseLighting);
+    passLighting = getDiffuseColor().xyz * diffuseLight + getAmbientColor().xyz * ambientLight + getEmissionColor().xyz;
+    shadowDiffuseLighting *= getDiffuseColor().xyz;
+#endif
+
 #if (@shadows_enabled)
     setupShadowCoords(viewPos, viewNormal);
 #endif
diff --git a/files/shaders/terrain_fragment.glsl b/files/shaders/terrain_fragment.glsl
index 7409ce045..4736bb9b7 100644
--- a/files/shaders/terrain_fragment.glsl
+++ b/files/shaders/terrain_fragment.glsl
@@ -18,13 +18,13 @@ varying float linearDepth;
 #define PER_PIXEL_LIGHTING (@normalMap || @forcePPL)
 
 #if !PER_PIXEL_LIGHTING
-centroid varying vec4 lighting;
+centroid varying vec3 passLighting;
 centroid varying vec3 shadowDiffuseLighting;
 #endif
-centroid varying vec4 passColor;
 varying vec3 passViewPos;
 varying vec3 passNormal;
 
+#include "vertexcolors.glsl"
 #include "shadows_fragment.glsl"
 #include "lighting.glsl"
 #include "parallax.glsl"
@@ -68,30 +68,33 @@ void main()
     gl_FragData[0].a *= texture2D(blendMap, blendMapUV).a;
 #endif
 
-    float shadowing = unshadowedLightRatio(linearDepth);
+    vec4 diffuseColor = getDiffuseColor();
+    gl_FragData[0].a *= diffuseColor.a;
 
+    float shadowing = unshadowedLightRatio(linearDepth);
+    vec3 lighting;
 #if !PER_PIXEL_LIGHTING
-
-#if @clamp
-    gl_FragData[0] *= clamp(lighting + vec4(shadowDiffuseLighting * shadowing, 0), vec4(0.0), vec4(1.0));
+    lighting = passLighting + shadowDiffuseLighting * shadowing;
 #else
-    gl_FragData[0] *= lighting + vec4(shadowDiffuseLighting * shadowing, 0);
+    vec3 diffuseLight, ambientLight;
+    doLighting(passViewPos, normalize(viewNormal), shadowing, diffuseLight, ambientLight);
+    lighting = diffuseColor.xyz * diffuseLight + getAmbientColor().xyz * ambientLight + getEmissionColor().xyz;
 #endif
 
+#if @clamp
+    lighting = clamp(lighting, vec3(0.0), vec3(1.0));
 #else
-    gl_FragData[0] *= doLighting(passViewPos, normalize(viewNormal), passColor, shadowing);
+    lighting = max(lighting, 0.0);
 #endif
 
+    gl_FragData[0].xyz *= lighting;
+
 #if @specularMap
     float shininess = 128.0; // TODO: make configurable
     vec3 matSpec = vec3(diffuseTex.a);
 #else
     float shininess = gl_FrontMaterial.shininess;
-    vec3 matSpec;
-    if (colorMode == ColorMode_Specular)
-        matSpec = passColor.xyz;
-    else
-        matSpec = gl_FrontMaterial.specular.xyz;
+    vec3 matSpec = getSpecularColor().xyz;
 #endif
 
     if (matSpec != vec3(0.0))
diff --git a/files/shaders/terrain_vertex.glsl b/files/shaders/terrain_vertex.glsl
index bf337cf54..4b00ea525 100644
--- a/files/shaders/terrain_vertex.glsl
+++ b/files/shaders/terrain_vertex.glsl
@@ -7,13 +7,13 @@ varying float linearDepth;
 #define PER_PIXEL_LIGHTING (@normalMap || @forcePPL)
 
 #if !PER_PIXEL_LIGHTING
-centroid varying vec4 lighting;
+centroid varying vec3 passLighting;
 centroid varying vec3 shadowDiffuseLighting;
 #endif
-centroid varying vec4 passColor;
 varying vec3 passViewPos;
 varying vec3 passNormal;
 
+#include "vertexcolors.glsl"
 #include "shadows_vertex.glsl"
 
 #include "lighting.glsl"
@@ -31,13 +31,17 @@ void main(void)
     vec3 viewNormal = normalize((gl_NormalMatrix * gl_Normal).xyz);
 #endif
 
-#if !PER_PIXEL_LIGHTING
-    lighting = doLighting(viewPos.xyz, viewNormal, gl_Color, shadowDiffuseLighting);
-#endif
     passColor = gl_Color;
     passNormal = gl_Normal.xyz;
     passViewPos = viewPos.xyz;
 
+#if !PER_PIXEL_LIGHTING
+    vec3 diffuseLight, ambientLight;
+    doLighting(viewPos.xyz, viewNormal, diffuseLight, ambientLight, shadowDiffuseLighting);
+    passLighting = getDiffuseColor().xyz * diffuseLight + getAmbientColor().xyz * ambientLight + getEmissionColor().xyz;
+    shadowDiffuseLighting *= getDiffuseColor().xyz;
+#endif
+
     uv = gl_MultiTexCoord0.xy;
 
 #if (@shadows_enabled)
diff --git a/files/shaders/vertexcolors.glsl b/files/shaders/vertexcolors.glsl
new file mode 100644
index 000000000..38026d6ac
--- /dev/null
+++ b/files/shaders/vertexcolors.glsl
@@ -0,0 +1,38 @@
+centroid varying vec4 passColor;
+
+uniform int colorMode;
+
+const int ColorMode_None = 0;
+const int ColorMode_Emission = 1;
+const int ColorMode_AmbientAndDiffuse = 2;
+const int ColorMode_Ambient = 3;
+const int ColorMode_Diffuse = 4;
+const int ColorMode_Specular = 5;
+
+vec4 getEmissionColor()
+{
+    if (colorMode == ColorMode_Emission)
+        return passColor;
+    return gl_FrontMaterial.emission;
+}
+
+vec4 getAmbientColor()
+{
+    if (colorMode == ColorMode_AmbientAndDiffuse || colorMode == ColorMode_Ambient)
+        return passColor;
+    return gl_FrontMaterial.ambient;
+}
+
+vec4 getDiffuseColor()
+{
+    if (colorMode == ColorMode_AmbientAndDiffuse || colorMode == ColorMode_Diffuse)
+        return passColor;
+    return gl_FrontMaterial.diffuse;
+}
+
+vec4 getSpecularColor()
+{
+    if (colorMode == ColorMode_Specular)
+        return passColor;
+    return gl_FrontMaterial.specular;
+}