From 26ecf2088b6c9037230b154ee8671287c226f8ff Mon Sep 17 00:00:00 2001
From: AnyOldName3 <krizdjali+github@gmail.com>
Date: Thu, 21 Sep 2017 01:59:02 +0100
Subject: [PATCH] Make shadows play nicely with per-vertex lighting

---
 files/shaders/lighting.glsl         | 34 +++++++++++++++++++++--------
 files/shaders/objects_fragment.glsl |  3 ++-
 files/shaders/objects_vertex.glsl   |  3 ++-
 files/shaders/terrain_fragment.glsl |  3 ++-
 files/shaders/terrain_vertex.glsl   |  3 ++-
 5 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/files/shaders/lighting.glsl b/files/shaders/lighting.glsl
index e50e2f9c2..fe298e160 100644
--- a/files/shaders/lighting.glsl
+++ b/files/shaders/lighting.glsl
@@ -1,21 +1,33 @@
 #define MAX_LIGHTS 8
 
-vec3 perLight(int lightIndex, vec3 viewPos, vec3 viewNormal, vec4 diffuse, vec3 ambient, float shadowing)
+vec3 lightAmbient(int lightIndex, vec3 ambient)
+{
+    return ambient * gl_LightSource[lightIndex].ambient.xyz;
+}
+
+vec3 lightDiffuse(int lightIndex, vec4 diffuse, vec3 viewNormal, vec3 lightDir)
+{
+    return diffuse.xyz * gl_LightSource[lightIndex].diffuse.xyz * max(dot(viewNormal.xyz, lightDir), 0.0);
+}
+
+void perLight(out vec3 ambientOut, out vec3 diffuseOut, int lightIndex, vec3 viewPos, vec3 viewNormal, vec4 diffuse, vec3 ambient)
 {
     vec3 lightDir;
-    float d;
+	float lightDistance;
 
-	lightDir = gl_LightSource[lightIndex].position.xyz - (viewPos.xyz * gl_LightSource[lightIndex].position.w);
-    d = length(lightDir);
+    lightDir = gl_LightSource[lightIndex].position.xyz - (viewPos.xyz * gl_LightSource[lightIndex].position.w);
+    lightDistance = length(lightDir);
     lightDir = normalize(lightDir);
+	float illumination = clamp(1.0 / (gl_LightSource[lightIndex].constantAttenuation + gl_LightSource[lightIndex].linearAttenuation * lightDistance + gl_LightSource[lightIndex].quadraticAttenuation * lightDistance * lightDistance), 0.0, 1.0);
 
-    return (ambient * gl_LightSource[lightIndex].ambient.xyz + diffuse.xyz * gl_LightSource[lightIndex].diffuse.xyz * max(dot(viewNormal.xyz, lightDir), 0.0)) * clamp(1.0 / (gl_LightSource[lightIndex].constantAttenuation + gl_LightSource[lightIndex].linearAttenuation * d + gl_LightSource[lightIndex].quadraticAttenuation * d * d), 0.0, 1.0) * shadowing;
+    ambientOut = lightAmbient(lightIndex, ambient) * illumination;
+    diffuseOut = lightDiffuse(lightIndex, diffuse, viewNormal, lightDir) * illumination;
 }
 
 #ifdef FRAGMENT
 vec4 doLighting(vec3 viewPos, vec3 viewNormal, vec4 vertexColor, float shadowing)
 #else
-vec4 doLighting(vec3 viewPos, vec3 viewNormal, vec4 vertexColor)
+vec4 doLighting(vec3 viewPos, vec3 viewNormal, vec4 vertexColor, out vec3 shadowDiffuse)
 #endif
 {
 #if @colorMode == 3
@@ -30,14 +42,18 @@ vec4 doLighting(vec3 viewPos, vec3 viewNormal, vec4 vertexColor)
 #endif
     vec4 lightResult = vec4(0.0, 0.0, 0.0, diffuse.a);
 
+    vec3 diffuseLight, ambientLight;
+    perLight(ambientLight, diffuseLight, 0, viewPos, viewNormal, diffuse, ambient);
 #ifdef FRAGMENT
-	lightResult.xyz += perLight(0, viewPos, viewNormal, diffuse, ambient, shadowing);
+    lightResult.xyz += ambientLight + diffuseLight * shadowing;
 #else
-	lightResult.xyz += perLight(0, viewPos, viewNormal, diffuse, ambient, 1.0);
+    shadowDiffuse = diffuseLight;
+	lightResult.xyz += ambientLight;
 #endif
     for (int i=1; i<MAX_LIGHTS; ++i)
     {
-        lightResult.xyz += perLight(i, viewPos, viewNormal, diffuse, ambient, 1.0);
+        perLight(ambientLight, diffuseLight, i, viewPos, viewNormal, diffuse, ambient);
+        lightResult.xyz += ambientLight + diffuseLight;
     }
 
     lightResult.xyz += gl_LightModel.ambient.xyz * ambient;
diff --git a/files/shaders/objects_fragment.glsl b/files/shaders/objects_fragment.glsl
index d4823e9c1..f90e5f8ba 100644
--- a/files/shaders/objects_fragment.glsl
+++ b/files/shaders/objects_fragment.glsl
@@ -50,6 +50,7 @@ varying float depth;
 
 #if !PER_PIXEL_LIGHTING
 varying vec4 lighting;
+varying vec3 shadowDiffuseLighting;
 #else
 varying vec4 passColor;
 #endif
@@ -120,7 +121,7 @@ void main()
 	float shadowing = shadow2DProj(shadowTexture, shadowSpaceCoords).r;
 
 #if !PER_PIXEL_LIGHTING
-    gl_FragData[0] *= lighting * shadowing;
+    gl_FragData[0] *= lighting + vec4(shadowDiffuseLighting * shadowing, 0);
 #else
     gl_FragData[0] *= doLighting(passViewPos, normalize(viewNormal), passColor, shadowing);
 #endif
diff --git a/files/shaders/objects_vertex.glsl b/files/shaders/objects_vertex.glsl
index 398156b98..42949dccb 100644
--- a/files/shaders/objects_vertex.glsl
+++ b/files/shaders/objects_vertex.glsl
@@ -39,6 +39,7 @@ varying float depth;
 
 #if !PER_PIXEL_LIGHTING
 varying vec4 lighting;
+varying vec3 shadowDiffuseLighting;
 #else
 varying vec4 passColor;
 #endif
@@ -95,7 +96,7 @@ void main(void)
 #endif
 
 #if !PER_PIXEL_LIGHTING
-    lighting = doLighting(viewPos.xyz, viewNormal, gl_Color);
+    lighting = doLighting(viewPos.xyz, viewNormal, gl_Color, shadowDiffuseLighting);
 #else
     passColor = gl_Color;
 #endif
diff --git a/files/shaders/terrain_fragment.glsl b/files/shaders/terrain_fragment.glsl
index cca9a57dd..2e61caeb0 100644
--- a/files/shaders/terrain_fragment.glsl
+++ b/files/shaders/terrain_fragment.glsl
@@ -20,6 +20,7 @@ varying float depth;
 
 #if !PER_PIXEL_LIGHTING
 varying vec4 lighting;
+varying vec3 shadowDiffuseLighting;
 #else
 varying vec4 passColor;
 #endif
@@ -72,7 +73,7 @@ void main()
 	float shadowing = shadow2DProj(shadowTexture, shadowSpaceCoords).r;
 
 #if !PER_PIXEL_LIGHTING
-    gl_FragData[0] *= lighting * shadowing;
+    gl_FragData[0] *= lighting + vec4(shadowDiffuseLighting * shadowing, 0);
 #else
     gl_FragData[0] *= doLighting(passViewPos, normalize(viewNormal), passColor, shadowing);
 #endif
diff --git a/files/shaders/terrain_vertex.glsl b/files/shaders/terrain_vertex.glsl
index 1bc2337f8..89a2307a1 100644
--- a/files/shaders/terrain_vertex.glsl
+++ b/files/shaders/terrain_vertex.glsl
@@ -7,6 +7,7 @@ varying float depth;
 
 #if !PER_PIXEL_LIGHTING
 varying vec4 lighting;
+varying vec3 shadowDiffuseLighting;
 #else
 varying vec4 passColor;
 #endif
@@ -27,7 +28,7 @@ void main(void)
 
 #if !PER_PIXEL_LIGHTING
     vec3 viewNormal = normalize((gl_NormalMatrix * gl_Normal).xyz);
-    lighting = doLighting(viewPos.xyz, viewNormal, gl_Color);
+    lighting = doLighting(viewPos.xyz, viewNormal, gl_Color, shadowDiffuseLighting);
 #else
     passColor = gl_Color;
 #endif