File melonds_PR1967.patch of Package melonds

Overview Repositories Revisions Requests Users Attributes Meta

File melonds_PR1967.patch of Package melonds (Revision dc7f18f7e77a55952fa921cd8dd272e1)

Currently displaying revision dc7f18f7e77a55952fa921cd8dd272e1 , Show latest

From ad70c23ed80c279f0ef13ee38961a412c2779bb9 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Sun, 28 Jan 2024 15:55:34 -0500
Subject: [PATCH 01/18] maintain precision until all lights are calculated

fixes lugia on the soul silver title screen
---
 src/GPU3D.cpp | 38 ++++++++++++++++++++++----------------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 47abae2f0c..0749443743 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1463,11 +1463,13 @@ void GPU3D::CalculateLighting() noexcept
     normaltrans[1] = (Normal[0]*VecMatrix[1] + Normal[1]*VecMatrix[5] + Normal[2]*VecMatrix[9]) >> 12;
     normaltrans[2] = (Normal[0]*VecMatrix[2] + Normal[1]*VecMatrix[6] + Normal[2]*VecMatrix[10]) >> 12;
 
-    VertexColor[0] = MatEmission[0];
-    VertexColor[1] = MatEmission[1];
-    VertexColor[2] = MatEmission[2];
-
     s32 c = 0;
+    u32 vtxbuff[3] =
+    {
+        (u32)MatEmission[0] << 13,
+        (u32)MatEmission[1] << 13,
+        (u32)MatEmission[2] << 13
+    };
     for (int i = 0; i < 4; i++)
     {
         if (!(CurPolygonAttr & (1<<i)))
@@ -1500,24 +1502,28 @@ void GPU3D::CalculateLighting() noexcept
             shinelevel = ShininessTable[shinelevel];
         }
 
-        VertexColor[0] += ((MatSpecular[0] * LightColor[i][0] * shinelevel) >> 13);
-        VertexColor[0] += ((MatDiffuse[0] * LightColor[i][0] * difflevel) >> 13);
-        VertexColor[0] += ((MatAmbient[0] * LightColor[i][0]) >> 5);
+        vtxbuff[0] += ((MatSpecular[0] * LightColor[i][0] * shinelevel));
+        vtxbuff[0] += ((MatDiffuse[0] * LightColor[i][0] * difflevel));
+        vtxbuff[0] += ((MatAmbient[0] * LightColor[i][0]) << 8);
 
-        VertexColor[1] += ((MatSpecular[1] * LightColor[i][1] * shinelevel) >> 13);
-        VertexColor[1] += ((MatDiffuse[1] * LightColor[i][1] * difflevel) >> 13);
-        VertexColor[1] += ((MatAmbient[1] * LightColor[i][1]) >> 5);
+        vtxbuff[1] += ((MatSpecular[1] * LightColor[i][1] * shinelevel));
+        vtxbuff[1] += ((MatDiffuse[1] * LightColor[i][1] * difflevel));
+        vtxbuff[1] += ((MatAmbient[1] * LightColor[i][1]) << 8);
 
-        VertexColor[2] += ((MatSpecular[2] * LightColor[i][2] * shinelevel) >> 13);
-        VertexColor[2] += ((MatDiffuse[2] * LightColor[i][2] * difflevel) >> 13);
-        VertexColor[2] += ((MatAmbient[2] * LightColor[i][2]) >> 5);
+        vtxbuff[2] += ((MatSpecular[2] * LightColor[i][2] * shinelevel));
+        vtxbuff[2] += ((MatDiffuse[2] * LightColor[i][2] * difflevel));
+        vtxbuff[2] += ((MatAmbient[2] * LightColor[i][2]) << 8);
 
-        if (VertexColor[0] > 31) VertexColor[0] = 31;
-        if (VertexColor[1] > 31) VertexColor[1] = 31;
-        if (VertexColor[2] > 31) VertexColor[2] = 31;
+        if (vtxbuff[0] > 31 << 13) vtxbuff[0] = 31 << 13;
+        if (vtxbuff[1] > 31 << 13) vtxbuff[1] = 31 << 13;
+        if (vtxbuff[2] > 31 << 13) vtxbuff[2] = 31 << 13;
 
         c++;
     }
+    
+    VertexColor[0] = vtxbuff[0] >> 13;
+    VertexColor[1] = vtxbuff[1] >> 13;
+    VertexColor[2] = vtxbuff[2] >> 13;
 
     if (c < 1) c = 1;
     NormalPipeline = 7;

From e8e478a34417f89d1980530373f9a3ecc9931a9f Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Sun, 28 Jan 2024 16:06:44 -0500
Subject: [PATCH 02/18] small optimization

---
 src/GPU3D.cpp | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 0749443743..93d2de225b 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1502,17 +1502,20 @@ void GPU3D::CalculateLighting() noexcept
             shinelevel = ShininessTable[shinelevel];
         }
 
-        vtxbuff[0] += ((MatSpecular[0] * LightColor[i][0] * shinelevel));
-        vtxbuff[0] += ((MatDiffuse[0] * LightColor[i][0] * difflevel));
-        vtxbuff[0] += ((MatAmbient[0] * LightColor[i][0]) << 8);
-
-        vtxbuff[1] += ((MatSpecular[1] * LightColor[i][1] * shinelevel));
-        vtxbuff[1] += ((MatDiffuse[1] * LightColor[i][1] * difflevel));
-        vtxbuff[1] += ((MatAmbient[1] * LightColor[i][1]) << 8);
-
-        vtxbuff[2] += ((MatSpecular[2] * LightColor[i][2] * shinelevel));
-        vtxbuff[2] += ((MatDiffuse[2] * LightColor[i][2] * difflevel));
-        vtxbuff[2] += ((MatAmbient[2] * LightColor[i][2]) << 8);
+        vtxbuff[0] += (MatSpecular[0] * shinelevel +
+                      MatDiffuse[0] * difflevel +
+                      (MatAmbient[0] << 8)) *
+                      LightColor[i][0];
+
+        vtxbuff[1] += (MatSpecular[1] * shinelevel +
+                      MatDiffuse[1] * difflevel +
+                      (MatAmbient[1] << 8)) *
+                      LightColor[i][1];
+
+        vtxbuff[2] += (MatSpecular[2] * shinelevel +
+                      MatDiffuse[2] * difflevel +
+                      (MatAmbient[2] << 8)) *
+                      LightColor[i][2];
 
         if (vtxbuff[0] > 31 << 13) vtxbuff[0] = 31 << 13;
         if (vtxbuff[1] > 31 << 13) vtxbuff[1] = 31 << 13;

From b852beac6b2097ac750059c61d2770de4315b422 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Sun, 28 Jan 2024 16:36:37 -0500
Subject: [PATCH 03/18] small note

---
 src/GPU3D.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 93d2de225b..a2597616ba 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1504,7 +1504,7 @@ void GPU3D::CalculateLighting() noexcept
 
         vtxbuff[0] += (MatSpecular[0] * shinelevel +
                       MatDiffuse[0] * difflevel +
-                      (MatAmbient[0] << 8)) *
+                      (MatAmbient[0] << 8)) * // ambient seems to be shifted left 8 rather than multiplied by 255(?)
                       LightColor[i][0];
 
         vtxbuff[1] += (MatSpecular[1] * shinelevel +

From ed5796d0009f93bb569f70fcb40dffb79a1757e0 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Mon, 29 Jan 2024 23:23:49 -0500
Subject: [PATCH 04/18] small cleanup/notes

shouldn't need to check that every time, since the variable shouldn't be able to overflow
---
 src/GPU3D.cpp | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index a2597616ba..e589ea8e6c 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1480,7 +1480,8 @@ void GPU3D::CalculateLighting() noexcept
         // * diffuse level is saturated to 255
         // * shininess level mirrors back to 0 and is ANDed with 0xFF, that before being squared
         // TODO: check how it behaves when the computed shininess is >=0x200
-
+        // TODO: seems to be some minor precision issue with how difflevel is calculated?
+        
         s32 difflevel = (-(LightDirection[i][0]*normaltrans[0] +
                          LightDirection[i][1]*normaltrans[1] +
                          LightDirection[i][2]*normaltrans[2])) >> 10;
@@ -1504,7 +1505,7 @@ void GPU3D::CalculateLighting() noexcept
 
         vtxbuff[0] += (MatSpecular[0] * shinelevel +
                       MatDiffuse[0] * difflevel +
-                      (MatAmbient[0] << 8)) * // ambient seems to be shifted left 8 rather than multiplied by 255(?)
+                      (MatAmbient[0] << 8)) * // ambient seems to be a plain bitshift
                       LightColor[i][0];
 
         vtxbuff[1] += (MatSpecular[1] * shinelevel +
@@ -1517,16 +1518,11 @@ void GPU3D::CalculateLighting() noexcept
                       (MatAmbient[2] << 8)) *
                       LightColor[i][2];
 
-        if (vtxbuff[0] > 31 << 13) vtxbuff[0] = 31 << 13;
-        if (vtxbuff[1] > 31 << 13) vtxbuff[1] = 31 << 13;
-        if (vtxbuff[2] > 31 << 13) vtxbuff[2] = 31 << 13;
-
         c++;
     }
-    
-    VertexColor[0] = vtxbuff[0] >> 13;
-    VertexColor[1] = vtxbuff[1] >> 13;
-    VertexColor[2] = vtxbuff[2] >> 13;
+    VertexColor[0] = (vtxbuff[0] >> 13 > 31) ? 31 : vtxbuff[0] >> 13;
+    VertexColor[1] = (vtxbuff[1] >> 13 > 31) ? 31 : vtxbuff[1] >> 13;
+    VertexColor[2] = (vtxbuff[2] >> 13 > 31) ? 31 : vtxbuff[2] >> 13;
 
     if (c < 1) c = 1;
     NormalPipeline = 7;

From c837ef290e3f8cc044d0c0a25e79d8938fc53ac3 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Thu, 8 Feb 2024 09:22:56 -0500
Subject: [PATCH 05/18] hw doesn't cap difflevel at 255

Should it cap at all?
Can vtx colors overflow...?
---
 src/GPU3D.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index e589ea8e6c..b7222ec3fa 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1477,7 +1477,7 @@ void GPU3D::CalculateLighting() noexcept
 
         // overflow handling (for example, if the normal length is >1)
         // according to some hardware tests
-        // * diffuse level is saturated to 255
+        // * diffuse level seems to keep going (TODO: does it cap? can it cause vtx colors to overflow?)
         // * shininess level mirrors back to 0 and is ANDed with 0xFF, that before being squared
         // TODO: check how it behaves when the computed shininess is >=0x200
         // TODO: seems to be some minor precision issue with how difflevel is calculated?
@@ -1486,7 +1486,6 @@ void GPU3D::CalculateLighting() noexcept
                          LightDirection[i][1]*normaltrans[1] +
                          LightDirection[i][2]*normaltrans[2])) >> 10;
         if (difflevel < 0) difflevel = 0;
-        else if (difflevel > 255) difflevel = 255;
 
         s32 shinelevel = -(((LightDirection[i][0]>>1)*normaltrans[0] +
                           (LightDirection[i][1]>>1)*normaltrans[1] +

From 58d3c23a2382787c935b0e34b764863710262dcc Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Thu, 8 Feb 2024 09:42:39 -0500
Subject: [PATCH 06/18] diffuse level appears to be shifted right by 9

fixes some minor inaccuracies
---
 src/GPU3D.cpp | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index b7222ec3fa..120fedffc1 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1466,9 +1466,9 @@ void GPU3D::CalculateLighting() noexcept
     s32 c = 0;
     u32 vtxbuff[3] =
     {
-        (u32)MatEmission[0] << 13,
-        (u32)MatEmission[1] << 13,
-        (u32)MatEmission[2] << 13
+        (u32)MatEmission[0] << 14,
+        (u32)MatEmission[1] << 14,
+        (u32)MatEmission[2] << 14
     };
     for (int i = 0; i < 4; i++)
     {
@@ -1480,11 +1480,10 @@ void GPU3D::CalculateLighting() noexcept
         // * diffuse level seems to keep going (TODO: does it cap? can it cause vtx colors to overflow?)
         // * shininess level mirrors back to 0 and is ANDed with 0xFF, that before being squared
         // TODO: check how it behaves when the computed shininess is >=0x200
-        // TODO: seems to be some minor precision issue with how difflevel is calculated?
         
         s32 difflevel = (-(LightDirection[i][0]*normaltrans[0] +
                          LightDirection[i][1]*normaltrans[1] +
-                         LightDirection[i][2]*normaltrans[2])) >> 10;
+                         LightDirection[i][2]*normaltrans[2])) >> 9;
         if (difflevel < 0) difflevel = 0;
 
         s32 shinelevel = -(((LightDirection[i][0]>>1)*normaltrans[0] +
@@ -1501,27 +1500,28 @@ void GPU3D::CalculateLighting() noexcept
             shinelevel >>= 1;
             shinelevel = ShininessTable[shinelevel];
         }
+        shinelevel <<= 1;
 
         vtxbuff[0] += (MatSpecular[0] * shinelevel +
                       MatDiffuse[0] * difflevel +
-                      (MatAmbient[0] << 8)) * // ambient seems to be a plain bitshift
+                      (MatAmbient[0] << 9)) * // ambient seems to be a plain bitshift
                       LightColor[i][0];
 
         vtxbuff[1] += (MatSpecular[1] * shinelevel +
                       MatDiffuse[1] * difflevel +
-                      (MatAmbient[1] << 8)) *
+                      (MatAmbient[1] << 9)) *
                       LightColor[i][1];
 
         vtxbuff[2] += (MatSpecular[2] * shinelevel +
                       MatDiffuse[2] * difflevel +
-                      (MatAmbient[2] << 8)) *
+                      (MatAmbient[2] << 9)) *
                       LightColor[i][2];
 
         c++;
     }
-    VertexColor[0] = (vtxbuff[0] >> 13 > 31) ? 31 : vtxbuff[0] >> 13;
-    VertexColor[1] = (vtxbuff[1] >> 13 > 31) ? 31 : vtxbuff[1] >> 13;
-    VertexColor[2] = (vtxbuff[2] >> 13 > 31) ? 31 : vtxbuff[2] >> 13;
+    VertexColor[0] = (vtxbuff[0] >> 14 > 31) ? 31 : vtxbuff[0] >> 14;
+    VertexColor[1] = (vtxbuff[1] >> 14 > 31) ? 31 : vtxbuff[1] >> 14;
+    VertexColor[2] = (vtxbuff[2] >> 14 > 31) ? 31 : vtxbuff[2] >> 14;
 
     if (c < 1) c = 1;
     NormalPipeline = 7;

From fce490eb26c711399aca1e9854b784a97c815c18 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Thu, 8 Feb 2024 19:49:42 -0500
Subject: [PATCH 07/18] improve specular lighting a little

---
 src/GPU3D.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 120fedffc1..80ceeff74a 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1488,19 +1488,19 @@ void GPU3D::CalculateLighting() noexcept
 
         s32 shinelevel = -(((LightDirection[i][0]>>1)*normaltrans[0] +
                           (LightDirection[i][1]>>1)*normaltrans[1] +
-                          ((LightDirection[i][2]-0x200)>>1)*normaltrans[2]) >> 10);
+                          ((LightDirection[i][2]-0x200)>>1)*normaltrans[2]) >> 9);
         if (shinelevel < 0) shinelevel = 0;
-        else if (shinelevel > 255) shinelevel = (0x100 - shinelevel) & 0xFF;
-        shinelevel = ((shinelevel * shinelevel) >> 7) - 0x100; // really (2*shinelevel*shinelevel)-1
+        else if (shinelevel > 511) shinelevel = (0x200 - shinelevel) & 0x1FF;
+        shinelevel = ((shinelevel * shinelevel) >> 8) - 0x200; // really (2*shinelevel*shinelevel)-1
         if (shinelevel < 0) shinelevel = 0;
 
         if (UseShininessTable)
         {
             // checkme
-            shinelevel >>= 1;
+            shinelevel >>= 2;
             shinelevel = ShininessTable[shinelevel];
+            shinelevel <<= 1;
         }
-        shinelevel <<= 1;
 
         vtxbuff[0] += (MatSpecular[0] * shinelevel +
                       MatDiffuse[0] * difflevel +

From 9cb75e07133c100004b47dc88dfeeb46b253ae53 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Fri, 9 Feb 2024 07:49:28 -0500
Subject: [PATCH 08/18] small improvement to diffuse lighting

fixes a few off by ones
- finding by azusa
---
 src/GPU3D.cpp | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 80ceeff74a..602873813d 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1477,13 +1477,15 @@ void GPU3D::CalculateLighting() noexcept
 
         // overflow handling (for example, if the normal length is >1)
         // according to some hardware tests
-        // * diffuse level seems to keep going (TODO: does it cap? can it cause vtx colors to overflow?)
+        // * diffuse level seems to keep going (TODO: until roughly == 1024, where it starts to wraparound? invert?)
         // * shininess level mirrors back to 0 and is ANDed with 0xFF, that before being squared
         // TODO: check how it behaves when the computed shininess is >=0x200
         
-        s32 difflevel = (-(LightDirection[i][0]*normaltrans[0] +
-                         LightDirection[i][1]*normaltrans[1] +
-                         LightDirection[i][2]*normaltrans[2])) >> 9;
+        // negating first apparently matters for some reason
+        // precision is discarded after each addition
+        s32 difflevel = (-LightDirection[i][0]*normaltrans[0] & ~0x1FF) +
+                        (-LightDirection[i][1]*normaltrans[1] & ~0x1FF) +
+                        (-LightDirection[i][2]*normaltrans[2]) >> 9;
         if (difflevel < 0) difflevel = 0;
 
         s32 shinelevel = -(((LightDirection[i][0]>>1)*normaltrans[0] +

From 48e33d633ac0dc54e70ba87c332a31f8b31b6826 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Fri, 9 Feb 2024 19:08:05 -0500
Subject: [PATCH 09/18] small tweaks

---
 src/GPU3D.cpp | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 602873813d..cda6ed3035 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1481,11 +1481,11 @@ void GPU3D::CalculateLighting() noexcept
         // * shininess level mirrors back to 0 and is ANDed with 0xFF, that before being squared
         // TODO: check how it behaves when the computed shininess is >=0x200
         
-        // negating first apparently matters for some reason
-        // precision is discarded after each addition
-        s32 difflevel = (-LightDirection[i][0]*normaltrans[0] & ~0x1FF) +
-                        (-LightDirection[i][1]*normaltrans[1] & ~0x1FF) +
-                        (-LightDirection[i][2]*normaltrans[2]) >> 9;
+        // negating first matters for some reason
+        // bottom 9 bits are discarded after multiplying and before adding (todo: does this apply to any other dot product calculations?)
+        s32 difflevel = (-LightDirection[i][0]*normaltrans[0] >> 9) +
+                        (-LightDirection[i][1]*normaltrans[1] >> 9) +
+                        (-LightDirection[i][2]*normaltrans[2] >> 9);
         if (difflevel < 0) difflevel = 0;
 
         s32 shinelevel = -(((LightDirection[i][0]>>1)*normaltrans[0] +
@@ -1521,6 +1521,7 @@ void GPU3D::CalculateLighting() noexcept
 
         c++;
     }
+
     VertexColor[0] = (vtxbuff[0] >> 14 > 31) ? 31 : vtxbuff[0] >> 14;
     VertexColor[1] = (vtxbuff[1] >> 14 > 31) ? 31 : vtxbuff[1] >> 14;
     VertexColor[2] = (vtxbuff[2] >> 14 > 31) ? 31 : vtxbuff[2] >> 14;

From a9540bfaec9ba5a35f32b2bea5f9bd05c735dcdb Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Sat, 10 Feb 2024 00:03:50 -0500
Subject: [PATCH 10/18] handle overflows of diffuse lighting properly

-credits to azusa once more
---
 src/GPU3D.cpp | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index cda6ed3035..c4707474b0 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1477,20 +1477,45 @@ void GPU3D::CalculateLighting() noexcept
 
         // overflow handling (for example, if the normal length is >1)
         // according to some hardware tests
-        // * diffuse level seems to keep going (TODO: until roughly == 1024, where it starts to wraparound? invert?)
+        // * diffuse level seems to keep going until 1024, at which point it overflows
+        // resulting in diff level resetting to 0, and (light color * diffuse color) mirroring around 512 (strange behavior)
         // * shininess level mirrors back to 0 and is ANDed with 0xFF, that before being squared
         // TODO: check how it behaves when the computed shininess is >=0x200
         
+        // -- diffuse lighting --
         // negating first matters for some reason
-        // bottom 9 bits are discarded after multiplying and before adding (todo: does this apply to any other dot product calculations?)
+        // bottom 9 bits are discarded after multiplying and before adding (TODO: does this apply to any other dot product calculations?)
         s32 difflevel = (-LightDirection[i][0]*normaltrans[0] >> 9) +
                         (-LightDirection[i][1]*normaltrans[1] >> 9) +
                         (-LightDirection[i][2]*normaltrans[2] >> 9);
-        if (difflevel < 0) difflevel = 0;
 
+        if (difflevel <= 0); // if less than or equal to 0 add nothing
+        else if (difflevel >= 1024) // integer overflow (1 bit whole + 9 bits fractional)
+        {
+            vtxbuff[0] += (MatDiffuse[0] == 0 || LightColor[i][0] == 0) ? 0 :         // if diffuse color or light color are 0, outcome is 0
+                          ((512 - (MatDiffuse[0] * LightColor[i][0] - 512)) * 1024) + // product of diffuse * lightcolor is mirrored around 512
+                          (MatDiffuse[0] * LightColor[i][0] * (difflevel - 1024));    // diff level has 1024 subtracted to emulate overflow
+
+            vtxbuff[1] += (MatDiffuse[1] == 0 || LightColor[i][1] == 0) ? 0 :
+                          ((512 - (MatDiffuse[1] * LightColor[i][1] - 512)) * 1024) +
+                          (MatDiffuse[1] * LightColor[i][1] * (difflevel - 1024));
+
+            vtxbuff[2] += (MatDiffuse[2] == 0 || LightColor[i][2] == 0) ? 0 :
+                          ((512 - (MatDiffuse[2] * LightColor[i][2] - 512)) * 1024) +
+                          (MatDiffuse[2] * LightColor[i][2] * (difflevel - 1024));
+        }
+        else // normal handling
+        {
+            vtxbuff[0] += MatDiffuse[0] * LightColor[i][0] * difflevel;
+            vtxbuff[1] += MatDiffuse[1] * LightColor[i][1] * difflevel;
+            vtxbuff[2] += MatDiffuse[2] * LightColor[i][2] * difflevel;
+        }
+
+        // -- specular lighting --
         s32 shinelevel = -(((LightDirection[i][0]>>1)*normaltrans[0] +
                           (LightDirection[i][1]>>1)*normaltrans[1] +
                           ((LightDirection[i][2]-0x200)>>1)*normaltrans[2]) >> 9);
+
         if (shinelevel < 0) shinelevel = 0;
         else if (shinelevel > 511) shinelevel = (0x200 - shinelevel) & 0x1FF;
         shinelevel = ((shinelevel * shinelevel) >> 8) - 0x200; // really (2*shinelevel*shinelevel)-1
@@ -1505,17 +1530,14 @@ void GPU3D::CalculateLighting() noexcept
         }
 
         vtxbuff[0] += (MatSpecular[0] * shinelevel +
-                      MatDiffuse[0] * difflevel +
                       (MatAmbient[0] << 9)) * // ambient seems to be a plain bitshift
                       LightColor[i][0];
 
         vtxbuff[1] += (MatSpecular[1] * shinelevel +
-                      MatDiffuse[1] * difflevel +
                       (MatAmbient[1] << 9)) *
                       LightColor[i][1];
 
         vtxbuff[2] += (MatSpecular[2] * shinelevel +
-                      MatDiffuse[2] * difflevel +
                       (MatAmbient[2] << 9)) *
                       LightColor[i][2];

From 0e3c9f1813fc4f0021e3a7d27ec25095ad821385 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Sun, 11 Feb 2024 11:15:16 -0500
Subject: [PATCH 11/18] attempt at improving specular lighting calcs

still far from correct, but its a start.
fixes: https://github.com/melonDS-emu/melonDS/issues/1545
---
 src/GPU3D.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index c4707474b0..12b551c9fa 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1512,15 +1512,15 @@ void GPU3D::CalculateLighting() noexcept
         }
 
         // -- specular lighting --
-        s32 shinelevel = -(((LightDirection[i][0]>>1)*normaltrans[0] +
-                          (LightDirection[i][1]>>1)*normaltrans[1] +
-                          ((LightDirection[i][2]-0x200)>>1)*normaltrans[2]) >> 9);
+        s32 shinelevel = -(((LightDirection[i][0]>>1)*normaltrans[0] >> 9) +
+                          ((LightDirection[i][1]>>1)*normaltrans[1] >> 9) +
+                          (((LightDirection[i][2]-0x200)>>1)*normaltrans[2] >> 9));
 
         if (shinelevel < 0) shinelevel = 0;
         else if (shinelevel > 511) shinelevel = (0x200 - shinelevel) & 0x1FF;
-        shinelevel = ((shinelevel * shinelevel) >> 8) - 0x200; // really (2*shinelevel*shinelevel)-1
+        shinelevel = ((shinelevel * shinelevel) >> 7) - 0x200; // really (2*shinelevel*shinelevel)-1
         if (shinelevel < 0) shinelevel = 0;
-
+        if (shinelevel > 511) shinelevel = 511; // sort of a guess, but the exact value it caps at shouldn't matter as long as it results in light color 29.
         if (UseShininessTable)
         {
             // checkme

From 741276c06cb3629afece4fa61692ce28bc7b9292 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Sun, 11 Feb 2024 14:43:40 -0500
Subject: [PATCH 12/18] meh

---
 src/GPU3D.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 12b551c9fa..7a886eca70 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1504,7 +1504,7 @@ void GPU3D::CalculateLighting() noexcept
                           ((512 - (MatDiffuse[2] * LightColor[i][2] - 512)) * 1024) +
                           (MatDiffuse[2] * LightColor[i][2] * (difflevel - 1024));
         }
-        else // normal handling
+        else // handle lighting normally
         {
             vtxbuff[0] += MatDiffuse[0] * LightColor[i][0] * difflevel;
             vtxbuff[1] += MatDiffuse[1] * LightColor[i][1] * difflevel;

From e0690709e330c072dfffd271bc42cf9076ee2c0e Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Thu, 15 Feb 2024 19:33:11 -0500
Subject: [PATCH 13/18] improve specular lighting further

---
 src/GPU3D.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 7a886eca70..9a6f90054a 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1512,22 +1512,22 @@ void GPU3D::CalculateLighting() noexcept
         }
 
         // -- specular lighting --
-        s32 shinelevel = -(((LightDirection[i][0]>>1)*normaltrans[0] >> 9) +
-                          ((LightDirection[i][1]>>1)*normaltrans[1] >> 9) +
-                          (((LightDirection[i][2]-0x200)>>1)*normaltrans[2] >> 9));
+        s32 shinelevel = (-(LightDirection[i][0]>>1)*normaltrans[0]) +
+                         (-(LightDirection[i][1]>>1)*normaltrans[1]) +
+                         (-((LightDirection[i][2]-0x200)>>1)*normaltrans[2]);
 
         if (shinelevel < 0) shinelevel = 0;
-        else if (shinelevel > 511) shinelevel = (0x200 - shinelevel) & 0x1FF;
-        shinelevel = ((shinelevel * shinelevel) >> 7) - 0x200; // really (2*shinelevel*shinelevel)-1
+        else if (shinelevel > 0x3FFFF) shinelevel = (0x40000 - shinelevel) & 0x3FFFF;
+        shinelevel = (((s64)shinelevel * shinelevel) >> 26) - 0x100; // really (4*shinelevel*shinelevel)-1
         if (shinelevel < 0) shinelevel = 0;
-        if (shinelevel > 511) shinelevel = 511; // sort of a guess, but the exact value it caps at shouldn't matter as long as it results in light color 29.
+        if (shinelevel > 255) shinelevel = 255; // sort of a guess, but the exact value it caps at shouldn't matter as long as it results in light color 29.
         if (UseShininessTable)
         {
             // checkme
-            shinelevel >>= 2;
+            shinelevel >>= 1;
             shinelevel = ShininessTable[shinelevel];
-            shinelevel <<= 1;
         }
+        shinelevel <<= 1;
 
         vtxbuff[0] += (MatSpecular[0] * shinelevel +
                       (MatAmbient[0] << 9)) * // ambient seems to be a plain bitshift

From 8cf8e947c42faed7857e8ee9ed09f1d7ce4ef1a1 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Thu, 15 Feb 2024 23:58:20 -0500
Subject: [PATCH 14/18] add notes

---
 src/GPU3D.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 9a6f90054a..e90243f4a0 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1512,6 +1512,7 @@ void GPU3D::CalculateLighting() noexcept
         }
 
         // -- specular lighting --
+        // TODO: Z light dir is not handled properly currently: current theory is it's used for some sort of borked normalization?
         s32 shinelevel = (-(LightDirection[i][0]>>1)*normaltrans[0]) +
                          (-(LightDirection[i][1]>>1)*normaltrans[1]) +
                          (-((LightDirection[i][2]-0x200)>>1)*normaltrans[2]);
@@ -1520,7 +1521,8 @@ void GPU3D::CalculateLighting() noexcept
         else if (shinelevel > 0x3FFFF) shinelevel = (0x40000 - shinelevel) & 0x3FFFF;
         shinelevel = (((s64)shinelevel * shinelevel) >> 26) - 0x100; // really (4*shinelevel*shinelevel)-1
         if (shinelevel < 0) shinelevel = 0;
-        if (shinelevel > 255) shinelevel = 255; // sort of a guess, but the exact value it caps at shouldn't matter as long as it results in light color 29.
+        if (shinelevel > 255) shinelevel = 255; // todo: seems to break when > 511?
+
         if (UseShininessTable)
         {
             // checkme

From 4cecccccd8528b3f097dcc1ad4fafa461bd6d613 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Tue, 20 Feb 2024 22:46:03 -0500
Subject: [PATCH 15/18] theory: add half vec instead of subt 1

---
 src/GPU3D.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index e90243f4a0..fcda997e62 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1519,7 +1519,7 @@ void GPU3D::CalculateLighting() noexcept
 
         if (shinelevel < 0) shinelevel = 0;
         else if (shinelevel > 0x3FFFF) shinelevel = (0x40000 - shinelevel) & 0x3FFFF;
-        shinelevel = (((s64)shinelevel * shinelevel) >> 26) - 0x100; // really (4*shinelevel*shinelevel)-1
+        shinelevel = (((s64)shinelevel * shinelevel) >> 26) + (LightDirection[i][2]-0x200>>1); // really (4*shinelevel*shinelevel)+HalfVector[z]
         if (shinelevel < 0) shinelevel = 0;
         if (shinelevel > 255) shinelevel = 255; // todo: seems to break when > 511?

From ef2dade05b4b8e345fb0983d77a24843a08d1f0c Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Wed, 21 Feb 2024 07:24:19 -0500
Subject: [PATCH 16/18] implement azusa's specular lighting algorithm

---
 src/GPU3D.cpp | 68 +++++++++++++++++++++++++++++++--------------------
 1 file changed, 41 insertions(+), 27 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index fcda997e62..03102a2130 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1479,57 +1479,71 @@ void GPU3D::CalculateLighting() noexcept
         // according to some hardware tests
         // * diffuse level seems to keep going until 1024, at which point it overflows
         // resulting in diff level resetting to 0, and (light color * diffuse color) mirroring around 512 (strange behavior)
-        // * shininess level mirrors back to 0 and is ANDed with 0xFF, that before being squared
-        // TODO: check how it behaves when the computed shininess is >=0x200
+        // * shininess level mirrors back to 0 and is ANDed with 0x3FF, that before being squared
         
-        // -- diffuse lighting --
-        // negating first matters for some reason
+        // calculate dot product
         // bottom 9 bits are discarded after multiplying and before adding (TODO: does this apply to any other dot product calculations?)
-        s32 difflevel = (-LightDirection[i][0]*normaltrans[0] >> 9) +
-                        (-LightDirection[i][1]*normaltrans[1] >> 9) +
-                        (-LightDirection[i][2]*normaltrans[2] >> 9);
+        s32 dot = (-LightDirection[i][0]*normaltrans[0] >> 9) +
+                  (-LightDirection[i][1]*normaltrans[1] >> 9) +
+                  (-LightDirection[i][2]*normaltrans[2] >> 9);
+
+        // -- diffuse lighting --
 
-        if (difflevel <= 0); // if less than or equal to 0 add nothing
-        else if (difflevel >= 1024) // integer overflow (1 bit whole + 9 bits fractional)
+        if (dot <= 0); // if less than or equal to 0 add nothing
+        else if (dot >= 1024) // integer overflow (1 bit whole + 9 bits fractional)
         {
             vtxbuff[0] += (MatDiffuse[0] == 0 || LightColor[i][0] == 0) ? 0 :         // if diffuse color or light color are 0, outcome is 0
                           ((512 - (MatDiffuse[0] * LightColor[i][0] - 512)) * 1024) + // product of diffuse * lightcolor is mirrored around 512
-                          (MatDiffuse[0] * LightColor[i][0] * (difflevel - 1024));    // diff level has 1024 subtracted to emulate overflow
+                          (MatDiffuse[0] * LightColor[i][0] * (dot - 1024));          // the dot has 1024 subtracted to emulate overflow
 
             vtxbuff[1] += (MatDiffuse[1] == 0 || LightColor[i][1] == 0) ? 0 :
                           ((512 - (MatDiffuse[1] * LightColor[i][1] - 512)) * 1024) +
-                          (MatDiffuse[1] * LightColor[i][1] * (difflevel - 1024));
+                          (MatDiffuse[1] * LightColor[i][1] * (dot - 1024));
 
             vtxbuff[2] += (MatDiffuse[2] == 0 || LightColor[i][2] == 0) ? 0 :
                           ((512 - (MatDiffuse[2] * LightColor[i][2] - 512)) * 1024) +
-                          (MatDiffuse[2] * LightColor[i][2] * (difflevel - 1024));
+                          (MatDiffuse[2] * LightColor[i][2] * (dot - 1024));
         }
         else // handle lighting normally
         {
-            vtxbuff[0] += MatDiffuse[0] * LightColor[i][0] * difflevel;
-            vtxbuff[1] += MatDiffuse[1] * LightColor[i][1] * difflevel;
-            vtxbuff[2] += MatDiffuse[2] * LightColor[i][2] * difflevel;
+            vtxbuff[0] += MatDiffuse[0] * LightColor[i][0] * dot;
+            vtxbuff[1] += MatDiffuse[1] * LightColor[i][1] * dot;
+            vtxbuff[2] += MatDiffuse[2] * LightColor[i][2] * dot;
         }
 
         // -- specular lighting --
-        // TODO: Z light dir is not handled properly currently: current theory is it's used for some sort of borked normalization?
-        s32 shinelevel = (-(LightDirection[i][0]>>1)*normaltrans[0]) +
-                         (-(LightDirection[i][1]>>1)*normaltrans[1]) +
-                         (-((LightDirection[i][2]-0x200)>>1)*normaltrans[2]);
+        
+        s32 shinelevel;
+        if (dot <= 0) shinelevel = 0; // skip if dot equals 0
+        else
+        {
+            // reuse the dot product from diffuse lighting
+            dot += normaltrans[2];
 
-        if (shinelevel < 0) shinelevel = 0;
-        else if (shinelevel > 0x3FFFF) shinelevel = (0x40000 - shinelevel) & 0x3FFFF;
-        shinelevel = (((s64)shinelevel * shinelevel) >> 26) + (LightDirection[i][2]-0x200>>1); // really (4*shinelevel*shinelevel)+HalfVector[z]
-        if (shinelevel < 0) shinelevel = 0;
-        if (shinelevel > 255) shinelevel = 255; // todo: seems to break when > 511?
+            // mirror around 1024, but in such a manner as to make it bug out at the mirror point
+            if (dot >= 1024) dot = (1024 - (dot - 1024)) & 0x3FF;
+            if (dot < 0) shinelevel = 0; // skip if negative
+            else
+            {
+                s32 recip = (1 << 18) / (-LightDirection[i][2] + (1<<9));
+                // square value, mult by reciprocal, subtract '1'
+                shinelevel = ((dot * dot >> 10) * recip >> 8) - (1<<9);
+
+                // sign extend as if it was a 14 bit integer
+                shinelevel = (shinelevel & 0x3FFF) << 18 >> 18;
+
+                if (shinelevel < 0) shinelevel = 0;
+                else if (shinelevel > 511) shinelevel = 511;
+            }
+        }
 
+        // convert shinelevel to use for lookup in shininess table.
         if (UseShininessTable)
         {
-            // checkme
-            shinelevel >>= 1;
+            shinelevel >>= 2;
             shinelevel = ShininessTable[shinelevel];
+            shinelevel <<= 1;
         }
-        shinelevel <<= 1;
 
         vtxbuff[0] += (MatSpecular[0] * shinelevel +
                       (MatAmbient[0] << 9)) * // ambient seems to be a plain bitshift

From 4a34a544c0ce44053d4e8681dfdf9c56abc11868 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Thu, 22 Feb 2024 19:16:33 -0500
Subject: [PATCH 17/18] fix minor edge case with spec lighting

---
 src/GPU3D.cpp | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 03102a2130..6afd402ae0 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1522,19 +1522,16 @@ void GPU3D::CalculateLighting() noexcept
 
             // mirror around 1024, but in such a manner as to make it bug out at the mirror point
             if (dot >= 1024) dot = (1024 - (dot - 1024)) & 0x3FF;
-            if (dot < 0) shinelevel = 0; // skip if negative
-            else
-            {
-                s32 recip = (1 << 18) / (-LightDirection[i][2] + (1<<9));
-                // square value, mult by reciprocal, subtract '1'
-                shinelevel = ((dot * dot >> 10) * recip >> 8) - (1<<9);
 
-                // sign extend as if it was a 14 bit integer
-                shinelevel = (shinelevel & 0x3FFF) << 18 >> 18;
+            s32 recip = (1 << 18) / (-LightDirection[i][2] + (1<<9));
+            // square value, mult by reciprocal, subtract '1'
+            shinelevel = ((dot * dot >> 10) * recip >> 8) - (1<<9);
 
-                if (shinelevel < 0) shinelevel = 0;
-                else if (shinelevel > 511) shinelevel = 511;
-            }
+            // sign extend to convert to signed 14 bit integer
+            shinelevel = shinelevel << 18 >> 18;
+
+            if (shinelevel < 0) shinelevel = 0;
+            else if (shinelevel > 511) shinelevel = 511;
         }
 
         // convert shinelevel to use for lookup in shininess table.

From def578563a1c083fb64ea4570804a73e18ce272d Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Sun, 26 May 2024 09:35:04 -0400
Subject: [PATCH 18/18] give proper credit in comments

---
 src/GPU3D.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 6afd402ae0..a07a86226e 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -1475,6 +1475,7 @@ void GPU3D::CalculateLighting() noexcept
         if (!(CurPolygonAttr & (1<<i)))
             continue;
 
+        // (credit to azusa for working out most of the details of the diff. algorithm, and essentially the entire spec. algorithm)
         // overflow handling (for example, if the normal length is >1)
         // according to some hardware tests
         // * diffuse level seems to keep going until 1024, at which point it overflows

Places

File melonds_PR1967.patch of Package melonds (Revision dc7f18f7e77a55952fa921cd8dd272e1)

Places