From 61e7a38b33b425bba3af969a60116e7a0cf61dbc Mon Sep 17 00:00:00 2001
From: Matchu <matchu1993@gmail.com>
Date: Wed, 7 Apr 2021 17:25:58 -0700
Subject: [PATCH] Normalize names for Waka, and log mismatches

By printing out this logging data, of item names we have that Waka doesn't, vs item names Waka has that we don't, I was able to solve a lot of them with new code in `normalizeItemName`!

Here's the mismatches that are left at time of writing:

```
[Item: Y, Waka: N] No Waka value for NC DTI item "goldenlulumedallion" (43034)
[Item: Y, Waka: N] No Waka value for NC DTI item "faelliebirthdaybagsurprise" (51350)
[Item: Y, Waka: N] No Waka value for NC DTI item "neopets11thbirthdaycommemorativemysterycapsule" (53448)
[Item: Y, Waka: N] No Waka value for NC DTI item "dyeworksblue:iscawig-blue" (70896)
[Item: Y, Waka: N] No Waka value for NC DTI item "mysteriousdoorwithlocks" (75601)
[Item: Y, Waka: N] No Waka value for NC DTI item "grapefruitnecklace" (77779)
[Item: Y, Waka: N] No Waka value for NC DTI item "discofeverbackground" (79250)
[Item: Y, Waka: N] No Waka value for NC DTI item "featherflaredshoes" (80047)
[Item: Y, Waka: N] No Waka value for NC DTI item "dyeworksredradioactivemutantmarkings" (80441)
[Item: N, Waka: Y] No NC DTI data for Waka item "7thbirthdaycakeslice#1"
[Item: N, Waka: Y] No NC DTI data for Waka item "7thbirthdaycakeslice#2"
[Item: N, Waka: Y] No NC DTI data for Waka item "7thbirthdaycakeslice#3"
[Item: N, Waka: Y] No NC DTI data for Waka item "8thbirthdayrainbowcupcake"
[Item: N, Waka: Y] No NC DTI data for Waka item "8thbirthdaysparklercupcake"
[Item: N, Waka: Y] No NC DTI data for Waka item "8thbirthdaytiedwithabowcupcake"
[Item: N, Waka: Y] No NC DTI data for Waka item "babyspringdress"
[Item: N, Waka: Y] No NC DTI data for Waka item "butterflydress(fromfaeriefestivalevent)"
[Item: N, Waka: Y] No NC DTI data for Waka item "discofever"
[Item: N, Waka: Y] No NC DTI data for Waka item "dyeworksblue:iscawig"
[Item: N, Waka: Y] No NC DTI data for Waka item "dyeworksred:radioactivemutantmarkings"
[Item: N, Waka: Y] No NC DTI data for Waka item "featherflairedshoes"
[Item: N, Waka: Y] No NC DTI data for Waka item "festivebooktree"
[Item: N, Waka: Y] No NC DTI data for Waka item "floralblackcardigan"
[Item: N, Waka: Y] No NC DTI data for Waka item "grapefruitneckace"
[Item: N, Waka: Y] No NC DTI data for Waka item "mysteriousdoorwithlocksbackground"
[Item: N, Waka: Y] No NC DTI data for Waka item "valiantchampionwings"
[Item: N, Waka: Y] No NC DTI data for Waka item "waxcrayonwig"
[Item: N, Waka: Y] No NC DTI data for Waka item "youngsophiesdress"
```
---
 api/allWakaValues.js | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/api/allWakaValues.js b/api/allWakaValues.js
index 5cb9f15..1c50b04 100644
--- a/api/allWakaValues.js
+++ b/api/allWakaValues.js
@@ -30,6 +30,23 @@ async function handle(req, res) {
   for (const { name, id } of allNcItemNamesAndIds) {
     if (name in itemValuesByName) {
       itemValues[id] = itemValuesByName[name];
+    } else {
+      console.warn(
+        `[Item: Y, Waka: N] No Waka value for NC DTI item ${JSON.stringify(
+          name
+        )} (${id})`
+      );
+    }
+  }
+
+  const allNcItemNames = new Set(allNcItemNamesAndIds.map(({ name }) => name));
+  for (const name of Object.keys(itemValuesByName)) {
+    if (!allNcItemNames.has(name)) {
+      console.warn(
+        `[Item: N, Waka: Y] No NC DTI data for Waka item ${JSON.stringify(
+          name
+        )}`
+      );
     }
   }
 
@@ -56,7 +73,7 @@ async function loadAllNcItemNamesAndIds() {
         AND item_translations.locale = "en"
   `);
 
-  return rows;
+  return rows.map(({ id, name }) => ({ id, name: normalizeItemName(name) }));
 }
 
 async function loadWakaValuesByName() {
@@ -96,12 +113,27 @@ async function loadWakaValuesByName() {
   //       That's why we set `""` as the default `value`.
   const itemValuesByName = {};
   for (const [itemName, value = ""] of rows) {
-    itemValuesByName[itemName] = { value };
+    const normalizedItemName = normalizeItemName(itemName);
+    itemValuesByName[normalizedItemName] = { value };
   }
 
   return itemValuesByName;
 }
 
+function normalizeItemName(name) {
+  return (
+    name
+      // Remove all spaces, they're a common source of inconsistency
+      .replace(/\s+/g, "")
+      // Lower case, because capitalization is another common source
+      .toLowerCase()
+      // Remove diacritics: https://stackoverflow.com/a/37511463/107415
+      // Waka has some stray ones in item names, not sure why!
+      .normalize("NFD")
+      .replace(/[\u0300-\u036f]/g, "")
+  );
+}
+
 export default async (req, res) => {
   beeline.withTrace({ name: "allWakaValues" }, () => handle(req, res));
 };