Use latest ~owls NC trade values API

They're moving away from the bulk endpoint to individual item data lookups, so we're updating to match!

package.json

@@ -81,7 +81,8 @@
     "build-cached-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/build-cached-data.js",
     "cache-asset-manifests": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/cache-asset-manifests.js",
     "delete-user": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/delete-user.js",
-    "export-users-to-auth0": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/export-users-to-auth0.js"
+    "export-users-to-auth0": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/export-users-to-auth0.js",
+    "validate-owls-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/validate-owls-data.js"
   },
   "browserslist": {
     "production": [

pages/api/allNCTradeValues.js

@@ -1,3 +1,10 @@
+/**
+ * /api/allNCTradeValues returns all the NC trade values OWLS has!
+ *
+ * NOTE: We no longer use API endpoint as a caching layer for individual item
+ *       data requests. See `nc-trade-values.js` for the new system we use!
+ *       This endpoint is therefore deprecated and might vanish.
+ */
 const beeline = require("honeycomb-beeline")({
   writeKey: process.env["HONEYCOMB_WRITE_KEY"],
   dataset:

scripts/validate-owls-data.js

@@ -0,0 +1,72 @@
+/**
+ * This script compares the data we get from the OWLS bulk endpoint to the data
+ * we get by loading the data with the new individual endpoint! This will help
+ * us check for bugs as we switch over!
+ */
+const fetch = require("node-fetch");
+const connectToDb = require("../src/server/db");
+const buildLoaders = require("../src/server/loaders");
+const { getOWLSTradeValue } = require("../src/server/nc-trade-values");
+
+async function main() {
+  const db = await connectToDb();
+  const { itemTranslationLoader } = buildLoaders(db);
+
+  // Load the bulk data. We're gonna loop through it all!
+  const bulkEndpointRes = await fetch(
+    `http://localhost:3000/api/allNCTradeValues`
+  );
+  const bulkEndpointData = await bulkEndpointRes.json();
+
+  // Load the item names, bc our bulk data is keyed by item ID.
+  const itemIds = Object.keys(bulkEndpointData);
+  const itemTranslations = await itemTranslationLoader.loadMany(itemIds);
+
+  // Load the OWLs data! I don't do any of it in parallel, because I'm worried
+  // about upsetting the server, y'know?
+  let numChecked = 0;
+  let numSuccesses = 0;
+  let numFailures = 0;
+  for (const { name, itemId } of itemTranslations) {
+    if (numChecked % 100 === 0) {
+      console.info(`Checked ${numChecked} items`);
+    }
+    numChecked++;
+    const expectedValueText = bulkEndpointData[itemId].valueText;
+    let actualValue;
+    try {
+      actualValue = await getOWLSTradeValue(name);
+    } catch (error) {
+      console.error(`[${itemId} / ${name}]: Error loading data:\n`, error);
+      numFailures++;
+      continue;
+    }
+    if (actualValue == null) {
+      console.error(`[${itemId} / ${name}]: No value found.`);
+      numFailures++;
+      continue;
+    }
+    const actualValueText = actualValue.valueText;
+    if (expectedValueText !== actualValueText) {
+      console.error(
+        `[${itemId}]: Value did not match. ` +
+          `Expected: ${JSON.stringify(expectedValueText)}. ` +
+          `Actual: ${JSON.stringify(actualValueText)}`
+      );
+      numFailures++;
+      continue;
+    }
+    numSuccesses++;
+  }
+  console.info(
+    `Checked all ${numChecked} items. ` +
+      `${numSuccesses} successes, ${numFailures} failures.`
+  );
+}
+
+main()
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  })
+  .then(() => process.exit());

src/server/loaders.js

@@ -1,5 +1,4 @@
 import DataLoader from "dataloader";
-import fetch from "node-fetch";
 import { normalizeRow } from "./util";
 
 const buildClosetListLoader = (db) =>
@@ -826,31 +825,6 @@ const buildItemTradesLoader = (db, loaders) =>
     { cacheKeyFn: ({ itemId, isOwned }) => `${itemId}-${isOwned}` }
   );
 
-const buildItemNCTradeValueLoader = () =>
-  new DataLoader(async (itemIds) => {
-    // This loader calls our /api/allNCTradeValues endpoint, to take advantage
-    // of the CDN caching. This helps us respond a bit faster than calling the
-    // API directly would, and avoids putting network pressure or caching
-    // complexity on our ~owls friends! (It would also be pretty reasonable to
-    // do this as a process-level cache or something instead, but I'm reusing
-    // Waka code from when we were on a more distributed system where that
-    // wouldn't have worked out, and I don't think the effort to refactor this
-    // just for the potential perf win is worthy!)
-    const url = process.env.NODE_ENV === "production"
-      ? "https://impress-2020.openneo.net/api/allNCTradeValues"
-      : "http://localhost:3000/api/allNCTradeValues";
-    const res = await fetch(url);
-    if (!res.ok) {
-      throw new Error(
-        `Error loading /api/allNCTradeValues: ${res.status} ${res.statusText}`
-      );
-    }
-
-    const allNCTradeValues = await res.json();
-
-    return itemIds.map((itemId) => allNCTradeValues[itemId]);
-  });
-
 const buildPetTypeLoader = (db, loaders) =>
   new DataLoader(async (petTypeIds) => {
     const qs = petTypeIds.map((_) => "?").join(",");
@@ -1521,7 +1495,6 @@ function buildLoaders(db) {
     db
   );
   loaders.itemTradesLoader = buildItemTradesLoader(db, loaders);
-  loaders.itemNCTradeValueLoader = buildItemNCTradeValueLoader();
   loaders.petTypeLoader = buildPetTypeLoader(db, loaders);
   loaders.petTypeBySpeciesAndColorLoader = buildPetTypeBySpeciesAndColorLoader(
     db,

src/server/nc-trade-values.js

@@ -0,0 +1,48 @@
+import LRU from "lru-cache";
+import fetch from "node-fetch";
+
+// NOTE: I didn't validate any of these cache settings very carefully, just
+//       that the cache works basically at all. I figure if it's misconfigured
+//       in a way that causes stale data or memory issues, we'll discover that
+//       when it becomes a problem!
+const owlsTradeValueCache = new LRU({
+  // Cache up to 500 entries (they're small!), for 15 minutes each. (The 15min
+  // cap should keep the cache much smaller than that in practice I think!)
+  max: 500,
+  ttl: 1000 * 60 * 15,
+
+  // We also enforce a ~5MB total limit, just to make sure some kind of issue
+  // in the API communication won't cause huge memory leaks. (Size in memory is
+  // approximated as the length of the key string and the length of the value
+  // object in JSON. Not exactly accurate, but very close!)
+  maxSize: 5_000_000,
+  sizeCalculation: (value, key) => JSON.stringify(value).length + key.length,
+});
+
+export async function getOWLSTradeValue(itemName) {
+  const cachedValue = owlsTradeValueCache.get(itemName);
+  if (cachedValue != null) {
+    console.debug("[getOWLSTradeValue] Serving cached value", cachedValue);
+    return cachedValue;
+  }
+
+  const newValue = await loadOWLSTradeValueFromAPI(itemName);
+  owlsTradeValueCache.set(itemName, newValue);
+  return newValue;
+}
+
+async function loadOWLSTradeValueFromAPI(itemName) {
+  const res = await fetch(
+    `https://neo-owls.net/itemdata/${encodeURIComponent(itemName)}`
+  );
+  if (!res.ok) {
+    // TODO: Differentiate between 500 and 404. (Right now, when the item isn't
+    // found, it returns a 500, so it's hard to say.)
+    return null;
+  }
+  const data = await res.json();
+  return {
+    valueText: data.owls_value,
+    lastUpdated: data.last_updated,
+  };
+}

src/server/types/Item.js

@@ -1,4 +1,5 @@
 import { gql } from "apollo-server";
+import { getOWLSTradeValue } from "../nc-trade-values";
 import {
   getRestrictedZoneIds,
   normalizeRow,
@@ -332,9 +333,7 @@ const resolvers = {
     },
     isNc: async ({ id }, _, { itemLoader }) => {
       const item = await itemLoader.load(id);
-      return (
+      return isNC(item);
-        item.rarityIndex === 500 || item.rarityIndex === 0 || item.isManuallyNc
-      );
     },
     isPb: async ({ id }, _, { itemTranslationLoader }) => {
       const translation = await itemTranslationLoader.load(id);
@@ -356,10 +355,31 @@ const resolvers = {
       // This feature is deprecated, so now we just always return unknown value.
       return null;
     },
-    ncTradeValueText: async ({ id }, _, { itemNCTradeValueLoader }) => {
+    ncTradeValueText: async (
+      { id },
+      _,
+      { itemLoader, itemTranslationLoader }
+    ) => {
+      // Skip this lookup for non-NC items, as a perf optimization.
+      const item = await itemLoader.load(id);
+      if (!isNC(item)) {
+        return;
+      }
+
+      // Get the item name, which is how we look things up in ~owls.
+      const itemTranslation = await itemTranslationLoader.load(id);
+      let itemName = itemTranslation.name;
+
+      // HACK: The name "Butterfly Dress" is used for two different items.
+      //       Here's what ~owls does to distinguish!
+      if (id === "76073") {
+        itemName = "Butterfly Dress (from Faerie Festival event)";
+      }
+
+      // Load the NC trade value from ~owls, if any.
       let ncTradeValue;
       try {
-        ncTradeValue = await itemNCTradeValueLoader.load(id);
+        ncTradeValue = await getOWLSTradeValue(itemName);
       } catch (e) {
         console.error(
           `Error loading ncTradeValueText for item ${id}, skipping:`
@@ -368,6 +388,7 @@ const resolvers = {
         ncTradeValue = null;
       }
 
+      // If there was a value, get the text. If not, return null.
       return ncTradeValue ? ncTradeValue.valueText : null;
     },
 
@@ -1225,4 +1246,10 @@ async function loadClosetListOrDefaultList(listId, closetListLoader) {
   return null;
 }
 
+function isNC(item) {
+  return (
+    item.rarityIndex === 500 || item.rarityIndex === 0 || item.isManuallyNc
+  );
+}
+
 module.exports = { typeDefs, resolvers };