diff --git a/package.json b/package.json index 77855d5..d925c90 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,8 @@ "build-cached-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/build-cached-data.js", "cache-asset-manifests": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/cache-asset-manifests.js", "delete-user": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/delete-user.js", - "export-users-to-auth0": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/export-users-to-auth0.js" + "export-users-to-auth0": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/export-users-to-auth0.js", + "validate-owls-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/validate-owls-data.js" }, "browserslist": { "production": [ diff --git a/pages/api/allNCTradeValues.js b/pages/api/allNCTradeValues.js index e5ab75c..3b5a1c5 100644 --- a/pages/api/allNCTradeValues.js +++ b/pages/api/allNCTradeValues.js @@ -1,3 +1,10 @@ +/** + * /api/allNCTradeValues returns all the NC trade values OWLS has! + * + * NOTE: We no longer use API endpoint as a caching layer for individual item + * data requests. See `nc-trade-values.js` for the new system we use! + * This endpoint is therefore deprecated and might vanish. + */ const beeline = require("honeycomb-beeline")({ writeKey: process.env["HONEYCOMB_WRITE_KEY"], dataset: diff --git a/scripts/validate-owls-data.js b/scripts/validate-owls-data.js new file mode 100644 index 0000000..cad73f1 --- /dev/null +++ b/scripts/validate-owls-data.js @@ -0,0 +1,72 @@ +/** + * This script compares the data we get from the OWLS bulk endpoint to the data + * we get by loading the data with the new individual endpoint! This will help + * us check for bugs as we switch over! + */ +const fetch = require("node-fetch"); +const connectToDb = require("../src/server/db"); +const buildLoaders = require("../src/server/loaders"); +const { getOWLSTradeValue } = require("../src/server/nc-trade-values"); + +async function main() { + const db = await connectToDb(); + const { itemTranslationLoader } = buildLoaders(db); + + // Load the bulk data. We're gonna loop through it all! + const bulkEndpointRes = await fetch( + `http://localhost:3000/api/allNCTradeValues` + ); + const bulkEndpointData = await bulkEndpointRes.json(); + + // Load the item names, bc our bulk data is keyed by item ID. + const itemIds = Object.keys(bulkEndpointData); + const itemTranslations = await itemTranslationLoader.loadMany(itemIds); + + // Load the OWLs data! I don't do any of it in parallel, because I'm worried + // about upsetting the server, y'know? + let numChecked = 0; + let numSuccesses = 0; + let numFailures = 0; + for (const { name, itemId } of itemTranslations) { + if (numChecked % 100 === 0) { + console.info(`Checked ${numChecked} items`); + } + numChecked++; + const expectedValueText = bulkEndpointData[itemId].valueText; + let actualValue; + try { + actualValue = await getOWLSTradeValue(name); + } catch (error) { + console.error(`[${itemId} / ${name}]: Error loading data:\n`, error); + numFailures++; + continue; + } + if (actualValue == null) { + console.error(`[${itemId} / ${name}]: No value found.`); + numFailures++; + continue; + } + const actualValueText = actualValue.valueText; + if (expectedValueText !== actualValueText) { + console.error( + `[${itemId}]: Value did not match. ` + + `Expected: ${JSON.stringify(expectedValueText)}. ` + + `Actual: ${JSON.stringify(actualValueText)}` + ); + numFailures++; + continue; + } + numSuccesses++; + } + console.info( + `Checked all ${numChecked} items. ` + + `${numSuccesses} successes, ${numFailures} failures.` + ); +} + +main() + .catch((e) => { + console.error(e); + process.exit(1); + }) + .then(() => process.exit()); diff --git a/src/server/loaders.js b/src/server/loaders.js index 133a567..44703ca 100644 --- a/src/server/loaders.js +++ b/src/server/loaders.js @@ -1,5 +1,4 @@ import DataLoader from "dataloader"; -import fetch from "node-fetch"; import { normalizeRow } from "./util"; const buildClosetListLoader = (db) => @@ -826,31 +825,6 @@ const buildItemTradesLoader = (db, loaders) => { cacheKeyFn: ({ itemId, isOwned }) => `${itemId}-${isOwned}` } ); -const buildItemNCTradeValueLoader = () => - new DataLoader(async (itemIds) => { - // This loader calls our /api/allNCTradeValues endpoint, to take advantage - // of the CDN caching. This helps us respond a bit faster than calling the - // API directly would, and avoids putting network pressure or caching - // complexity on our ~owls friends! (It would also be pretty reasonable to - // do this as a process-level cache or something instead, but I'm reusing - // Waka code from when we were on a more distributed system where that - // wouldn't have worked out, and I don't think the effort to refactor this - // just for the potential perf win is worthy!) - const url = process.env.NODE_ENV === "production" - ? "https://impress-2020.openneo.net/api/allNCTradeValues" - : "http://localhost:3000/api/allNCTradeValues"; - const res = await fetch(url); - if (!res.ok) { - throw new Error( - `Error loading /api/allNCTradeValues: ${res.status} ${res.statusText}` - ); - } - - const allNCTradeValues = await res.json(); - - return itemIds.map((itemId) => allNCTradeValues[itemId]); - }); - const buildPetTypeLoader = (db, loaders) => new DataLoader(async (petTypeIds) => { const qs = petTypeIds.map((_) => "?").join(","); @@ -1521,7 +1495,6 @@ function buildLoaders(db) { db ); loaders.itemTradesLoader = buildItemTradesLoader(db, loaders); - loaders.itemNCTradeValueLoader = buildItemNCTradeValueLoader(); loaders.petTypeLoader = buildPetTypeLoader(db, loaders); loaders.petTypeBySpeciesAndColorLoader = buildPetTypeBySpeciesAndColorLoader( db, diff --git a/src/server/nc-trade-values.js b/src/server/nc-trade-values.js new file mode 100644 index 0000000..929656d --- /dev/null +++ b/src/server/nc-trade-values.js @@ -0,0 +1,48 @@ +import LRU from "lru-cache"; +import fetch from "node-fetch"; + +// NOTE: I didn't validate any of these cache settings very carefully, just +// that the cache works basically at all. I figure if it's misconfigured +// in a way that causes stale data or memory issues, we'll discover that +// when it becomes a problem! +const owlsTradeValueCache = new LRU({ + // Cache up to 500 entries (they're small!), for 15 minutes each. (The 15min + // cap should keep the cache much smaller than that in practice I think!) + max: 500, + ttl: 1000 * 60 * 15, + + // We also enforce a ~5MB total limit, just to make sure some kind of issue + // in the API communication won't cause huge memory leaks. (Size in memory is + // approximated as the length of the key string and the length of the value + // object in JSON. Not exactly accurate, but very close!) + maxSize: 5_000_000, + sizeCalculation: (value, key) => JSON.stringify(value).length + key.length, +}); + +export async function getOWLSTradeValue(itemName) { + const cachedValue = owlsTradeValueCache.get(itemName); + if (cachedValue != null) { + console.debug("[getOWLSTradeValue] Serving cached value", cachedValue); + return cachedValue; + } + + const newValue = await loadOWLSTradeValueFromAPI(itemName); + owlsTradeValueCache.set(itemName, newValue); + return newValue; +} + +async function loadOWLSTradeValueFromAPI(itemName) { + const res = await fetch( + `https://neo-owls.net/itemdata/${encodeURIComponent(itemName)}` + ); + if (!res.ok) { + // TODO: Differentiate between 500 and 404. (Right now, when the item isn't + // found, it returns a 500, so it's hard to say.) + return null; + } + const data = await res.json(); + return { + valueText: data.owls_value, + lastUpdated: data.last_updated, + }; +} diff --git a/src/server/types/Item.js b/src/server/types/Item.js index 8ba162f..04f6fbf 100644 --- a/src/server/types/Item.js +++ b/src/server/types/Item.js @@ -1,4 +1,5 @@ import { gql } from "apollo-server"; +import { getOWLSTradeValue } from "../nc-trade-values"; import { getRestrictedZoneIds, normalizeRow, @@ -332,9 +333,7 @@ const resolvers = { }, isNc: async ({ id }, _, { itemLoader }) => { const item = await itemLoader.load(id); - return ( - item.rarityIndex === 500 || item.rarityIndex === 0 || item.isManuallyNc - ); + return isNC(item); }, isPb: async ({ id }, _, { itemTranslationLoader }) => { const translation = await itemTranslationLoader.load(id); @@ -356,10 +355,31 @@ const resolvers = { // This feature is deprecated, so now we just always return unknown value. return null; }, - ncTradeValueText: async ({ id }, _, { itemNCTradeValueLoader }) => { + ncTradeValueText: async ( + { id }, + _, + { itemLoader, itemTranslationLoader } + ) => { + // Skip this lookup for non-NC items, as a perf optimization. + const item = await itemLoader.load(id); + if (!isNC(item)) { + return; + } + + // Get the item name, which is how we look things up in ~owls. + const itemTranslation = await itemTranslationLoader.load(id); + let itemName = itemTranslation.name; + + // HACK: The name "Butterfly Dress" is used for two different items. + // Here's what ~owls does to distinguish! + if (id === "76073") { + itemName = "Butterfly Dress (from Faerie Festival event)"; + } + + // Load the NC trade value from ~owls, if any. let ncTradeValue; try { - ncTradeValue = await itemNCTradeValueLoader.load(id); + ncTradeValue = await getOWLSTradeValue(itemName); } catch (e) { console.error( `Error loading ncTradeValueText for item ${id}, skipping:` @@ -368,6 +388,7 @@ const resolvers = { ncTradeValue = null; } + // If there was a value, get the text. If not, return null. return ncTradeValue ? ncTradeValue.valueText : null; }, @@ -1225,4 +1246,10 @@ async function loadClosetListOrDefaultList(listId, closetListLoader) { return null; } +function isNC(item) { + return ( + item.rarityIndex === 500 || item.rarityIndex === 0 || item.isManuallyNc + ); +} + module.exports = { typeDefs, resolvers };