From 16b86fc65e79a3148483808057cfeee4f1ddf7ed Mon Sep 17 00:00:00 2001 From: Matchu Date: Mon, 15 Aug 2022 18:39:29 -0700 Subject: [PATCH 1/3] Playing with using OWLS Pricer data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This isn't a partnership we've actually talked through with the team, I'm just validating whether we could reuse our Waka code if it were to come up! and playing with it for fun ๐Ÿ˜Š --- pages/api/allNCTradeValues.js | 121 ++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 pages/api/allNCTradeValues.js diff --git a/pages/api/allNCTradeValues.js b/pages/api/allNCTradeValues.js new file mode 100644 index 0000000..06381e5 --- /dev/null +++ b/pages/api/allNCTradeValues.js @@ -0,0 +1,121 @@ +const beeline = require("honeycomb-beeline")({ + writeKey: process.env["HONEYCOMB_WRITE_KEY"], + dataset: + process.env["NODE_ENV"] === "production" + ? "Dress to Impress (2020)" + : "Dress to Impress (2020, dev)", + serviceName: "impress-2020-gql-server", +}); + +import fetch from "node-fetch"; + +import connectToDb from "../../src/server/db"; + +async function handle(req, res) { + const allNcItemNamesAndIdsPromise = loadAllNcItemNamesAndIds(); + + let itemValuesByIdOrName; + try { + itemValuesByIdOrName = await loadOWLSValuesByIdOrName(); + } catch (e) { + console.error(e); + res.setHeader("Content-Type", "text/plain; charset=utf8"); + res.status(500).send("Error loading OWLS Pricer data"); + return; + } + + // Restructure the value data to use IDs as keys, instead of names. + const allNcItemNamesAndIds = await allNcItemNamesAndIdsPromise; + const itemValues = {}; + for (const { name, id } of allNcItemNamesAndIds) { + if (id in itemValuesByIdOrName) { + itemValues[id] = itemValuesByIdOrName[id]; + } else if (name in itemValuesByIdOrName) { + itemValues[id] = itemValuesByIdOrName[name]; + } + } + + // Cache for 1 minute, and immediately serve stale data for a day after. + // This should keep it fast and responsive, and stay well within our API key + // limits. (This will cause the client to send more requests than necessary, + // but the CDN cache should generally respond quickly with a small 304 Not + // Modified, unless the data really did change.) + res.setHeader( + "Cache-Control", + "public, max-age=3600, stale-while-revalidate=86400" + ); + return res.send(itemValues); +} + +async function loadAllNcItemNamesAndIds() { + const db = await connectToDb(); + + const [rows] = await db.query(` + SELECT items.id, item_translations.name FROM items + INNER JOIN item_translations ON item_translations.item_id = items.id + WHERE + (items.rarity_index IN (0, 500) OR is_manually_nc = 1) + AND item_translations.locale = "en" + `); + + return rows.map(({ id, name }) => ({ id, name: normalizeItemName(name) })); +} + +/** + * Load all OWLS Pricer values from the spreadsheet. Returns an object keyed by + * ID or name - that is, if the item ID is provided, we use that as the key; or + * if not, we use the name as the key. + */ +async function loadOWLSValuesByIdOrName() { + const res = await fetch( + `https://neo-owls.herokuapp.com/itemdata/owls_script/` + ); + const json = await res.json(); + + if (!res.ok) { + throw new Error( + `Could not load OWLS Pricer data: ${res.status} ${res.statusText}` + ); + } + + const itemValuesByIdOrName = {}; + for (const [itemName, value] of Object.entries(json)) { + // OWLS returns an empty string for NC Mall items they don't have a trade + // value for, to allow the script to distinguish between NP items vs + // no-data NC items. We omit it from our data instead, because our UI is + // already aware of whether the item is NP or NC. + if (value.trim() === "") { + continue; + } + + // TODO: OWLS doesn't currently provide item IDs ever. Add support for it + // if it does! (I'm keeping the rest of the code the same because I + // think that might happen for disambiguation, like Waka did.) + const normalizedItemName = normalizeItemName(itemName); + itemValuesByIdOrName[normalizedItemName] = value; + } + + return itemValuesByIdOrName; +} + +function normalizeItemName(name) { + return ( + name + // Remove all spaces, they're a common source of inconsistency + .replace(/\s+/g, "") + // Lower case, because capitalization is another common source + .toLowerCase() + // Remove diacritics: https://stackoverflow.com/a/37511463/107415 + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + ); +} + +async function handleWithBeeline(req, res) { + beeline.withTrace( + { name: "api/allNCTradeValues", operation_name: "api/allNCTradeValues" }, + () => handle(req, res) + ); +} + +export default handleWithBeeline; From 240a683e719d7f57d76d5a29dcba2f9fc205c48d Mon Sep 17 00:00:00 2001 From: Matchu Date: Mon, 15 Aug 2022 23:57:33 -0700 Subject: [PATCH 2/3] Finish wiring up ~owls data Hey nice it looks like it's working! :3 "Bright Speckled Parasol" is a nice test case, it has a long text string! And when the NC value is not included in the ~owls list, we indeed don't show the badge! --- pages/api/allNCTradeValues.js | 14 +++-- src/app/ItemPage.js | 12 +++- src/app/ItemPageLayout.js | 102 +++------------------------------- src/app/ItemTradesPage.js | 2 +- src/server/loaders.js | 27 +++++++++ src/server/types/Item.js | 26 +++++++++ 6 files changed, 81 insertions(+), 102 deletions(-) diff --git a/pages/api/allNCTradeValues.js b/pages/api/allNCTradeValues.js index 06381e5..3112776 100644 --- a/pages/api/allNCTradeValues.js +++ b/pages/api/allNCTradeValues.js @@ -79,20 +79,26 @@ async function loadOWLSValuesByIdOrName() { } const itemValuesByIdOrName = {}; - for (const [itemName, value] of Object.entries(json)) { + for (const [itemName, valueText] of Object.entries(json)) { // OWLS returns an empty string for NC Mall items they don't have a trade // value for, to allow the script to distinguish between NP items vs // no-data NC items. We omit it from our data instead, because our UI is // already aware of whether the item is NP or NC. - if (value.trim() === "") { + if (valueText.trim() === "") { continue; } // TODO: OWLS doesn't currently provide item IDs ever. Add support for it // if it does! (I'm keeping the rest of the code the same because I - // think that might happen for disambiguation, like Waka did.) + // think that might happen for disambiguation, like Waka did.) Until + // then, we just always key by name. const normalizedItemName = normalizeItemName(itemName); - itemValuesByIdOrName[normalizedItemName] = value; + + // We wrap it in an object with the key `valueText`, just to not break + // potential external consumers of this endpoint if we add more fields. + // (This is kinda silly and unnecessary, but it should get gzipped out and + // shouldn't add substantial time to building or parsing, so like w/e!) + itemValuesByIdOrName[normalizedItemName] = { valueText }; } return itemValuesByIdOrName; diff --git a/src/app/ItemPage.js b/src/app/ItemPage.js index e759010..ae6e6b7 100644 --- a/src/app/ItemPage.js +++ b/src/app/ItemPage.js @@ -36,7 +36,13 @@ import { useQuery, useMutation } from "@apollo/client"; import { Link, useParams } from "react-router-dom"; import ItemPageLayout, { SubtleSkeleton } from "./ItemPageLayout"; -import { Delay, logAndCapture, useLocalStorage, usePageTitle } from "./util"; +import { + Delay, + logAndCapture, + MajorErrorMessage, + useLocalStorage, + usePageTitle, +} from "./util"; import HTML5Badge, { layerUsesHTML5 } from "./components/HTML5Badge"; import { itemAppearanceFragment, @@ -77,7 +83,7 @@ export function ItemPageContent({ itemId, isEmbedded }) { thumbnailUrl description createdAt - wakaValueText + ncTradeValueText # For Support users. rarityIndex @@ -91,7 +97,7 @@ export function ItemPageContent({ itemId, isEmbedded }) { usePageTitle(data?.item?.name, { skip: isEmbedded }); if (error) { - return {error.message}; + return ; } const item = data?.item; diff --git a/src/app/ItemPageLayout.js b/src/app/ItemPageLayout.js index 562af96..e393a89 100644 --- a/src/app/ItemPageLayout.js +++ b/src/app/ItemPageLayout.js @@ -5,7 +5,6 @@ import { Flex, Popover, PopoverArrow, - PopoverBody, PopoverContent, PopoverTrigger, Portal, @@ -16,12 +15,7 @@ import { useToast, VStack, } from "@chakra-ui/react"; -import { - ExternalLinkIcon, - ChevronRightIcon, - QuestionIcon, - WarningTwoIcon, -} from "@chakra-ui/icons"; +import { ExternalLinkIcon, ChevronRightIcon } from "@chakra-ui/icons"; import { gql, useMutation } from "@apollo/client"; import { @@ -159,44 +153,15 @@ function ItemPageBadges({ item, isEmbedded }) { {item.isNc && ( - {shouldShowWaka() && item.wakaValueText && ( - <> - {/* For hover-y devices, use a hover popover over the badge. */} - - - - - Waka: {item.wakaValueText} - - - - {/* For touch-y devices, use a tappable help icon. */} - - - - Waka: {item.wakaValueText} - - - - - - - - + {item.ncTradeValueText && ( + + OWLS: {item.ncTradeValueText} + )} )} @@ -439,55 +404,4 @@ function ShortTimestamp({ when }) { ); } -function WakaPopover({ children, ...props }) { - return ( - - {children} - - - - -

- - The Waka Guide for NC trade values is closing down! - {" "} - We're sad to see them go, but excited that the team gets to move - onto the next phase in their life ๐Ÿ’– -

- -

- The Waka guide was last updated August 7, 2021, so this value - might not be accurate anymore. Consider checking in with the - Neoboards! -

- -

- Thanks again to the Waka team for letting us experiment with - sharing their trade values here. Best wishes for everything to - come! ๐Ÿ’œ -

- - - - - ); -} - -// August 21, 2021. (The month uses 0-indexing, but nothing else does! ๐Ÿ™ƒ) -const STOP_SHOWING_WAKA_AFTER = new Date(2021, 7, 21, 0, 0, 0, 0); - -/** - * shouldShowWaka returns true if, according to the browser, it's not yet - * August 21, 2021. It starts returning false at midnight on Aug 21. - * - * That way, our Waka deprecation message is on an auto-timer. After Aug 21, - * it's safe to remove all Waka UI code, and the Waka API endpoint and GraphQL - * fields. (It might be kind to return a placeholder string for the GraphQL - * case!) - */ -function shouldShowWaka() { - const now = new Date(); - return now < STOP_SHOWING_WAKA_AFTER; -} - export default ItemPageLayout; diff --git a/src/app/ItemTradesPage.js b/src/app/ItemTradesPage.js index 953956f..c3c8359 100644 --- a/src/app/ItemTradesPage.js +++ b/src/app/ItemTradesPage.js @@ -102,7 +102,7 @@ function ItemTradesPage({ thumbnailUrl description createdAt - wakaValueText + ncTradeValueText } } `, diff --git a/src/server/loaders.js b/src/server/loaders.js index 44703ca..133a567 100644 --- a/src/server/loaders.js +++ b/src/server/loaders.js @@ -1,4 +1,5 @@ import DataLoader from "dataloader"; +import fetch from "node-fetch"; import { normalizeRow } from "./util"; const buildClosetListLoader = (db) => @@ -825,6 +826,31 @@ const buildItemTradesLoader = (db, loaders) => { cacheKeyFn: ({ itemId, isOwned }) => `${itemId}-${isOwned}` } ); +const buildItemNCTradeValueLoader = () => + new DataLoader(async (itemIds) => { + // This loader calls our /api/allNCTradeValues endpoint, to take advantage + // of the CDN caching. This helps us respond a bit faster than calling the + // API directly would, and avoids putting network pressure or caching + // complexity on our ~owls friends! (It would also be pretty reasonable to + // do this as a process-level cache or something instead, but I'm reusing + // Waka code from when we were on a more distributed system where that + // wouldn't have worked out, and I don't think the effort to refactor this + // just for the potential perf win is worthy!) + const url = process.env.NODE_ENV === "production" + ? "https://impress-2020.openneo.net/api/allNCTradeValues" + : "http://localhost:3000/api/allNCTradeValues"; + const res = await fetch(url); + if (!res.ok) { + throw new Error( + `Error loading /api/allNCTradeValues: ${res.status} ${res.statusText}` + ); + } + + const allNCTradeValues = await res.json(); + + return itemIds.map((itemId) => allNCTradeValues[itemId]); + }); + const buildPetTypeLoader = (db, loaders) => new DataLoader(async (petTypeIds) => { const qs = petTypeIds.map((_) => "?").join(","); @@ -1495,6 +1521,7 @@ function buildLoaders(db) { db ); loaders.itemTradesLoader = buildItemTradesLoader(db, loaders); + loaders.itemNCTradeValueLoader = buildItemNCTradeValueLoader(); loaders.petTypeLoader = buildPetTypeLoader(db, loaders); loaders.petTypeBySpeciesAndColorLoader = buildPetTypeBySpeciesAndColorLoader( db, diff --git a/src/server/types/Item.js b/src/server/types/Item.js index 343ec21..8ba162f 100644 --- a/src/server/types/Item.js +++ b/src/server/types/Item.js @@ -33,6 +33,18 @@ const typeDefs = gql` """ wakaValueText: String @cacheControl(maxAge: ${oneHour}) + """ + This item's NC trade value as a human-readable string. Returns null if the + value is not known. + + Note that the format of this string is not well-specifiedโ€”it's fully + human-curated and may include surprising words or extra notes! We recommend + presenting the text exactly as-is, rather than trying to parse and math it. + + This data is currently curated by neopets.com/~owls, thank you!! <3 + """ + ncTradeValueText: String @cacheControl(maxAge: ${oneHour}) + currentUserOwnsThis: Boolean! @cacheControl(maxAge: 0, scope: PRIVATE) currentUserWantsThis: Boolean! @cacheControl(maxAge: 0, scope: PRIVATE) @@ -344,6 +356,20 @@ const resolvers = { // This feature is deprecated, so now we just always return unknown value. return null; }, + ncTradeValueText: async ({ id }, _, { itemNCTradeValueLoader }) => { + let ncTradeValue; + try { + ncTradeValue = await itemNCTradeValueLoader.load(id); + } catch (e) { + console.error( + `Error loading ncTradeValueText for item ${id}, skipping:` + ); + console.error(e); + ncTradeValue = null; + } + + return ncTradeValue ? ncTradeValue.valueText : null; + }, currentUserOwnsThis: async ( { id }, From a8b48329765b88c6047fcb1aa2e97f8173b728a1 Mon Sep 17 00:00:00 2001 From: Matchu Date: Tue, 16 Aug 2022 00:12:58 -0700 Subject: [PATCH 3/3] Hide the ~owls badge until you type "~owls" Just a cute little way to let us preview it without having to spin up a separate instance of the app or use a feature flag system! This means we can safely merge and push this to production, without worrying about leaking the feature before the ~owls team signs off. --- src/app/ItemPageLayout.js | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/app/ItemPageLayout.js b/src/app/ItemPageLayout.js index e393a89..38577bb 100644 --- a/src/app/ItemPageLayout.js +++ b/src/app/ItemPageLayout.js @@ -105,6 +105,7 @@ export function SubtleSkeleton({ isLoaded, ...props }) { function ItemPageBadges({ item, isEmbedded }) { const searchBadgesAreLoaded = item?.name != null && item?.isNc != null; + const shouldShowOwls = useShouldShowOwls(); return ( @@ -150,7 +151,7 @@ function ItemPageBadges({ item, isEmbedded }) { Jellyneo - {item.isNc && ( + {item.isNc && shouldShowOwls && ( { + const onKeyPress = (e) => { + const newMostRecentKeys = [...mostRecentKeys, e.key].slice(-5); + if (newMostRecentKeys.join("") === "~owls") { + SHOULD_SHOW_OWLS = true; + setShouldShowOwls(true); + } + setMostRecentKeys(newMostRecentKeys); + }; + + window.addEventListener("keypress", onKeyPress); + return () => window.removeEventListener("keypress", onKeyPress); + }); + + return shouldShowOwls; +} + export default ItemPageLayout;