From e20364cc0a06f45aa9f27bf242b6bb3917c5ed82 Mon Sep 17 00:00:00 2001 From: Matchu Date: Mon, 17 Aug 2020 18:23:39 -0700 Subject: [PATCH] create cache-asset-manifests script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just gonna bulk load all those manifests into the db, and then that should make most loads notably faster by removing the net request! 🤞 We'll still load manifests inline sometimes, but only the first time anyone pulls up the layer in impress-2020. After that, it should be cached forever! --- package.json | 6 +- .../build-cached-data.js | 7 +- scripts/cache-asset-manifests.js | 91 +++++++++++++++++++ yarn.lock | 5 + 4 files changed, 104 insertions(+), 5 deletions(-) rename build-cached-data.js => scripts/build-cached-data.js (82%) create mode 100644 scripts/cache-asset-manifests.js diff --git a/package.json b/package.json index 703653f..cd0974a 100644 --- a/package.json +++ b/package.json @@ -33,13 +33,14 @@ }, "scripts": { "start": "yarn build-cached-data && react-app-rewired start", - "build-cached-data": "node -r dotenv/config build-cached-data.js", + "build-cached-data": "node -r dotenv/config scripts/build-cached-data.js", "build": "react-app-rewired build && yarn build-cached-data", "test": "react-app-rewired test --env=jsdom", "eject": "react-scripts eject", "setup-mysql-user": "mysql -h impress.openneo.net -u matchu -p < setup-mysql-user.sql", "mysql": "mysql --host=impress.openneo.net --user=$(dotenv -p IMPRESS_MYSQL_USER) --password=$(dotenv -p IMPRESS_MYSQL_PASSWORD) --database=openneo_impress", - "mysql-admin": "mysql --host=impress.openneo.net --user=matchu --password --database=openneo_impress" + "mysql-admin": "mysql --host=impress.openneo.net --user=matchu --password --database=openneo_impress", + "cache-asset-manifests": "node -r dotenv/config scripts/cache-asset-manifests.js" }, "eslintConfig": { "extends": "react-app" @@ -61,6 +62,7 @@ "customize-cra": "^1.0.0", "customize-cra-react-refresh": "^1.1.0", "dotenv-cli": "^3.1.0", + "es6-promise-pool": "^2.5.0", "prettier": "^2.0.5", "react-app-rewired": "^2.1.6" } diff --git a/build-cached-data.js b/scripts/build-cached-data.js similarity index 82% rename from build-cached-data.js rename to scripts/build-cached-data.js index 2bf59a4..6d78c56 100644 --- a/build-cached-data.js +++ b/scripts/build-cached-data.js @@ -1,10 +1,11 @@ +// We run this on build to cache some stable database tables on the server! const fs = require("fs").promises; const path = require("path"); -const connectToDb = require("./src/server/db"); -const { normalizeRow } = require("./src/server/util"); +const connectToDb = require("../src/server/db"); +const { normalizeRow } = require("../src/server/util"); -const cachedDataPath = path.join(__dirname, "build", "cached-data"); +const cachedDataPath = path.join(__dirname, "..", "build", "cached-data"); async function buildZonesCache(db) { const [rows] = await db.query(`SELECT * FROM zones;`); diff --git a/scripts/cache-asset-manifests.js b/scripts/cache-asset-manifests.js new file mode 100644 index 0000000..df3a3c8 --- /dev/null +++ b/scripts/cache-asset-manifests.js @@ -0,0 +1,91 @@ +// This is a big bulk script to load the asset manifest from images.neopets.com +// for every asset, and save it to the database for fast loading! +// +// The site works fine without this: when it runs into an asset where we don't +// have the manifest cached, it loads it and caches it in real time. But this +// is a nice way to warm things up to get started! +// +// We shouldn't have to run this regularly in general, but we might want to +// re-run it once Neopets adds more manifests. Right now, we save an empty +// placeholder when no manifest exists, but someday we want to fill it in +// instead! +const PromisePool = require("es6-promise-pool"); + +const connectToDb = require("../src/server/db"); +const neopets = require("../src/server/neopets"); + +async function cacheAssetManifests(db) { + const [rows] = await db.query( + `SELECT id, url FROM swf_assets WHERE manifest IS NULL` + ); + + const numRowsTotal = rows.length; + let numRowsStarted = 0; + let numRowsDone = 0; + + async function cacheAssetManifest(row) { + try { + let manifest = await neopets.loadAssetManifest(row.url); + + // After loading, write the new manifest. We make sure to write an empty + // string if there was no manifest, to signify that it doesn't exist, so + // we don't need to bother looking it up again. + // + // TODO: Someday the manifests will all exist, right? So we'll want to + // reload all the missing ones at that time. + manifest = manifest || ""; + const [ + result, + ] = await db.execute( + `UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`, + [manifest, row.id] + ); + if (result.affectedRows !== 1) { + throw new Error( + `Expected to affect 1 asset, but affected ${result.affectedRows}` + ); + } + + numRowsDone++; + + const percent = Math.floor((numRowsDone / numRowsTotal) * 100); + console.log( + `${percent}% ${numRowsDone}/${numRowsTotal} ` + + `(Exists? ${Boolean(manifest)}. Layer: ${row.id}, ${row.url}.)` + ); + } catch (e) { + console.error(`Error loading layer ${row.id}, ${row.url}.`, e); + } + } + + function promiseProducer() { + if (numRowsStarted < numRowsTotal) { + const promise = cacheAssetManifest(rows[numRowsStarted]); + numRowsStarted++; + return promise; + } else { + return null; + } + } + + const pool = new PromisePool(promiseProducer, 10); + await pool.start(); + + console.log("Done!"); +} + +async function main() { + const db = await connectToDb(); + try { + await cacheAssetManifests(db); + } catch (e) { + db.close(); + throw e; + } + db.close(); +} + +main().catch((e) => { + console.error(e); + process.exitCode = 1; +}); diff --git a/yarn.lock b/yarn.lock index 0d8ad39..df7d103 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5665,6 +5665,11 @@ es6-iterator@2.0.3, es6-iterator@~2.0.3: es5-ext "^0.10.35" es6-symbol "^3.1.1" +es6-promise-pool@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/es6-promise-pool/-/es6-promise-pool-2.5.0.tgz#147c612b36b47f105027f9d2bf54a598a99d9ccb" + integrity sha1-FHxhKza0fxBQJ/nSv1SlmKmdnMs= + es6-promise@^4.0.3: version "4.2.8" resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.8.tgz#4eb21594c972bc40553d276e510539143db53e0a"