create cache-asset-manifests script

Just gonna bulk load all those manifests into the db, and then that should make most loads notably faster by removing the net request! 🤞

We'll still load manifests inline sometimes, but only the first time anyone pulls up the layer in impress-2020. After that, it should be cached forever!

package.json

@@ -33,13 +33,14 @@
   },
   "scripts": {
     "start": "yarn build-cached-data && react-app-rewired start",
-    "build-cached-data": "node -r dotenv/config build-cached-data.js",
+    "build-cached-data": "node -r dotenv/config scripts/build-cached-data.js",
     "build": "react-app-rewired build && yarn build-cached-data",
     "test": "react-app-rewired test --env=jsdom",
     "eject": "react-scripts eject",
     "setup-mysql-user": "mysql -h impress.openneo.net -u matchu -p < setup-mysql-user.sql",
     "mysql": "mysql --host=impress.openneo.net --user=$(dotenv -p IMPRESS_MYSQL_USER) --password=$(dotenv -p IMPRESS_MYSQL_PASSWORD) --database=openneo_impress",
-    "mysql-admin": "mysql --host=impress.openneo.net --user=matchu --password --database=openneo_impress"
+    "mysql-admin": "mysql --host=impress.openneo.net --user=matchu --password --database=openneo_impress",
+    "cache-asset-manifests": "node -r dotenv/config scripts/cache-asset-manifests.js"
   },
   "eslintConfig": {
     "extends": "react-app"
@@ -61,6 +62,7 @@
     "customize-cra": "^1.0.0",
     "customize-cra-react-refresh": "^1.1.0",
     "dotenv-cli": "^3.1.0",
+    "es6-promise-pool": "^2.5.0",
     "prettier": "^2.0.5",
     "react-app-rewired": "^2.1.6"
   }

scripts/build-cached-data.js

@@ -1,10 +1,11 @@
+// We run this on build to cache some stable database tables on the server!
 const fs = require("fs").promises;
 const path = require("path");
 
-const connectToDb = require("./src/server/db");
+const connectToDb = require("../src/server/db");
-const { normalizeRow } = require("./src/server/util");
+const { normalizeRow } = require("../src/server/util");
 
-const cachedDataPath = path.join(__dirname, "build", "cached-data");
+const cachedDataPath = path.join(__dirname, "..", "build", "cached-data");
 
 async function buildZonesCache(db) {
   const [rows] = await db.query(`SELECT * FROM zones;`);

scripts/cache-asset-manifests.js

@@ -0,0 +1,91 @@
+// This is a big bulk script to load the asset manifest from images.neopets.com
+// for every asset, and save it to the database for fast loading!
+//
+// The site works fine without this: when it runs into an asset where we don't
+// have the manifest cached, it loads it and caches it in real time. But this
+// is a nice way to warm things up to get started!
+//
+// We shouldn't have to run this regularly in general, but we might want to
+// re-run it once Neopets adds more manifests. Right now, we save an empty
+// placeholder when no manifest exists, but someday we want to fill it in
+// instead!
+const PromisePool = require("es6-promise-pool");
+
+const connectToDb = require("../src/server/db");
+const neopets = require("../src/server/neopets");
+
+async function cacheAssetManifests(db) {
+  const [rows] = await db.query(
+    `SELECT id, url FROM swf_assets WHERE manifest IS NULL`
+  );
+
+  const numRowsTotal = rows.length;
+  let numRowsStarted = 0;
+  let numRowsDone = 0;
+
+  async function cacheAssetManifest(row) {
+    try {
+      let manifest = await neopets.loadAssetManifest(row.url);
+
+      // After loading, write the new manifest. We make sure to write an empty
+      // string if there was no manifest, to signify that it doesn't exist, so
+      // we don't need to bother looking it up again.
+      //
+      // TODO: Someday the manifests will all exist, right? So we'll want to
+      //       reload all the missing ones at that time.
+      manifest = manifest || "";
+      const [
+        result,
+      ] = await db.execute(
+        `UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
+        [manifest, row.id]
+      );
+      if (result.affectedRows !== 1) {
+        throw new Error(
+          `Expected to affect 1 asset, but affected ${result.affectedRows}`
+        );
+      }
+
+      numRowsDone++;
+
+      const percent = Math.floor((numRowsDone / numRowsTotal) * 100);
+      console.log(
+        `${percent}% ${numRowsDone}/${numRowsTotal} ` +
+          `(Exists? ${Boolean(manifest)}. Layer: ${row.id}, ${row.url}.)`
+      );
+    } catch (e) {
+      console.error(`Error loading layer ${row.id}, ${row.url}.`, e);
+    }
+  }
+
+  function promiseProducer() {
+    if (numRowsStarted < numRowsTotal) {
+      const promise = cacheAssetManifest(rows[numRowsStarted]);
+      numRowsStarted++;
+      return promise;
+    } else {
+      return null;
+    }
+  }
+
+  const pool = new PromisePool(promiseProducer, 10);
+  await pool.start();
+
+  console.log("Done!");
+}
+
+async function main() {
+  const db = await connectToDb();
+  try {
+    await cacheAssetManifests(db);
+  } catch (e) {
+    db.close();
+    throw e;
+  }
+  db.close();
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exitCode = 1;
+});

yarn.lock

@@ -5665,6 +5665,11 @@ es6-iterator@2.0.3, es6-iterator@~2.0.3:
     es5-ext "^0.10.35"
     es6-symbol "^3.1.1"
 
+es6-promise-pool@^2.5.0:
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/es6-promise-pool/-/es6-promise-pool-2.5.0.tgz#147c612b36b47f105027f9d2bf54a598a99d9ccb"
+  integrity sha1-FHxhKza0fxBQJ/nSv1SlmKmdnMs=
+
 es6-promise@^4.0.3:
   version "4.2.8"
   resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.8.tgz#4eb21594c972bc40553d276e510539143db53e0a"