impress-2020/scripts/cache-asset-manifests.js

// This is a big bulk script to load the asset manifest from images.neopets.com
// for every asset, and save it to the database for fast loading!
//
// The site works fine without this: when it runs into an asset where we don't
// have the manifest cached, it loads it and caches it in real time. But this
// is a nice way to warm things up to get started!
//
// We shouldn't have to run this regularly in general, but we might want to
// re-run it once Neopets adds more manifests. Right now, we save an empty
// placeholder when no manifest exists, but someday we want to fill it in
// instead!
const { argv } = require("yargs");
const PromisePool = require("es6-promise-pool");

const connectToDb = require("../src/server/db");
const neopetsAssets = require("../src/server/neopets-assets");

async function cacheAssetManifests(db) {
  // Normally, we skip manifests that we already have. But you can run with
  // --resync-existing to check old manifests for changes! (This might take,
  // like, multiple minutes to even run this first query!)
  const condition = argv.resyncExisting
    ? `1`
    : `manifest IS NULL OR manifest = ""`;

  const [rows] = await db.execute(
    `SELECT id, url, manifest FROM swf_assets ` +
      `WHERE ${condition} AND id >= ? ` +
      `ORDER BY id`,
    [argv.start || 0]
  );

  const numRowsTotal = rows.length;
  let numRowsStarted = 0;
  let numRowsUpdated = 0;
  let numRowsDone = 0;

  async function cacheAssetManifest(row) {
    try {
      let manifest = await neopetsAssets.loadAssetManifest(row.url);

      // After loading, write the new manifest. We make sure to write an empty
      // string if there was no manifest, to signify that it doesn't exist, so
      // we don't need to bother looking it up again.
      //
      // TODO: Someday the manifests will all exist, right? So we'll want to
      //       reload all the missing ones at that time.
      manifest = manifest || "";
      let updated = false;
      if (row.manifest !== manifest) {
        if (argv.mode === "dump") {
          // Make it a JSON string, then escape the string for the query.
          // Hacky for sure!
          const escapedManifest = JSON.stringify(JSON.stringify(manifest));
          console.info(
            `UPDATE swf_assets SET manifest = ${escapedManifest}, ` +
              `manifest_cached_at = CURRENT_TIMESTAMP() ` +
              `WHERE id = ${row.id} LIMIT 1;`
          );
        } else {
          const [
            result,
          ] = await db.execute(
            `UPDATE swf_assets SET manifest = ?, ` +
              `manifest_cached_at = CURRENT_TIMESTAMP() ` +
              `WHERE id = ? LIMIT 1;`,
            [manifest, row.id]
          );
          if (result.affectedRows !== 1) {
            throw new Error(
              `Expected to affect 1 asset, but affected ${result.affectedRows}`
            );
          }
        }
        updated = true;
        numRowsUpdated++;
      }

      numRowsDone++;

      const percent = Math.floor((numRowsDone / numRowsTotal) * 100);
      const shouldSkipLogging = argv.skipUnchanged && !updated;
      if (!shouldSkipLogging) {
        // write to stderr, to not disrupt the dump
        console.error(
          `${percent}% ${numRowsDone}/${numRowsTotal} ` +
            `(Exists? ${Boolean(manifest)}. Updated? ${updated}. ` +
            `Layer: ${row.id}, ${row.url}.)`
        );
      }
    } catch (e) {
      console.error(`Error loading layer ${row.id}, ${row.url}.`, e);
    }
  }

  function promiseProducer() {
    if (numRowsStarted < numRowsTotal) {
      const promise = cacheAssetManifest(rows[numRowsStarted]);
      numRowsStarted++;
      return promise;
    } else {
      return null;
    }
  }

  const pool = new PromisePool(promiseProducer, 10);
  await pool.start();

  // write to stderr, to not disrupt the dump
  console.error(`Done! Updated ${numRowsUpdated} of ${numRowsDone} rows.`);
}

async function main() {
  const db = await connectToDb();
  try {
    await cacheAssetManifests(db);
  } catch (e) {
    db.end();
    throw e;
  }
  db.end();
}

main().catch((e) => {
  console.error(e);
  process.exitCode = 1;
});