add dump mode to cache-asset-manifests

I was finding the script too slow running on my local machine, because the SQL RTTs were too slow - and with one connection, they were essentially a serial bottleneck, not taking much advantage of our concurrency.

Here, I instead add a `--dump` option, which outputs SQL to stdout. I then uploaded the resulting SQL to the DTI box, and ran it up there. Doing the network part fast on my machine, and the SQL part fast on the cloud machine!

I first considered uploading this script to the cloud machine, but it's an old Ubuntu and I couldn't figure out how to install a recent NodeJS onto it 🙃

scripts/cache-asset-manifests.js

@@ -9,14 +9,19 @@
 // re-run it once Neopets adds more manifests. Right now, we save an empty
 // placeholder when no manifest exists, but someday we want to fill it in
 // instead!
+const { argv } = require("yargs");
 const PromisePool = require("es6-promise-pool");
 
 const connectToDb = require("../src/server/db");
 const neopets = require("../src/server/neopets");
 
 async function cacheAssetManifests(db) {
-  const [rows] = await db.query(
+  const [
-    `SELECT id, url FROM swf_assets WHERE manifest IS NULL`
+    rows,
+  ] = await db.execute(
+    `SELECT id, url FROM swf_assets WHERE manifest IS NULL AND id >= ? ` +
+      `ORDER BY id`,
+    [argv.start || 0]
   );
 
   const numRowsTotal = rows.length;
@@ -34,22 +39,33 @@ async function cacheAssetManifests(db) {
       // TODO: Someday the manifests will all exist, right? So we'll want to
       //       reload all the missing ones at that time.
       manifest = manifest || "";
-      const [
+      if (argv.mode === "dump") {
-        result,
+        // Make it a JSON string, then escape the string for the query.
-      ] = await db.execute(
+        // Hacky for sure!
-        `UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
+        const escapedManifest = JSON.stringify(JSON.stringify(manifest));
-        [manifest, row.id]
+        console.log(
-      );
+          `UPDATE swf_assets SET manifest = ${escapedManifest} ` +
-      if (result.affectedRows !== 1) {
+            `WHERE id = ${row.id} LIMIT 1;`
-        throw new Error(
-          `Expected to affect 1 asset, but affected ${result.affectedRows}`
         );
+      } else {
+        const [
+          result,
+        ] = await db.execute(
+          `UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
+          [manifest, row.id]
+        );
+        if (result.affectedRows !== 1) {
+          throw new Error(
+            `Expected to affect 1 asset, but affected ${result.affectedRows}`
+          );
+        }
       }
 
       numRowsDone++;
 
       const percent = Math.floor((numRowsDone / numRowsTotal) * 100);
-      console.log(
+      // write to stderr, to not disrupt the dump
+      console.error(
         `${percent}% ${numRowsDone}/${numRowsTotal} ` +
           `(Exists? ${Boolean(manifest)}. Layer: ${row.id}, ${row.url}.)`
       );
@@ -71,7 +87,8 @@ async function cacheAssetManifests(db) {
   const pool = new PromisePool(promiseProducer, 10);
   await pool.start();
 
-  console.log("Done!");
+  // write to stderr, to not disrupt the dump
+  console.error("Done!");
 }
 
 async function main() {