From 20523a95620d028e4ed2f5ffefabd4f11bfca78c Mon Sep 17 00:00:00 2001
From: Matchu <matchu1993@gmail.com>
Date: Wed, 19 Aug 2020 17:19:15 -0700
Subject: [PATCH] add dump mode to cache-asset-manifests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I was finding the script too slow running on my local machine, because the SQL RTTs were too slow - and with one connection, they were essentially a serial bottleneck, not taking much advantage of our concurrency.

Here, I instead add a `--dump` option, which outputs SQL to stdout. I then uploaded the resulting SQL to the DTI box, and ran it up there. Doing the network part fast on my machine, and the SQL part fast on the cloud machine!

I first considered uploading this script to the cloud machine, but it's an old Ubuntu and I couldn't figure out how to install a recent NodeJS onto it 🙃
---
 scripts/cache-asset-manifests.js | 43 ++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/scripts/cache-asset-manifests.js b/scripts/cache-asset-manifests.js
index df3a3c8..c8644ec 100644
--- a/scripts/cache-asset-manifests.js
+++ b/scripts/cache-asset-manifests.js
@@ -9,14 +9,19 @@
 // re-run it once Neopets adds more manifests. Right now, we save an empty
 // placeholder when no manifest exists, but someday we want to fill it in
 // instead!
+const { argv } = require("yargs");
 const PromisePool = require("es6-promise-pool");
 
 const connectToDb = require("../src/server/db");
 const neopets = require("../src/server/neopets");
 
 async function cacheAssetManifests(db) {
-  const [rows] = await db.query(
-    `SELECT id, url FROM swf_assets WHERE manifest IS NULL`
+  const [
+    rows,
+  ] = await db.execute(
+    `SELECT id, url FROM swf_assets WHERE manifest IS NULL AND id >= ? ` +
+      `ORDER BY id`,
+    [argv.start || 0]
   );
 
   const numRowsTotal = rows.length;
@@ -34,22 +39,33 @@ async function cacheAssetManifests(db) {
       // TODO: Someday the manifests will all exist, right? So we'll want to
       //       reload all the missing ones at that time.
       manifest = manifest || "";
-      const [
-        result,
-      ] = await db.execute(
-        `UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
-        [manifest, row.id]
-      );
-      if (result.affectedRows !== 1) {
-        throw new Error(
-          `Expected to affect 1 asset, but affected ${result.affectedRows}`
+      if (argv.mode === "dump") {
+        // Make it a JSON string, then escape the string for the query.
+        // Hacky for sure!
+        const escapedManifest = JSON.stringify(JSON.stringify(manifest));
+        console.log(
+          `UPDATE swf_assets SET manifest = ${escapedManifest} ` +
+            `WHERE id = ${row.id} LIMIT 1;`
         );
+      } else {
+        const [
+          result,
+        ] = await db.execute(
+          `UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
+          [manifest, row.id]
+        );
+        if (result.affectedRows !== 1) {
+          throw new Error(
+            `Expected to affect 1 asset, but affected ${result.affectedRows}`
+          );
+        }
       }
 
       numRowsDone++;
 
       const percent = Math.floor((numRowsDone / numRowsTotal) * 100);
-      console.log(
+      // write to stderr, to not disrupt the dump
+      console.error(
         `${percent}% ${numRowsDone}/${numRowsTotal} ` +
           `(Exists? ${Boolean(manifest)}. Layer: ${row.id}, ${row.url}.)`
       );
@@ -71,7 +87,8 @@ async function cacheAssetManifests(db) {
   const pool = new PromisePool(promiseProducer, 10);
   await pool.start();
 
-  console.log("Done!");
+  // write to stderr, to not disrupt the dump
+  console.error("Done!");
 }
 
 async function main() {