Start building archive:create:list-urls script

Just working on making an images.neopets.com mirror, just in case! To start, I'm extracting all the URLs we need to back up; and then I'll make a separate script whose job is to mirror all of the URLs in the list.

package.json

@@ -82,7 +82,8 @@
     "cache-asset-manifests": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/cache-asset-manifests.js",
     "delete-user": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/delete-user.js",
     "export-users-to-auth0": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/export-users-to-auth0.js",
-    "validate-owls-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/validate-owls-data.js"
+    "validate-owls-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/validate-owls-data.js",
+    "archive:create:list-urls": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/archive/create/list-urls.js"
   },
   "browserslist": {
     "production": [

scripts/archive/create/.gitignore

@@ -0,0 +1 @@
+/urls-cache.txt

scripts/archive/create/list-urls.js

@@ -0,0 +1,77 @@
+/**
+ * archive:create:list-urls generates a urls-cache.txt file, containing all of
+ * the images.neopets.com URLs for customization that Dress to Impress is aware
+ * of. This will enable us to back them all up in an archive!
+ *
+ * Asset types:
+ *   - Item thumbnails
+ *   - Asset SWFs
+ *   - Asset movie manifests
+ *   - Asset movie images
+ *   - Asset movie scripts
+ */
+const fs = require("fs/promises");
+const path = require("path");
+const connectToDb = require("../../../src/server/db");
+const { normalizeRow } = require("../../../src/server/util");
+
+async function main() {
+  const urlsCacheFileAlreadyExists = await checkIfUrlsCacheFileAlreadyExists();
+  if (urlsCacheFileAlreadyExists) {
+    console.error(
+      `urls-cache.txt already exists. Please remove it first if you really ` +
+        `want to rebuild it from scratch!`
+    );
+    return 1;
+  }
+
+  const db = await connectToDb();
+  const file = await createUrlsCacheFile();
+
+  const numItems = await loadNumItems(db);
+  for (let i = 0; i < numItems; i += 1000) {
+    console.info(`Loading items ${i + 1}–${i + 1000} of ${numItems}…`);
+    const items = await loadItems(i, 1000, db);
+    const thumbnailUrls = items.map((i) => i.thumbnailUrl);
+    const lines = thumbnailUrls.map((url) => url + "\n");
+    await file.write(lines.join(""), null, "utf8");
+  }
+  console.info(`Done writing item URLs.`);
+}
+
+async function checkIfUrlsCacheFileAlreadyExists() {
+  const urlsCacheFilePath = path.join(__dirname, "urls-cache.txt");
+  try {
+    await fs.access(urlsCacheFilePath, fs.constants.R_OK);
+  } catch (error) {
+    return false;
+  }
+  return true;
+}
+
+async function createUrlsCacheFile() {
+  const urlsCacheFilePath = path.join(__dirname, "urls-cache.txt");
+  return await fs.open(urlsCacheFilePath, "w");
+}
+
+async function loadNumItems(db) {
+  const [rows] = await db.query(`SELECT count(*) FROM items`);
+  return rows[0]["count(*)"];
+}
+
+async function loadItems(offset, limit, db) {
+  const [
+    rows,
+  ] = await db.query(
+    `SELECT thumbnail_url FROM items ORDER BY id LIMIT ? OFFSET ?;`,
+    [limit, offset]
+  );
+  return rows.map(normalizeRow);
+}
+
+main()
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  })
+  .then((code = 0) => process.exit(code));