OpenNeo/impress-2020
OpenNeo/impress-2020
1
0
Fork 0
Code Pull requests Activity

add dump mode to cache-asset-manifests

Browse source 
I was finding the script too slow running on my local machine, because the SQL RTTs were too slow - and with one connection, they were essentially a serial bottleneck, not taking much advantage of our concurrency.

Here, I instead add a `--dump` option, which outputs SQL to stdout. I then uploaded the resulting SQL to the DTI box, and ran it up there. Doing the network part fast on my machine, and the SQL part fast on the cloud machine!

I first considered uploading this script to the cloud machine, but it's an old Ubuntu and I couldn't figure out how to install a recent NodeJS onto it 🙃
This commit is contained in:
Emi Matchu 2020-08-19 17:19:15 -07:00
parent 7a3a7eeaf9
commit 20523a9562
1 changed files with 30 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
scripts/cache-asset-manifests.js
View file

@ -9,14 +9,19 @@
// re-run it once Neopets adds more manifests. Right now, we save an empty
// placeholder when no manifest exists, but someday we want to fill it in
// instead!
const { argv } = require("yargs");
const PromisePool = require("es6-promise-pool");
const connectToDb = require("../src/server/db");
const neopets = require("../src/server/neopets");
async function cacheAssetManifests(db) {
const [rows] = await db.query(
`SELECT id, url FROM swf_assets WHERE manifest IS NULL`
const [
rows,
] = await db.execute(
`SELECT id, url FROM swf_assets WHERE manifest IS NULL AND id >= ? ` +
`ORDER BY id`,
[argv.start || 0]
);
const numRowsTotal = rows.length;
@ -34,22 +39,33 @@ async function cacheAssetManifests(db) {
// TODO: Someday the manifests will all exist, right? So we'll want to
// reload all the missing ones at that time.
manifest = manifest || "";
const [
result,
] = await db.execute(
`UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
[manifest, row.id]
);
if (result.affectedRows !== 1) {
throw new Error(
`Expected to affect 1 asset, but affected ${result.affectedRows}`
if (argv.mode === "dump") {
// Make it a JSON string, then escape the string for the query.
// Hacky for sure!
const escapedManifest = JSON.stringify(JSON.stringify(manifest));
console.log(
`UPDATE swf_assets SET manifest = ${escapedManifest} ` +
`WHERE id = ${row.id} LIMIT 1;`
);
} else {
const [
result,
] = await db.execute(
`UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
[manifest, row.id]
);
if (result.affectedRows !== 1) {
throw new Error(
`Expected to affect 1 asset, but affected ${result.affectedRows}`
);
}
}
numRowsDone++;
const percent = Math.floor((numRowsDone / numRowsTotal) * 100);
console.log(
// write to stderr, to not disrupt the dump
console.error(
`${percent}% ${numRowsDone}/${numRowsTotal} ` +
`(Exists? ${Boolean(manifest)}. Layer: ${row.id}, ${row.url}.)`
);
@ -71,7 +87,8 @@ async function cacheAssetManifests(db) {
const pool = new PromisePool(promiseProducer, 10);
await pool.start();
console.log("Done!");
// write to stderr, to not disrupt the dump
console.error("Done!");
}
async function main() {