Matchu
20523a9562
I was finding the script too slow running on my local machine, because the SQL RTTs were too slow - and with one connection, they were essentially a serial bottleneck, not taking much advantage of our concurrency.
Here, I instead add a `--dump` option, which outputs SQL to stdout. I then uploaded the resulting SQL to the DTI box, and ran it up there. Doing the network part fast on my machine, and the SQL part fast on the cloud machine!
I first considered uploading this script to the cloud machine, but it's an old Ubuntu and I couldn't figure out how to install a recent NodeJS onto it 🙃
108 lines
3.2 KiB
JavaScript
108 lines
3.2 KiB
JavaScript
// This is a big bulk script to load the asset manifest from images.neopets.com
|
|
// for every asset, and save it to the database for fast loading!
|
|
//
|
|
// The site works fine without this: when it runs into an asset where we don't
|
|
// have the manifest cached, it loads it and caches it in real time. But this
|
|
// is a nice way to warm things up to get started!
|
|
//
|
|
// We shouldn't have to run this regularly in general, but we might want to
|
|
// re-run it once Neopets adds more manifests. Right now, we save an empty
|
|
// placeholder when no manifest exists, but someday we want to fill it in
|
|
// instead!
|
|
const { argv } = require("yargs");
|
|
const PromisePool = require("es6-promise-pool");
|
|
|
|
const connectToDb = require("../src/server/db");
|
|
const neopets = require("../src/server/neopets");
|
|
|
|
async function cacheAssetManifests(db) {
|
|
const [
|
|
rows,
|
|
] = await db.execute(
|
|
`SELECT id, url FROM swf_assets WHERE manifest IS NULL AND id >= ? ` +
|
|
`ORDER BY id`,
|
|
[argv.start || 0]
|
|
);
|
|
|
|
const numRowsTotal = rows.length;
|
|
let numRowsStarted = 0;
|
|
let numRowsDone = 0;
|
|
|
|
async function cacheAssetManifest(row) {
|
|
try {
|
|
let manifest = await neopets.loadAssetManifest(row.url);
|
|
|
|
// After loading, write the new manifest. We make sure to write an empty
|
|
// string if there was no manifest, to signify that it doesn't exist, so
|
|
// we don't need to bother looking it up again.
|
|
//
|
|
// TODO: Someday the manifests will all exist, right? So we'll want to
|
|
// reload all the missing ones at that time.
|
|
manifest = manifest || "";
|
|
if (argv.mode === "dump") {
|
|
// Make it a JSON string, then escape the string for the query.
|
|
// Hacky for sure!
|
|
const escapedManifest = JSON.stringify(JSON.stringify(manifest));
|
|
console.log(
|
|
`UPDATE swf_assets SET manifest = ${escapedManifest} ` +
|
|
`WHERE id = ${row.id} LIMIT 1;`
|
|
);
|
|
} else {
|
|
const [
|
|
result,
|
|
] = await db.execute(
|
|
`UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
|
|
[manifest, row.id]
|
|
);
|
|
if (result.affectedRows !== 1) {
|
|
throw new Error(
|
|
`Expected to affect 1 asset, but affected ${result.affectedRows}`
|
|
);
|
|
}
|
|
}
|
|
|
|
numRowsDone++;
|
|
|
|
const percent = Math.floor((numRowsDone / numRowsTotal) * 100);
|
|
// write to stderr, to not disrupt the dump
|
|
console.error(
|
|
`${percent}% ${numRowsDone}/${numRowsTotal} ` +
|
|
`(Exists? ${Boolean(manifest)}. Layer: ${row.id}, ${row.url}.)`
|
|
);
|
|
} catch (e) {
|
|
console.error(`Error loading layer ${row.id}, ${row.url}.`, e);
|
|
}
|
|
}
|
|
|
|
function promiseProducer() {
|
|
if (numRowsStarted < numRowsTotal) {
|
|
const promise = cacheAssetManifest(rows[numRowsStarted]);
|
|
numRowsStarted++;
|
|
return promise;
|
|
} else {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
const pool = new PromisePool(promiseProducer, 10);
|
|
await pool.start();
|
|
|
|
// write to stderr, to not disrupt the dump
|
|
console.error("Done!");
|
|
}
|
|
|
|
async function main() {
|
|
const db = await connectToDb();
|
|
try {
|
|
await cacheAssetManifests(db);
|
|
} catch (e) {
|
|
db.close();
|
|
throw e;
|
|
}
|
|
db.close();
|
|
}
|
|
|
|
main().catch((e) => {
|
|
console.error(e);
|
|
process.exitCode = 1;
|
|
});
|