Matchu
e20364cc0a
Just gonna bulk load all those manifests into the db, and then that should make most loads notably faster by removing the net request! 🤞
We'll still load manifests inline sometimes, but only the first time anyone pulls up the layer in impress-2020. After that, it should be cached forever!
91 lines
2.7 KiB
JavaScript
91 lines
2.7 KiB
JavaScript
// This is a big bulk script to load the asset manifest from images.neopets.com
|
|
// for every asset, and save it to the database for fast loading!
|
|
//
|
|
// The site works fine without this: when it runs into an asset where we don't
|
|
// have the manifest cached, it loads it and caches it in real time. But this
|
|
// is a nice way to warm things up to get started!
|
|
//
|
|
// We shouldn't have to run this regularly in general, but we might want to
|
|
// re-run it once Neopets adds more manifests. Right now, we save an empty
|
|
// placeholder when no manifest exists, but someday we want to fill it in
|
|
// instead!
|
|
const PromisePool = require("es6-promise-pool");
|
|
|
|
const connectToDb = require("../src/server/db");
|
|
const neopets = require("../src/server/neopets");
|
|
|
|
async function cacheAssetManifests(db) {
|
|
const [rows] = await db.query(
|
|
`SELECT id, url FROM swf_assets WHERE manifest IS NULL`
|
|
);
|
|
|
|
const numRowsTotal = rows.length;
|
|
let numRowsStarted = 0;
|
|
let numRowsDone = 0;
|
|
|
|
async function cacheAssetManifest(row) {
|
|
try {
|
|
let manifest = await neopets.loadAssetManifest(row.url);
|
|
|
|
// After loading, write the new manifest. We make sure to write an empty
|
|
// string if there was no manifest, to signify that it doesn't exist, so
|
|
// we don't need to bother looking it up again.
|
|
//
|
|
// TODO: Someday the manifests will all exist, right? So we'll want to
|
|
// reload all the missing ones at that time.
|
|
manifest = manifest || "";
|
|
const [
|
|
result,
|
|
] = await db.execute(
|
|
`UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
|
|
[manifest, row.id]
|
|
);
|
|
if (result.affectedRows !== 1) {
|
|
throw new Error(
|
|
`Expected to affect 1 asset, but affected ${result.affectedRows}`
|
|
);
|
|
}
|
|
|
|
numRowsDone++;
|
|
|
|
const percent = Math.floor((numRowsDone / numRowsTotal) * 100);
|
|
console.log(
|
|
`${percent}% ${numRowsDone}/${numRowsTotal} ` +
|
|
`(Exists? ${Boolean(manifest)}. Layer: ${row.id}, ${row.url}.)`
|
|
);
|
|
} catch (e) {
|
|
console.error(`Error loading layer ${row.id}, ${row.url}.`, e);
|
|
}
|
|
}
|
|
|
|
function promiseProducer() {
|
|
if (numRowsStarted < numRowsTotal) {
|
|
const promise = cacheAssetManifest(rows[numRowsStarted]);
|
|
numRowsStarted++;
|
|
return promise;
|
|
} else {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
const pool = new PromisePool(promiseProducer, 10);
|
|
await pool.start();
|
|
|
|
console.log("Done!");
|
|
}
|
|
|
|
async function main() {
|
|
const db = await connectToDb();
|
|
try {
|
|
await cacheAssetManifests(db);
|
|
} catch (e) {
|
|
db.close();
|
|
throw e;
|
|
}
|
|
db.close();
|
|
}
|
|
|
|
main().catch((e) => {
|
|
console.error(e);
|
|
process.exitCode = 1;
|
|
});
|