OpenNeo/impress-2020
OpenNeo/impress-2020
1
0
Fork 0
Code Pull requests Activity

Perf & feedback upgrades to cache-asset-manifests

Browse source 
I want to start running this on a regular cron, and making the script faster (stop sending redundant queries) and clearer (# actually updated) is super useful for that!
This commit is contained in:
Emi Matchu 2021-01-20 09:49:05 -08:00
parent ccd4885aad
commit 6da2ddb453
1 changed files with 28 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
scripts/cache-asset-manifests.js
View file

@ -17,7 +17,7 @@ const neopetsAssets = require("../src/server/neopets-assets");
async function cacheAssetManifests(db) { async function cacheAssetManifests(db) {
const [rows] = await db.execute( const [rows] = await db.execute(
`SELECT id, url FROM swf_assets ` + `SELECT id, url, manifest FROM swf_assets ` +
`WHERE manifest IS NULL OR manifest = "" AND id >= ? ` + `WHERE manifest IS NULL OR manifest = "" AND id >= ? ` +
`ORDER BY id`, `ORDER BY id`,
[argv.start || 0] [argv.start || 0]
@ -25,6 +25,7 @@ async function cacheAssetManifests(db) {
const numRowsTotal = rows.length; const numRowsTotal = rows.length;
let numRowsStarted = 0; let numRowsStarted = 0;
let numRowsUpdated = 0;
let numRowsDone = 0; let numRowsDone = 0;
async function cacheAssetManifest(row) { async function cacheAssetManifest(row) {
@ -38,26 +39,31 @@ async function cacheAssetManifests(db) {
// TODO: Someday the manifests will all exist, right? So we'll want to // TODO: Someday the manifests will all exist, right? So we'll want to
// reload all the missing ones at that time. // reload all the missing ones at that time.
manifest = manifest || ""; manifest = manifest || "";
if (argv.mode === "dump") { let updated = false;
// Make it a JSON string, then escape the string for the query. if (row.manifest !== manifest) {
// Hacky for sure! if (argv.mode === "dump") {
const escapedManifest = JSON.stringify(JSON.stringify(manifest)); // Make it a JSON string, then escape the string for the query.
console.log( // Hacky for sure!
`UPDATE swf_assets SET manifest = ${escapedManifest} ` + const escapedManifest = JSON.stringify(JSON.stringify(manifest));
`WHERE id = ${row.id} LIMIT 1;` console.log(
); `UPDATE swf_assets SET manifest = ${escapedManifest} ` +
} else { `WHERE id = ${row.id} LIMIT 1;`
const [
result,
] = await db.execute(
`UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
[manifest, row.id]
);
if (result.affectedRows !== 1) {
throw new Error(
`Expected to affect 1 asset, but affected ${result.affectedRows}`
); );
} else {
const [
result,
] = await db.execute(
`UPDATE swf_assets SET manifest = ? WHERE id = ? LIMIT 1;`,
[manifest, row.id]
);
if (result.affectedRows !== 1) {
throw new Error(
`Expected to affect 1 asset, but affected ${result.affectedRows}`
);
}
} }
updated = true;
numRowsUpdated++;
} }
numRowsDone++; numRowsDone++;
@ -66,7 +72,8 @@ async function cacheAssetManifests(db) {
// write to stderr, to not disrupt the dump // write to stderr, to not disrupt the dump
console.error( console.error(
`${percent}% ${numRowsDone}/${numRowsTotal} ` + `${percent}% ${numRowsDone}/${numRowsTotal} ` +
`(Exists? ${Boolean(manifest)}. Layer: ${row.id}, ${row.url}.)` `(Exists? ${Boolean(manifest)}. Updated? ${updated}. ` +
`Layer: ${row.id}, ${row.url}.)`
); );
} catch (e) { } catch (e) {
console.error(`Error loading layer ${row.id}, ${row.url}.`, e); console.error(`Error loading layer ${row.id}, ${row.url}.`, e);
@ -87,7 +94,7 @@ async function cacheAssetManifests(db) {
await pool.start(); await pool.start();
// write to stderr, to not disrupt the dump // write to stderr, to not disrupt the dump
console.error("Done!"); console.error(`Done! Updated ${numRowsUpdated} of ${numRowsDone} rows.`);
} }
async function main() { async function main() {