Start building archive:create:list-urls script

Just working on making an images.neopets.com mirror, just in case! To start, I'm extracting all the URLs we need to back up; and then I'll make a separate script whose job is to mirror all of the URLs in the list.
This commit is contained in:
Emi Matchu 2022-09-12 15:53:22 -07:00
parent 08603af961
commit 3ce895d52f
3 changed files with 80 additions and 1 deletions

View file

@ -82,7 +82,8 @@
"cache-asset-manifests": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/cache-asset-manifests.js",
"delete-user": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/delete-user.js",
"export-users-to-auth0": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/export-users-to-auth0.js",
"validate-owls-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/validate-owls-data.js"
"validate-owls-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/validate-owls-data.js",
"archive:create:list-urls": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/archive/create/list-urls.js"
},
"browserslist": {
"production": [

1
scripts/archive/create/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/urls-cache.txt

View file

@ -0,0 +1,77 @@
/**
* archive:create:list-urls generates a urls-cache.txt file, containing all of
* the images.neopets.com URLs for customization that Dress to Impress is aware
* of. This will enable us to back them all up in an archive!
*
* Asset types:
* - Item thumbnails
* - Asset SWFs
* - Asset movie manifests
* - Asset movie images
* - Asset movie scripts
*/
const fs = require("fs/promises");
const path = require("path");
const connectToDb = require("../../../src/server/db");
const { normalizeRow } = require("../../../src/server/util");
async function main() {
const urlsCacheFileAlreadyExists = await checkIfUrlsCacheFileAlreadyExists();
if (urlsCacheFileAlreadyExists) {
console.error(
`urls-cache.txt already exists. Please remove it first if you really ` +
`want to rebuild it from scratch!`
);
return 1;
}
const db = await connectToDb();
const file = await createUrlsCacheFile();
const numItems = await loadNumItems(db);
for (let i = 0; i < numItems; i += 1000) {
console.info(`Loading items ${i + 1}${i + 1000} of ${numItems}`);
const items = await loadItems(i, 1000, db);
const thumbnailUrls = items.map((i) => i.thumbnailUrl);
const lines = thumbnailUrls.map((url) => url + "\n");
await file.write(lines.join(""), null, "utf8");
}
console.info(`Done writing item URLs.`);
}
async function checkIfUrlsCacheFileAlreadyExists() {
const urlsCacheFilePath = path.join(__dirname, "urls-cache.txt");
try {
await fs.access(urlsCacheFilePath, fs.constants.R_OK);
} catch (error) {
return false;
}
return true;
}
async function createUrlsCacheFile() {
const urlsCacheFilePath = path.join(__dirname, "urls-cache.txt");
return await fs.open(urlsCacheFilePath, "w");
}
async function loadNumItems(db) {
const [rows] = await db.query(`SELECT count(*) FROM items`);
return rows[0]["count(*)"];
}
async function loadItems(offset, limit, db) {
const [
rows,
] = await db.query(
`SELECT thumbnail_url FROM items ORDER BY id LIMIT ? OFFSET ?;`,
[limit, offset]
);
return rows.map(normalizeRow);
}
main()
.catch((e) => {
console.error(e);
process.exit(1);
})
.then((code = 0) => process.exit(code));