Matchu
d9ca07c9b0
Hey this is an exciting development! A list of URLs, that we want to clone onto our hard drive, turns out to be something `wget` is already very good at! Originally I used `wget`'s `--input-file` option to process the `urls-cache.txt` file, but then I learned how to parallelize it from this StackOverflow answer: https://stackoverflow.com/a/11850469/107415. (Following the guidance in the comments, I removed `-n 1`, to avoid the overhead of extra processes and allow `wget` instances to keep using shared connections over time. Idk why it was in there, maybe the author didn't know `wget` accepts multiple args?) Anyway yeah, it's working great, except for the weird images.neopets.com downtime! 😅 Specifically I'm noticing that all the item thumbnail images came back really fast, but the customization images are taking for-EV-er. I wonder if that's just caching properties, or if there's a different backing server for it and it's responding much more slowly? Who's to say! In any case, I'm keeping the timeout in this script pretty low (10 seconds), and just letting failures fail. We can try re-running it again sometime when the downtime is resolved or the cache is warmed up.
125 lines
5.9 KiB
JSON
125 lines
5.9 KiB
JSON
{
|
|
"name": "impress-2020-app",
|
|
"version": "0.1.0",
|
|
"private": true,
|
|
"dependencies": {
|
|
"@apollo/client": "^3.6.9",
|
|
"@apollographql/graphql-playground-html": "^1.6.24",
|
|
"@auth0/auth0-react": "^1.0.0",
|
|
"@chakra-ui/icons": "^1.0.4",
|
|
"@chakra-ui/react": "^1.6.0",
|
|
"@chakra-ui/theme-tools": "^1.0.3",
|
|
"@emotion/react": "^11.1.4",
|
|
"@emotion/styled": "^11.0.0",
|
|
"@loadable/component": "^5.12.0",
|
|
"@sendgrid/mail": "^7.2.6",
|
|
"@sentry/react": "^5.30.0",
|
|
"@sentry/tracing": "^5.30.0",
|
|
"@testing-library/jest-dom": "^4.2.4",
|
|
"@testing-library/react": "^9.3.2",
|
|
"@testing-library/user-event": "^7.1.2",
|
|
"@types/node": "^14.14.22",
|
|
"@types/react": "^17.0.1",
|
|
"@types/react-dom": "^17.0.0",
|
|
"apollo-link-persisted-queries": "^0.2.2",
|
|
"apollo-server": "^2.25.3",
|
|
"apollo-server-core": "^2.19.2",
|
|
"apollo-server-env": "^3.0.0",
|
|
"aws-sdk": "^2.814.0",
|
|
"canvas": "^2.6.1",
|
|
"chrome-aws-lambda": "^10.1.0",
|
|
"dataloader": "^2.0.0",
|
|
"dompurify": "^2.2.0",
|
|
"easeljs": "^1.0.2",
|
|
"escape-html": "^1.0.3",
|
|
"framer-motion": "^4.1.11",
|
|
"generic-pool": "^3.8.2",
|
|
"graphql": "^15.5.0",
|
|
"honeycomb-beeline": "^2.7.4",
|
|
"immer": "^9.0.6",
|
|
"jimp": "^0.14.0",
|
|
"jsonwebtoken": "^8.5.1",
|
|
"jwks-rsa": "^1.9.0",
|
|
"lru-cache": "^6.0.0",
|
|
"mysql2": "^2.1.0",
|
|
"next": "12.0.2",
|
|
"node-fetch": "^2.6.0",
|
|
"puppeteer": "^11.0.0",
|
|
"react": "^17.0.1",
|
|
"react-autosuggest": "^10.0.2",
|
|
"react-dom": "^17.0.1",
|
|
"react-icons": "^4.2.0",
|
|
"react-router-dom": "^5.1.2",
|
|
"react-router-hash-link": "^2.4.3",
|
|
"react-transition-group": "^4.3.0",
|
|
"react-virtualized": "^9.22.3",
|
|
"sharp": "^0.29.2",
|
|
"simple-markdown": "^0.7.2",
|
|
"tweenjs": "^1.0.2",
|
|
"typescript": "^4.4.4",
|
|
"xmlrpc": "^1.3.2"
|
|
},
|
|
"scripts": {
|
|
"prepare": "husky install",
|
|
"start": "next start",
|
|
"dev": "yarn build-cached-data && REACT_APP_IMPRESS_LOG_IN_AS=$IMPRESS_LOG_IN_AS TS_NODE_COMPILER=typescript-cached-transpile next dev",
|
|
"build": "yarn build-cached-data && next build",
|
|
"vercel-build": "yum install libuuid-devel libmount-devel && cp /lib64/{libuuid,libmount,libblkid}.so.1 node_modules/canvas/build/Release/",
|
|
"test": "jest test --env=jsdom",
|
|
"lint": "next lint --dir src --dir pages",
|
|
"deploy-setup": "echo $'Setup requires you to become the root user. You\\'ll need to enter the password for your account on the remote web server below, and you must be part of the `sudo` user group.' && ansible-playbook -K -i deploy/inventory.cfg deploy/playbooks/setup.yml",
|
|
"deploy": "ansible-playbook -i deploy/inventory.cfg deploy/playbooks/deploy.yml",
|
|
"deploy-skip-build": "ansible-playbook -i deploy/inventory.cfg deploy/playbooks/deploy.yml --extra-vars='{\"skip_build\": true}'",
|
|
"cypress": "cypress open",
|
|
"mysql": "mysql --host=impress.openneo.net --user=$(dotenv -p IMPRESS_MYSQL_USER) --password=$(dotenv -p IMPRESS_MYSQL_PASSWORD) --database=openneo_impress",
|
|
"mysql-dev": "mysql --host=localhost --user=impress_2020_dev --password=impress_2020_dev --database=impress_2020_dev",
|
|
"mysql-admin": "mysql --host=impress.openneo.net --user=matchu --password --database=openneo_impress",
|
|
"mysqldump": "mysqldump --host=impress.openneo.net --user=$(dotenv -p IMPRESS_MYSQL_USER) --password=$(dotenv -p IMPRESS_MYSQL_PASSWORD) --column-statistics=0",
|
|
"download-mysql-schema": "yarn --silent mysqldump --no-data openneo_impress closet_hangers closet_lists items item_translations modeling_logs parents_swf_assets pet_types pet_states swf_assets users | sed 's/ AUTO_INCREMENT=[0-9]*//g' > scripts/setup-mysql-dev-schema.sql && yarn --silent mysqldump openneo_impress species species_translations colors color_translations zones zone_translations > scripts/setup-mysql-dev-constants.sql",
|
|
"setup-mysql": "yarn mysql-admin < scripts/setup-mysql.sql",
|
|
"setup-mysql-dev": "yarn mysql-dev < scripts/setup-mysql-dev-constants.sql && yarn mysql-dev < scripts/setup-mysql-dev-schema.sql",
|
|
"build-cached-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/build-cached-data.js",
|
|
"cache-asset-manifests": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/cache-asset-manifests.js",
|
|
"delete-user": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/delete-user.js",
|
|
"export-users-to-auth0": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/export-users-to-auth0.js",
|
|
"validate-owls-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/validate-owls-data.js",
|
|
"archive:create:list-urls": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/archive/create/list-urls.js",
|
|
"archive:create:download-urls": "./scripts/archive/create/download-urls.sh"
|
|
},
|
|
"browserslist": {
|
|
"production": [
|
|
">0.2%",
|
|
"not dead",
|
|
"not op_mini all"
|
|
],
|
|
"development": [
|
|
"last 1 chrome version",
|
|
"last 1 firefox version",
|
|
"last 1 safari version"
|
|
]
|
|
},
|
|
"devDependencies": {
|
|
"5to6-codemod": "^1.8.0",
|
|
"@typescript-eslint/eslint-plugin": "^5.3.0",
|
|
"@typescript-eslint/parser": "^5.3.0",
|
|
"apollo-server-testing": "^2.12.0",
|
|
"auth0": "^2.28.0",
|
|
"cypress": "^7.2.0",
|
|
"cypress-plugin-snapshots": "^1.4.4",
|
|
"dotenv-cli": "^3.1.0",
|
|
"es6-promise-pool": "^2.5.0",
|
|
"eslint": "<8.0.0",
|
|
"eslint-config-next": "12.0.2",
|
|
"eslint-plugin-cypress": "^2.11.2",
|
|
"husky": "^6.0.0",
|
|
"imports-loader": "^1.2.0",
|
|
"inquirer": "^7.3.3",
|
|
"jest-image-snapshot": "^4.3.0",
|
|
"lint-staged": "^10.5.4",
|
|
"prettier": "^2.0.5",
|
|
"react-is": "^16.13.1",
|
|
"ts-node": "^9.1.1",
|
|
"typescript-cached-transpile": "^0.0.6",
|
|
"yargs": "^17.3.1"
|
|
}
|
|
}
|