diff --git a/package.json b/package.json index a86fbea..fc63bdd 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,8 @@ "delete-user": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/delete-user.js", "export-users-to-auth0": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/export-users-to-auth0.js", "validate-owls-data": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/validate-owls-data.js", - "archive:create:list-urls": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/archive/create/list-urls.js" + "archive:create:list-urls": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config scripts/archive/create/list-urls.js", + "archive:create:download-urls": "./scripts/archive/create/download-urls.sh" }, "browserslist": { "production": [ diff --git a/scripts/archive/create/download-urls.sh b/scripts/archive/create/download-urls.sh new file mode 100755 index 0000000..12297bb --- /dev/null +++ b/scripts/archive/create/download-urls.sh @@ -0,0 +1,2 @@ +echo 'Starting! (Note: If many of the URLs are already downloaded, it will take some time for wget to quietly check them all and find the new ones.)' +xargs --arg-file=$(dirname $0)/urls-cache.txt -P 8 wget --directory-prefix=${ARCHIVE_DIR=$(dirname $0)} --force-directories --no-clobber --timeout=10 --retry-connrefused --retry-on-host-error --no-cookies --compression=auto --https-only --no-verbose \ No newline at end of file