From 38cb22980e30e4e101e8f5464ec5894d787f8d3f Mon Sep 17 00:00:00 2001 From: Matchu Date: Sat, 24 Sep 2022 11:01:48 -0700 Subject: [PATCH] Add archive:create:upload script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will upload to our remote storage! We're using `s3cmd`, but our storage isn't actually Amazon S3, it's Linode Object Storage, which has an S3-compatible API. (And it's where our VPSes already are, and its pricing model is very generous for our relatively small scale of data.) I haven't _really_ tested this exactly yet, because while `archive:create:upload-test` works great and uploads the 3 targeted files successfully… running the big one takes a very long time to even _enumerate_ all the files on my machine. (This makes sense, because I'm keeping the ~100GB archive on my HDD, which is not a fast disk!) So I'm pushing ahead even though the script is untested, because I wanna work on other stuff too! --- package.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index a28ba79..585f7c7 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,10 @@ "export-users-to-auth0": "yarn run-script scripts/export-users-to-auth0.js", "validate-owls-data": "yarn run-script scripts/validate-owls-data.js", "archive:create:list-urls": "yarn run-script scripts/archive/create/list-urls.js", - "archive:create:download-urls": "dotenv -- ./scripts/archive/create/download-urls.sh" + "archive:create:download-urls": "dotenv -- ./scripts/archive/create/download-urls.sh", + "s3cmd": "s3cmd --host=$(dotenv -p ARCHIVE_STORAGE_HOST) --host-bucket='%(bucket)s.'$(dotenv -p ARCHIVE_STORAGE_HOST) --access_key=$(dotenv -p ARCHIVE_STORAGE_READWRITE_ACCESS_KEY) --secret_key=$(dotenv -p ARCHIVE_STORAGE_READWRITE_SECRET_KEY)", + "archive:create:upload": "yarn s3cmd sync $(dotenv -p ARCHIVE_DIR)/ s3://$(dotenv -p ARCHIVE_STORAGE_BUCKET) --verbose", + "archive:create:upload-test": "yarn s3cmd sync $(dotenv -p ARCHIVE_DIR)/images.neopets.com/cp/items/data/000/000/000/1_8422bedbf9/ s3://$(dotenv -p ARCHIVE_STORAGE_BUCKET)/images.neopets.com/cp/items/data/000/000/000/1_8422bedbf9/ --verbose" }, "browserslist": { "production": [