From ae568072f8511b98c3c37a94b646aa544e596bb3 Mon Sep 17 00:00:00 2001 From: Matchu Date: Sat, 24 Sep 2022 13:18:40 -0700 Subject: [PATCH] Use AWS CLI for archive upload instead of s3cmd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Okay there we go, I was following Linode's guidance and ended up using a non-Amazon S3 client, but it turns out you can get the official Amazon AWS client to play with private services too, and it doesn't do the thing s3cmd does of trying to list every single file before doing anything 😅 This command _is_ doing weird stall-outs here and there, but is mostly just chugging along. It's not exactly fast, I imagine it'll take some time, but the fact that it's like. working. is huge lmao --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 8b4445b..7deec33 100644 --- a/package.json +++ b/package.json @@ -86,9 +86,9 @@ "validate-owls-data": "yarn run-script scripts/validate-owls-data.js", "archive:create:list-urls": "yarn run-script scripts/archive/create/list-urls.js", "archive:create:download-urls": "dotenv -- ./scripts/archive/create/download-urls.sh", - "s3cmd": "s3cmd --host=$(dotenv -p ARCHIVE_STORAGE_HOST) --host-bucket='%(bucket)s.'$(dotenv -p ARCHIVE_STORAGE_HOST) --access_key=$(dotenv -p ARCHIVE_STORAGE_READWRITE_ACCESS_KEY) --secret_key=$(dotenv -p ARCHIVE_STORAGE_READWRITE_SECRET_KEY)", - "archive:create:upload": "yarn s3cmd sync $(dotenv -p ARCHIVE_DIR)/ s3://$(dotenv -p ARCHIVE_STORAGE_BUCKET) --verbose", - "archive:create:upload-test": "yarn s3cmd sync $(dotenv -p ARCHIVE_DIR)/images.neopets.com/cp/items/data/000/000/000/1_8422bedbf9/ s3://$(dotenv -p ARCHIVE_STORAGE_BUCKET)/images.neopets.com/cp/items/data/000/000/000/1_8422bedbf9/ --verbose" + "aws": "AWS_ACCESS_KEY_ID=$(dotenv -p ARCHIVE_STORAGE_READWRITE_ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(dotenv -p ARCHIVE_STORAGE_READWRITE_SECRET_KEY) aws --endpoint=https://$(dotenv -p ARCHIVE_STORAGE_HOST)", + "archive:create:upload": "yarn aws s3 sync $(dotenv -p ARCHIVE_DIR) s3://$(dotenv -p ARCHIVE_STORAGE_BUCKET)", + "archive:create:upload-test": "yarn aws s3 sync $(dotenv -p ARCHIVE_DIR)/images.neopets.com/cp/items/data/000/000/000/1_8422bedbf9/ s3://$(dotenv -p ARCHIVE_STORAGE_BUCKET)/images.neopets.com/cp/items/data/000/000/000/1_8422bedbf9/" }, "browserslist": { "production": [