From b84c8ba34e5330faf4083866dd6ef7aea9a7cf1e Mon Sep 17 00:00:00 2001 From: Matchu Date: Sun, 13 Nov 2022 07:03:35 -0800 Subject: [PATCH] Script to set up dev db with public DTI data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now, someone with production DB access can run `yarn db:export:public-data` to create `public-data-constants.sql` and `public-data-from-modeling.sql`. Then, someone setting up their dev database can run `yarn db:setup-dev:full` and get all the wearables data imported right into their dev database! I'm noticing just how poorly I'm keeping up with my own goals for finishing up DTI, and wondering if now is a good time to circle back to some old offers for code contributions I got last year… I also just figure that making this app Possible To Run with a backup of the basic public database is like. a pretty handy thing to have for archival's sake imo Note that, for this change, we also set up Git LFS (Large File Storage). Github should be automatically compatible with this! It's a way to not write the whole 30MB database dump into the repository history, and instead keep it in a secondary filestore, because Git's core algorithms aren't really built to handle large blobs of data very well. Users setting up their dev environment will therefore also need to have Git LFS installed for this script to work! (Otherwise, they'll see a "pointer" file in `public-data-from-modeling.sql.gz` that contains some metadata about the file state but not the data itself.) --- .gitattributes | 1 + .husky/post-checkout | 3 + .husky/post-commit | 3 + .husky/post-merge | 3 + .husky/pre-push | 3 + package.json | 7 +- scripts/db/export/public-data.sh | 7 ++ scripts/db/export/schema.sh | 11 ++ .../public-data-constants.sql} | 2 +- scripts/db/public-data-from-modeling.sql.gz | 3 + .../schema-for-id.sql} | 2 +- .../schema-for-impress.sql} | 107 +++++++++++++++++- scripts/db/setup-dev/full.sh | 3 + scripts/db/setup-dev/minimal.sh | 6 + 14 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 .gitattributes create mode 100755 .husky/post-checkout create mode 100755 .husky/post-commit create mode 100755 .husky/post-merge create mode 100755 .husky/pre-push create mode 100755 scripts/db/export/public-data.sh create mode 100755 scripts/db/export/schema.sh rename scripts/{setup-mysql-dev-constants.sql => db/public-data-constants.sql} (99%) create mode 100644 scripts/db/public-data-from-modeling.sql.gz rename scripts/{setup-mysql-dev-schema-id.sql => db/schema-for-id.sql} (98%) rename scripts/{setup-mysql-dev-schema-impress.sql => db/schema-for-impress.sql} (75%) create mode 100755 scripts/db/setup-dev/full.sh create mode 100755 scripts/db/setup-dev/minimal.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8aed542 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sql.gz filter=lfs diff=lfs merge=lfs -text diff --git a/.husky/post-checkout b/.husky/post-checkout new file mode 100755 index 0000000..cab40f2 --- /dev/null +++ b/.husky/post-checkout @@ -0,0 +1,3 @@ +#!/bin/sh +command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout.\n"; exit 2; } +git lfs post-checkout "$@" diff --git a/.husky/post-commit b/.husky/post-commit new file mode 100755 index 0000000..9443f41 --- /dev/null +++ b/.husky/post-commit @@ -0,0 +1,3 @@ +#!/bin/sh +command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-commit.\n"; exit 2; } +git lfs post-commit "$@" diff --git a/.husky/post-merge b/.husky/post-merge new file mode 100755 index 0000000..828b708 --- /dev/null +++ b/.husky/post-merge @@ -0,0 +1,3 @@ +#!/bin/sh +command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-merge.\n"; exit 2; } +git lfs post-merge "$@" diff --git a/.husky/pre-push b/.husky/pre-push new file mode 100755 index 0000000..81a9cc6 --- /dev/null +++ b/.husky/pre-push @@ -0,0 +1,3 @@ +#!/bin/sh +command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/pre-push.\n"; exit 2; } +git lfs pre-push "$@" diff --git a/package.json b/package.json index 41b2c15..f3721e3 100644 --- a/package.json +++ b/package.json @@ -75,7 +75,6 @@ "mysql-dev": "mysql --host=localhost --user=impress_2020_dev --password=impress_2020_dev --database=openneo_impress", "mysql-admin": "mysql --host=impress.openneo.net --user=matchu --password --database=openneo_impress", "mysqldump": "mysqldump --host=impress.openneo.net --user=$(dotenv -p IMPRESS_MYSQL_USER) --password=$(dotenv -p IMPRESS_MYSQL_PASSWORD)", - "download-mysql-schema": "yarn --silent mysqldump --no-data openneo_impress closet_hangers closet_lists items item_translations modeling_logs parents_swf_assets pet_types pet_states swf_assets users | sed 's/ AUTO_INCREMENT=[0-9]*//g' > scripts/setup-mysql-dev-schema-impress.sql && yarn --silent mysqldump --no-data openneo_id users | sed 's/ AUTO_INCREMENT=[0-9]*//g' > scripts/setup-mysql-dev-schema-id.sql && yarn --silent mysqldump openneo_impress species species_translations colors color_translations zones zone_translations > scripts/setup-mysql-dev-constants.sql", "setup-mysql": "yarn mysql-admin < scripts/setup-mysql.sql", "setup-mysql-dev": "yarn mysql-dev < scripts/setup-mysql-dev-constants.sql && yarn mysql-dev < scripts/setup-mysql-dev-schema-impress.sql && yarn mysql-dev --database=openneo_id < scripts/setup-mysql-dev-schema-id.sql", "run-script": "ts-node --compiler=typescript-cached-transpile --transpile-only -r dotenv/config", @@ -95,7 +94,11 @@ "archive:create:delta": "dotenv -- ./scripts/archive/create/delta.sh", "archive:upload:full": "dotenv -- ./scripts/archive/upload/full.sh", "archive:upload:delta": "dotenv -- ./scripts/archive/upload/delta.sh", - "archive:upload:test": "dotenv -- ./scripts/archive/upload/test.sh" + "archive:upload:test": "dotenv -- ./scripts/archive/upload/test.sh", + "db:export:schema": "./scripts/db/export/schema.sh", + "db:export:public-data": "./scripts/db/export/public-data.sh", + "db:setup-dev:minimal": "./scripts/db/setup-dev/minimal.sh", + "db:setup-dev:full": "./scripts/db/setup-dev/full.sh" }, "browserslist": { "production": [ diff --git a/scripts/db/export/public-data.sh b/scripts/db/export/public-data.sh new file mode 100755 index 0000000..366b039 --- /dev/null +++ b/scripts/db/export/public-data.sh @@ -0,0 +1,7 @@ +yarn --silent mysqldump openneo_impress species species_translations colors \ + color_translations zones zone_translations \ + > $(dirname $0)/../public-data-constants.sql \ +&& yarn --silent mysqldump openneo_impress items item_translations \ + parents_swf_assets pet_states pet_types swf_assets \ + | gzip -c \ + > $(dirname $0)/../public-data-from-modeling.sql.gz \ No newline at end of file diff --git a/scripts/db/export/schema.sh b/scripts/db/export/schema.sh new file mode 100755 index 0000000..5f296ed --- /dev/null +++ b/scripts/db/export/schema.sh @@ -0,0 +1,11 @@ +yarn --silent mysqldump --no-data openneo_impress closet_hangers closet_lists \ + colors color_translations items item_translations modeling_logs \ + parents_swf_assets pet_types pet_states species species_translations \ + swf_assets users zones zone_translations \ + | \ + sed 's/ AUTO_INCREMENT=[0-9]*//g' \ + > $(dirname $0)/../schema-for-impress.sql \ +&& yarn --silent mysqldump --no-data openneo_id users \ + | \ + sed 's/ AUTO_INCREMENT=[0-9]*//g' \ + > $(dirname $0)/../schema-for-id.sql \ No newline at end of file diff --git a/scripts/setup-mysql-dev-constants.sql b/scripts/db/public-data-constants.sql similarity index 99% rename from scripts/setup-mysql-dev-constants.sql rename to scripts/db/public-data-constants.sql index 1664bed..fc110dd 100644 --- a/scripts/setup-mysql-dev-constants.sql +++ b/scripts/db/public-data-constants.sql @@ -189,4 +189,4 @@ UNLOCK TABLES; /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2022-09-13 21:04:49 +-- Dump completed on 2022-11-13 6:46:52 diff --git a/scripts/db/public-data-from-modeling.sql.gz b/scripts/db/public-data-from-modeling.sql.gz new file mode 100644 index 0000000..71bf3cd --- /dev/null +++ b/scripts/db/public-data-from-modeling.sql.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f9457e53cc4043fae59cbe99dff0254a2bc7ca31c3ac396f56b206cc9b3ec1 +size 39718417 diff --git a/scripts/setup-mysql-dev-schema-id.sql b/scripts/db/schema-for-id.sql similarity index 98% rename from scripts/setup-mysql-dev-schema-id.sql rename to scripts/db/schema-for-id.sql index 385858b..d25c359 100644 --- a/scripts/setup-mysql-dev-schema-id.sql +++ b/scripts/db/schema-for-id.sql @@ -56,4 +56,4 @@ CREATE TABLE `users` ( /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2022-09-13 21:04:40 +-- Dump completed on 2022-11-13 6:46:08 diff --git a/scripts/setup-mysql-dev-schema-impress.sql b/scripts/db/schema-for-impress.sql similarity index 75% rename from scripts/setup-mysql-dev-schema-impress.sql rename to scripts/db/schema-for-impress.sql index 9b36184..6a8c06c 100644 --- a/scripts/setup-mysql-dev-schema-impress.sql +++ b/scripts/db/schema-for-impress.sql @@ -62,6 +62,42 @@ CREATE TABLE `closet_lists` ( ) ENGINE=InnoDB DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; +-- +-- Table structure for table `colors` +-- + +DROP TABLE IF EXISTS `colors`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `colors` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `basic` tinyint(1) DEFAULT NULL, + `standard` tinyint(1) DEFAULT NULL, + `prank` tinyint(1) NOT NULL DEFAULT '0', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `color_translations` +-- + +DROP TABLE IF EXISTS `color_translations`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `color_translations` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `color_id` int(11) DEFAULT NULL, + `locale` varchar(255) DEFAULT NULL, + `name` varchar(255) DEFAULT NULL, + `created_at` datetime DEFAULT NULL, + `updated_at` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `index_color_translations_on_color_id` (`color_id`), + KEY `index_color_translations_on_locale` (`locale`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + -- -- Table structure for table `items` -- @@ -201,6 +237,39 @@ CREATE TABLE `pet_states` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; +-- +-- Table structure for table `species` +-- + +DROP TABLE IF EXISTS `species`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `species` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `species_translations` +-- + +DROP TABLE IF EXISTS `species_translations`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `species_translations` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `species_id` int(11) DEFAULT NULL, + `locale` varchar(255) DEFAULT NULL, + `name` varchar(255) DEFAULT NULL, + `created_at` datetime DEFAULT NULL, + `updated_at` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `index_species_translations_on_species_id` (`species_id`), + KEY `index_species_translations_on_locale` (`locale`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + -- -- Table structure for table `swf_assets` -- @@ -255,6 +324,42 @@ CREATE TABLE `users` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `zones` +-- + +DROP TABLE IF EXISTS `zones`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `zones` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `depth` int(11) DEFAULT NULL, + `type_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `zone_translations` +-- + +DROP TABLE IF EXISTS `zone_translations`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `zone_translations` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `zone_id` int(11) DEFAULT NULL, + `locale` varchar(255) DEFAULT NULL, + `label` varchar(255) DEFAULT NULL, + `plain_label` varchar(255) DEFAULT NULL, + `created_at` datetime DEFAULT NULL, + `updated_at` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `index_zone_translations_on_zone_id` (`zone_id`), + KEY `index_zone_translations_on_locale` (`locale`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; @@ -265,4 +370,4 @@ CREATE TABLE `users` ( /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2022-09-13 21:04:38 +-- Dump completed on 2022-11-13 6:46:05 diff --git a/scripts/db/setup-dev/full.sh b/scripts/db/setup-dev/full.sh new file mode 100755 index 0000000..cb4c680 --- /dev/null +++ b/scripts/db/setup-dev/full.sh @@ -0,0 +1,3 @@ +$(dirname $0)/minimal.sh \ +&& gzip -d -c $(dirname $0)/../public-data-from-modeling.sql.gz \ + | yarn mysql-dev --database=openneo_impress \ No newline at end of file diff --git a/scripts/db/setup-dev/minimal.sh b/scripts/db/setup-dev/minimal.sh new file mode 100755 index 0000000..3cd4c54 --- /dev/null +++ b/scripts/db/setup-dev/minimal.sh @@ -0,0 +1,6 @@ +yarn mysql-dev --database=openneo_impress \ + < $(dirname $0)/../schema-for-impress.sql \ +&& yarn mysql-dev --database=openneo_impress \ + < $(dirname $0)/../public-data-constants.sql \ +&& yarn mysql-dev --database=openneo_id \ + < $(dirname $0)/../schema-for-id.sql \ No newline at end of file