diff --git a/.gitignore b/.gitignore index 183ccfbd..26c17459 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ tmp/**/* /app/assets/builds/* !/app/assets/builds/.keep +/public/public-data + /node_modules .pnp.* diff --git a/Gemfile b/Gemfile index fe320aec..4ccff860 100644 --- a/Gemfile +++ b/Gemfile @@ -79,6 +79,9 @@ gem "stackprof", "~> 0.2.25" gem "sentry-ruby", "~> 5.12" gem "sentry-rails", "~> 5.12" +# For tasks that use shell commands. +gem "shell", "~> 0.8.1" + # For automated testing. group :test do gem 'sqlite3', '~> 1.7' diff --git a/Gemfile.lock b/Gemfile.lock index 56689f5b..14075c2f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -139,6 +139,7 @@ GEM railties (>= 3.2) drb (2.2.0) ruby2_keywords + e2mmap (0.1.0) erubi (1.12.0) execjs (2.9.1) falcon (0.43.0) @@ -316,6 +317,9 @@ GEM sentry-ruby (~> 5.16.1) sentry-ruby (5.16.1) concurrent-ruby (~> 1.0, >= 1.0.2) + shell (0.8.1) + e2mmap + sync sprockets (4.2.1) concurrent-ruby (~> 1.0) rack (>= 2.2.4, < 4) @@ -327,6 +331,7 @@ GEM mini_portile2 (~> 2.8.0) stackprof (0.2.26) stringio (3.1.0) + sync (0.5.0) temple (0.10.3) terser (1.2.0) execjs (>= 0.3.0, < 3) @@ -385,6 +390,7 @@ DEPENDENCIES sass-rails (~> 6.0) sentry-rails (~> 5.12) sentry-ruby (~> 5.12) + shell (~> 0.8.1) sprockets (~> 4.2) sqlite3 (~> 1.7) stackprof (~> 0.2.25) diff --git a/config/environments/development.rb b/config/environments/development.rb index 6eac4c70..dbdfcab7 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -112,4 +112,8 @@ Rails.application.configure do # we keep this in a long-term location instead!) config.neopets_media_archive_root = Rails.root / "tmp" / "neopets_media_archive" / "development" + + # When developing the `public_data:commit` command, save to the local `tmp` + # folder. (In production, we keep this in a long-term location instead!) + config.public_data_root = Rails.root / "tmp" / "public_data" end diff --git a/config/environments/production.rb b/config/environments/production.rb index 20858d10..1a11dbb1 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -130,4 +130,8 @@ Rails.application.configure do # Save the Neopets Media Archive in `/var/lib/neopets-media-archive`, a # long-term storage location. config.neopets_media_archive_root = "/var/lib/neopets-media-archive" + + # Save our public data exports in `public/public-data`. (This should be + # symlinked to a shared folder persisted across all versions.) + config.public_data_root = Rails.root / "public" / "public-data" end diff --git a/deploy/deploy.yml b/deploy/deploy.yml index d6a5dd61..e40a4e5f 100644 --- a/deploy/deploy.yml +++ b/deploy/deploy.yml @@ -51,6 +51,12 @@ - "--exclude=.git" - "--filter=':- .gitignore'" + - name: Link the public-data folder to the shared public-data folder + file: + src: "{{ remote_project_root }}/shared/public-data" + dest: "{{ remote_app_root }}/public/public-data" + state: link + - name: Configure Bundler to run in deployment mode command: chdir: "{{ remote_app_root }}" diff --git a/deploy/files/sites-available/impress.conf b/deploy/files/sites-available/impress.conf index 91cbcb0c..3401a70b 100644 --- a/deploy/files/sites-available/impress.conf +++ b/deploy/files/sites-available/impress.conf @@ -33,6 +33,10 @@ server { add_header ETag ""; } + location /public-data/ { + autoindex on; + } + # On status 503, return the maintenance page. (We'll trigger this ourselves # in the @app location, if $maintenance is on.) error_page 503 /maintenance.html; diff --git a/deploy/setup.yml b/deploy/setup.yml index 2646fbe9..2c30014f 100644 --- a/deploy/setup.yml +++ b/deploy/setup.yml @@ -275,6 +275,13 @@ - Reload systemctl - Restart impress + - name: Create the shared public-data folder + become_user: impress + file: + dest: /srv/impress/shared/public-data + mode: "755" + state: directory + - name: Create service file for impress copy: src: files/impress.service diff --git a/lib/tasks/public_data.rake b/lib/tasks/public_data.rake new file mode 100644 index 00000000..d4416dc0 --- /dev/null +++ b/lib/tasks/public_data.rake @@ -0,0 +1,67 @@ +require "open3" + +desc "Tools to save and import DTI's public modeling data" +namespace :public_data do + desc "Save the local database's public data to a local file" + task :commit, [:name] => :environment do |_, args| + if Rails.env.development? + puts "NOTE: The `public_data:commit` task is primarily meant to be " + + "run in production, to create public data files we can copy to our " + + "development machines via `public_data:pull`. I'll still run it " + + "locally and save to #{Rails.configuration.public_data_root}, though!" + end + + config = ApplicationRecord.connection_db_config.configuration_hash + + # Generate a filename from the current time, and the option name argument + # provided to the command (e.g. `rails public_data:commit[scheduled]`). + timestamp = Time.now.utc.iso8601.gsub(':', '_') + name = args.fetch(:name, "manual") + filename = "#{timestamp}-#{name}.sql.gz" + dest_path = Rails.configuration.public_data_root / filename + + args = [] + + # The connection details for our database! + args << "--host=#{config[:host]}" if config[:host] + args << "--user=#{config[:username]}" if config[:username] + args << "--password=#{config[:password]}" if config[:password] + + # Don't lock the database to do it! + args << "--single-transaction" + + # Dump the public data tables from the primary database. + args << config.fetch(:database) + args += %w(species colors zones) # manual constants + args += %w(alt_styles items parents_swf_assets pet_states pet_types + swf_assets) # from modeling + + # Set up a shell, and register the commands we need. + Shell.def_system_command("mysqldump") + Shell.def_system_command("gzip") + sh = Shell.new + + # Ensure the output directory exists. + dest_path.dirname.mkpath + + # Run mysqldump, pipe it into gzip, and output to the destination file. + sh.mysqldump(*args) | sh.gzip("-c") > dest_path.to_s + puts "Saved dump to #{dest_path}" + + # Link this latest dump as `latest.sql.gz`. + latest_path = Rails.configuration.public_data_root / "latest.sql.gz" + File.unlink(latest_path) if File.exist?(latest_path) + File.symlink(dest_path, latest_path) + puts "Linked dump to #{latest_path}" + end + + desc "Pull and import the latest public data from production (dev only)" + task :pull do + unless Rails.env.development? + raise "Can only pull public data in development mode! This helps us " + + "ensure we won't overwrite the production database accidentally." + end + + raise NotImplementedError, "TODO!" + end +end diff --git a/vendor/cache/e2mmap-0.1.0.gem b/vendor/cache/e2mmap-0.1.0.gem new file mode 100644 index 00000000..c5d26939 Binary files /dev/null and b/vendor/cache/e2mmap-0.1.0.gem differ diff --git a/vendor/cache/shell-0.8.1.gem b/vendor/cache/shell-0.8.1.gem new file mode 100644 index 00000000..d7d8bf56 Binary files /dev/null and b/vendor/cache/shell-0.8.1.gem differ diff --git a/vendor/cache/sync-0.5.0.gem b/vendor/cache/sync-0.5.0.gem new file mode 100644 index 00000000..1aa977b6 Binary files /dev/null and b/vendor/cache/sync-0.5.0.gem differ