Refactor Neopets import tasks all into a neopets:import namespace

and with a `rails neopets:import` task you can call to do them all at
once!

I'm gonna do some other stuff here too to make `neopets:import` easier
to call all in one go, like prompting for the Neologin cookie just
once at the start.

Note that this changes the cron setup, so you gotta run
`bin/deploy:setup` after this deploys!
This commit is contained in:
Emi Matchu 2024-11-16 11:57:29 -08:00
parent 1d1dc15320
commit a57b3629db
5 changed files with 29 additions and 8 deletions

View file

@ -442,13 +442,21 @@
mode: "755" mode: "755"
state: directory state: directory
- name: Create 10min cron job to run `rails nc_mall:sync` - name: Remove 10min cron job to run `rails nc_mall:sync`
become_user: impress become_user: impress
cron: cron:
state: absent
name: "Impress: sync NC Mall data" name: "Impress: sync NC Mall data"
minute: "*/10" minute: "*/10"
job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails nc_mall:sync'" job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails nc_mall:sync'"
- name: Create 10min cron job to run `rails neopets:import:nc_mall`
become_user: impress
cron:
name: "Impress: import NC Mall data"
minute: "*/10"
job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails neopets:import:nc_mall'"
- name: Create weekly cron job to run `rails public_data:commit` - name: Create weekly cron job to run `rails public_data:commit`
become_user: impress become_user: impress
cron: cron:

View file

@ -0,0 +1,7 @@
namespace :neopets do
task :import => [
"neopets:import:nc_mall",
"neopets:import:rainbow_pool",
"neopets:import:styling_studio",
]
end

View file

@ -1,9 +1,11 @@
namespace :nc_mall do namespace "neopets:import" do
desc "Sync our NCMallRecord table with the live NC Mall" desc "Sync our NCMallRecord table with the live NC Mall"
task :sync => :environment do task :nc_mall => :environment do
# Log to STDOUT. # Log to STDOUT.
Rails.logger = Logger.new(STDOUT) Rails.logger = Logger.new(STDOUT)
puts "Importing from NC Mall…"
# First, load all records of what's being sold in the live NC Mall. We load # First, load all records of what's being sold in the live NC Mall. We load
# the homepage and all pages linked from the main document, and extract the # the homepage and all pages linked from the main document, and extract the
# items from each. (We also de-duplicate the items, which is important # items from each. (We also de-duplicate the items, which is important

View file

@ -1,11 +1,13 @@
require "addressable/template" require "addressable/template"
require "async/http/internet/instance" require "async/http/internet/instance"
namespace :rainbow_pool do namespace "neopets:import" do
desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes" desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes"
task :import => :environment do task :rainbow_pool => :environment do
neologin = STDIN.getpass("Neologin cookie: ") neologin = STDIN.getpass("Neologin cookie: ")
puts "Importing from Rainbow Pool…"
all_pet_types = PetType.all.to_a all_pet_types = PetType.all.to_a
all_pet_types_by_species_id_and_color_id = all_pet_types. all_pet_types_by_species_id_and_color_id = all_pet_types.
to_h { |pt| [[pt.species_id, pt.color_id], pt] } to_h { |pt| [[pt.species_id, pt.color_id], pt] }

View file

@ -1,8 +1,10 @@
namespace :alt_styles do namespace "neopets:import" do
desc "Import alt style info from the NC Mall" desc "Import alt style info from the NC Styling Studio"
task :import => :environment do task :styling_studio => :environment do
neologin = STDIN.getpass("Neologin cookie: ") neologin = STDIN.getpass("Neologin cookie: ")
puts "Importing from Styling Studio…"
all_species = Species.order(:name).to_a all_species = Species.order(:name).to_a
# Load 10 species pages from the NC Mall at a time. # Load 10 species pages from the NC Mall at a time.