Add more importing to cron

We're gonna try saving a neologin cookie in the environment variables, and see how long-lived it is.
This commit is contained in:
Emi Matchu 2025-11-02 06:00:50 +00:00
parent 3dca3fe05a
commit 59da1fa04d
5 changed files with 245 additions and 201 deletions

View file

@ -449,12 +449,12 @@
minute: "*/10"
job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails nc_mall:sync'"
- name: Create 10min cron job to run `rails neopets:import:nc_mall`
- name: Create 10min cron job to run `rails neopets:import`
become_user: impress
cron:
name: "Impress: import NC Mall data"
name: "Impress: import Neopets data"
minute: "*/10"
job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails neopets:import:nc_mall'"
job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails neopets:import'"
- name: Create weekly cron job to run `rails public_data:commit`
become_user: impress

View file

@ -22,9 +22,26 @@ namespace :neopets do
]
namespace :import do
# Gets the neologin cookie, either from ENV['NEOLOGIN_COOKIE'] or by prompting.
# The neologin cookie is required for authenticated Neopets requests (Rainbow Pool,
# Styling Studio). It's generally long-lived (~1 year), so it can be stored in the
# environment and rotated manually when it expires.
#
# To extract the cookie:
# 1. Log into Neopets.com in your browser
# 2. Open browser DevTools > Application/Storage > Cookies
# 3. Find the "neologin" cookie value
# 4. Set NEOLOGIN_COOKIE environment variable to that value
# 5. Update production.env and redeploy when the cookie expires
task :neologin do
unless Neologin.cookie?
Neologin.cookie = STDIN.getpass("Neologin cookie: ")
# Try environment variable first (for automated cron jobs)
if ENV['NEOLOGIN_COOKIE'].present?
Neologin.cookie = ENV['NEOLOGIN_COOKIE']
else
# Fall back to interactive prompt (for local development)
Neologin.cookie = STDIN.getpass("Neologin cookie: ")
end
end
end
end

View file

@ -1,76 +1,83 @@
namespace "neopets:import" do
desc "Sync our NCMallRecord table with the live NC Mall"
task :nc_mall => :environment do
# Log to STDOUT.
Rails.logger = Logger.new(STDOUT)
begin
# Log to STDOUT.
Rails.logger = Logger.new(STDOUT)
puts "Importing from NC Mall…"
puts "Importing from NC Mall…"
# First, load all records of what's being sold in the live NC Mall. We load
# all categories from the menu and fetch all items from each. (We also
# de-duplicate the items, which is important because the same item can
# appear in multiple categories!)
live_item_records = load_all_nc_mall_items.uniq { |item| item[:id] }
# First, load all records of what's being sold in the live NC Mall. We load
# all categories from the menu and fetch all items from each. (We also
# de-duplicate the items, which is important because the same item can
# appear in multiple categories!)
live_item_records = load_all_nc_mall_items.uniq { |item| item[:id] }
# Then, get the existing NC Mall records in our database. (We include the
# items, to be able to output the item name during logging.)
existing_records = NCMallRecord.includes(:item).all
existing_records_by_item_id = existing_records.to_h { |r| [r.item_id, r] }
# Then, get the existing NC Mall records in our database. (We include the
# items, to be able to output the item name during logging.)
existing_records = NCMallRecord.includes(:item).all
existing_records_by_item_id = existing_records.to_h { |r| [r.item_id, r] }
# Additionally, check which of the item IDs in the live records are items
# we've seen before. (We'll skip records for items we don't know.)
live_item_ids = live_item_records.map { |r| r[:id] }
recognized_item_ids = Item.where(id: live_item_ids).pluck(:id).to_set
Rails.logger.debug "We found #{live_item_records.size} items, and we " +
"recognize #{recognized_item_ids.size} of them."
# Additionally, check which of the item IDs in the live records are items
# we've seen before. (We'll skip records for items we don't know.)
live_item_ids = live_item_records.map { |r| r[:id] }
recognized_item_ids = Item.where(id: live_item_ids).pluck(:id).to_set
Rails.logger.debug "We found #{live_item_records.size} items, and we " +
"recognize #{recognized_item_ids.size} of them."
# For each record in the live NC Mall, check if there's an existing record.
# If so, update it, and remove it from the existing records hash. If not,
# create it.
live_item_records.each do |record_data|
# If we don't recognize this item ID in our database already, skip it.
next unless recognized_item_ids.include?(record_data[:id])
# For each record in the live NC Mall, check if there's an existing record.
# If so, update it, and remove it from the existing records hash. If not,
# create it.
live_item_records.each do |record_data|
# If we don't recognize this item ID in our database already, skip it.
next unless recognized_item_ids.include?(record_data[:id])
record = existing_records_by_item_id.delete(record_data[:id]) ||
NCMallRecord.new
record.item_id = record_data[:id]
record.price = record_data[:price]
record.discount_price = record_data.dig(:discount, :price)
record.discount_begins_at = record_data.dig(:discount, :begins_at)
record.discount_ends_at = record_data.dig(:discount, :ends_at)
record = existing_records_by_item_id.delete(record_data[:id]) ||
NCMallRecord.new
record.item_id = record_data[:id]
record.price = record_data[:price]
record.discount_price = record_data.dig(:discount, :price)
record.discount_begins_at = record_data.dig(:discount, :begins_at)
record.discount_ends_at = record_data.dig(:discount, :ends_at)
if !record.changed?
Rails.logger.info "Skipping record for item #{record_data[:name]} " +
"(unchanged)"
next
end
if record.save
if record.previously_new_record?
Rails.logger.info "Created record for item #{record_data[:name]}"
else
Rails.logger.info "Updated record for item #{record_data[:name]}"
if !record.changed?
Rails.logger.info "Skipping record for item #{record_data[:name]} " +
"(unchanged)"
next
end
else
Rails.logger.error "Failed to save record for item " +
"#{record_data[:name]}: " +
"#{record.errors.full_messages.join("; ")}: " +
"#{record.inspect}"
end
end
# For each existing record remaining in the existing records hash, this
# means there was no live record corresponding to it during this sync.
# Delete it!
existing_records_by_item_id.values.each do |record|
item_name = record.item&.name || "<item not found>"
if record.destroy
Rails.logger.info "Destroyed record #{record.id} for item " +
"#{item_name}"
else
Rails.logger.error "Failed to destroy record #{record.id} for " +
"item #{item_name}: #{record.inspect}"
if record.save
if record.previously_new_record?
Rails.logger.info "Created record for item #{record_data[:name]}"
else
Rails.logger.info "Updated record for item #{record_data[:name]}"
end
else
Rails.logger.error "Failed to save record for item " +
"#{record_data[:name]}: " +
"#{record.errors.full_messages.join("; ")}: " +
"#{record.inspect}"
end
end
# For each existing record remaining in the existing records hash, this
# means there was no live record corresponding to it during this sync.
# Delete it!
existing_records_by_item_id.values.each do |record|
item_name = record.item&.name || "<item not found>"
if record.destroy
Rails.logger.info "Destroyed record #{record.id} for item " +
"#{item_name}"
else
Rails.logger.error "Failed to destroy record #{record.id} for " +
"item #{item_name}: #{record.inspect}"
end
end
rescue => e
Rails.logger.error "Failed to import NC Mall data: #{e.message}"
Rails.logger.error e.backtrace.join("\n")
Sentry.capture_exception(e, tags: { task: "neopets:import:nc_mall" })
raise
end
end
end

View file

@ -3,71 +3,81 @@ require "addressable/template"
namespace "neopets:import" do
desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes"
task :rainbow_pool => ["neopets:import:neologin", :environment] do
puts "Importing from Rainbow Pool…"
begin
puts "Importing from Rainbow Pool…"
all_species = Species.order(:name).to_a
all_pet_types = PetType.all.to_a
all_pet_types_by_species_id_and_color_id = all_pet_types.
to_h { |pt| [[pt.species_id, pt.color_id], pt] }
all_colors_by_name = Color.all.to_h { |c| [c.human_name.downcase, c] }
all_species = Species.order(:name).to_a
all_pet_types = PetType.all.to_a
all_pet_types_by_species_id_and_color_id = all_pet_types.
to_h { |pt| [[pt.species_id, pt.color_id], pt] }
all_colors_by_name = Color.all.to_h { |c| [c.human_name.downcase, c] }
hashes_by_color_name_by_species_id = {}
DTIRequests.load_many(max_at_once: 10) do |task|
num_loaded = 0
num_total = all_species.size
print "0/#{num_total} species loaded"
hashes_by_color_name_by_species_id = {}
DTIRequests.load_many(max_at_once: 10) do |task|
num_loaded = 0
num_total = all_species.size
print "0/#{num_total} species loaded"
all_species.each do |species|
task.async do
begin
hashes_by_color_name_by_species_id[species.id] =
RainbowPool.load_hashes_for_species(species.id, Neologin.cookie)
rescue => error
puts "Failed to load #{species.name} page, skipping: #{error.message}"
Sentry.capture_exception(error,
tags: { task: "neopets:import:rainbow_pool" },
contexts: { species: { name: species.name, id: species.id } })
end
num_loaded += 1
print "\r#{num_loaded}/#{num_total} species loaded"
end
end
end
all_species.each do |species|
task.async do
begin
hashes_by_color_name_by_species_id[species.id] =
RainbowPool.load_hashes_for_species(species.id, Neologin.cookie)
rescue => error
puts "Failed to load #{species.name} page, skipping: #{error.message}"
hashes_by_color_name = hashes_by_color_name_by_species_id[species.id]
next if hashes_by_color_name.nil?
changed_pet_types = []
hashes_by_color_name.each do |color_name, image_hash|
color = all_colors_by_name[color_name.downcase]
if color.nil?
puts "Skipping unrecognized color name: #{color_name}"
next
end
pet_type = all_pet_types_by_species_id_and_color_id[
[species.id, color.id]]
if pet_type.nil?
puts "Skipping unrecognized pet type: " +
"#{color_name} #{species.human_name}"
next
end
if pet_type.basic_image_hash.nil?
puts "Found new image hash: #{image_hash} (#{pet_type.human_name})"
pet_type.basic_image_hash = image_hash
changed_pet_types << pet_type
elsif pet_type.basic_image_hash != image_hash
puts "Updating image hash: #{image_hash} ({#{pet_type.human_name})"
pet_type.basic_image_hash = image_hash
changed_pet_types << pet_type
else
# No need to do anything with image hashes that match!
end
num_loaded += 1
print "\r#{num_loaded}/#{num_total} species loaded"
end
PetType.transaction { changed_pet_types.each(&:save!) }
puts "Saved #{changed_pet_types.size} image hashes for " +
"#{species.human_name}"
end
end
all_species.each do |species|
hashes_by_color_name = hashes_by_color_name_by_species_id[species.id]
next if hashes_by_color_name.nil?
changed_pet_types = []
hashes_by_color_name.each do |color_name, image_hash|
color = all_colors_by_name[color_name.downcase]
if color.nil?
puts "Skipping unrecognized color name: #{color_name}"
next
end
pet_type = all_pet_types_by_species_id_and_color_id[
[species.id, color.id]]
if pet_type.nil?
puts "Skipping unrecognized pet type: " +
"#{color_name} #{species.human_name}"
next
end
if pet_type.basic_image_hash.nil?
puts "Found new image hash: #{image_hash} (#{pet_type.human_name})"
pet_type.basic_image_hash = image_hash
changed_pet_types << pet_type
elsif pet_type.basic_image_hash != image_hash
puts "Updating image hash: #{image_hash} ({#{pet_type.human_name})"
pet_type.basic_image_hash = image_hash
changed_pet_types << pet_type
else
# No need to do anything with image hashes that match!
end
end
PetType.transaction { changed_pet_types.each(&:save!) }
puts "Saved #{changed_pet_types.size} image hashes for " +
"#{species.human_name}"
rescue => e
puts "Failed to import Rainbow Pool data: #{e.message}"
puts e.backtrace.join("\n")
Sentry.capture_exception(e, tags: { task: "neopets:import:rainbow_pool" })
raise
end
end
end

View file

@ -1,99 +1,109 @@
namespace "neopets:import" do
desc "Import alt style info from the NC Styling Studio"
task :styling_studio => ["neopets:import:neologin", :environment] do
puts "Importing from Styling Studio…"
begin
puts "Importing from Styling Studio…"
all_species = Species.order(:name).to_a
all_species = Species.order(:name).to_a
# Load 10 species pages from the NC Mall at a time.
styles_by_species_id = {}
DTIRequests.load_many(max_at_once: 10) do |task|
num_loaded = 0
num_total = all_species.size
print "0/#{num_total} species loaded"
# Load 10 species pages from the NC Mall at a time.
styles_by_species_id = {}
DTIRequests.load_many(max_at_once: 10) do |task|
num_loaded = 0
num_total = all_species.size
print "0/#{num_total} species loaded"
all_species.each do |species|
task.async {
begin
styles_by_species_id[species.id] = Neopets::NCMall.load_styles(
species_id: species.id,
neologin: Neologin.cookie,
)
rescue => error
puts "\n⚠️ Error loading for #{species.human_name}, skipping: #{error.message}"
Sentry.capture_exception(error,
tags: { task: "neopets:import:styling_studio" },
contexts: { species: { name: species.human_name, id: species.id } })
end
num_loaded += 1
print "\r#{num_loaded}/#{num_total} species loaded"
}
end
end
print "\n"
style_ids = styles_by_species_id.values.flatten(1).map { |s| s[:oii] }
style_records_by_id =
AltStyle.where(id: style_ids).to_h { |as| [as.id, as] }
all_species.each do |species|
task.async {
begin
styles_by_species_id[species.id] = Neopets::NCMall.load_styles(
species_id: species.id,
neologin: Neologin.cookie,
)
rescue => error
puts "\n⚠️ Error loading for #{species.human_name}, skipping: #{error.message}"
styles = styles_by_species_id[species.id]
next if styles.nil?
counts = {changed: 0, unchanged: 0, skipped: 0}
styles.each do |style|
record = style_records_by_id[style[:oii]]
label = "#{style[:name]} (#{style[:oii]})"
if record.nil?
puts "❔ [#{label}]: Not modeled yet, skipping"
counts[:skipped] += 1
next
end
num_loaded += 1
print "\r#{num_loaded}/#{num_total} species loaded"
}
end
end
print "\n"
style_ids = styles_by_species_id.values.flatten(1).map { |s| s[:oii] }
style_records_by_id =
AltStyle.where(id: style_ids).to_h { |as| [as.id, as] }
if !record.real_full_name?
record.full_name = style[:name]
puts "✅ [#{label}]: Full name is now #{style[:name].inspect}"
elsif record.full_name != style[:name]
puts "⚠️ [#{label}: Full name may have changed, handle manually? " +
"#{record.full_name.inspect} -> #{style[:name].inspect}"
end
all_species.each do |species|
styles = styles_by_species_id[species.id]
next if styles.nil?
if !record.real_thumbnail_url?
record.thumbnail_url = style[:image]
puts "✅ [#{label}]: Thumbnail URL is now #{style[:image].inspect}"
elsif record.thumbnail_url != style[:image]
puts "⚠️ [#{label}: Thumbnail URL may have changed, handle manually? " +
"#{record.thumbnail_url.inspect} -> #{style[:image].inspect}"
end
counts = {changed: 0, unchanged: 0, skipped: 0}
styles.each do |style|
record = style_records_by_id[style[:oii]]
label = "#{style[:name]} (#{style[:oii]})"
if record.nil?
puts "❔ [#{label}]: Not modeled yet, skipping"
counts[:skipped] += 1
next
end
if !record.real_full_name?
record.full_name = style[:name]
puts "✅ [#{label}]: Full name is now #{style[:name].inspect}"
elsif record.full_name != style[:name]
puts "⚠️ [#{label}: Full name may have changed, handle manually? " +
"#{record.full_name.inspect} -> #{style[:name].inspect}"
end
if !record.real_thumbnail_url?
record.thumbnail_url = style[:image]
puts "✅ [#{label}]: Thumbnail URL is now #{style[:image].inspect}"
elsif record.thumbnail_url != style[:image]
puts "⚠️ [#{label}: Thumbnail URL may have changed, handle manually? " +
"#{record.thumbnail_url.inspect} -> #{style[:image].inspect}"
end
if style[:name].end_with?(record.pet_name)
new_series_name = style[:name].split(record.pet_name).first.strip
if !record.real_series_name?
record.series_name = new_series_name
puts "✅ [#{label}]: Series name is now #{new_series_name.inspect}"
elsif record.series_name != new_series_name
if ENV['FORCE'] == '1'
puts "❗ [#{label}]: Series name forcibly changed: " +
"#{record.series_name.inspect} -> #{new_series_name.inspect}"
if style[:name].end_with?(record.pet_name)
new_series_name = style[:name].split(record.pet_name).first.strip
if !record.real_series_name?
record.series_name = new_series_name
else
puts "⚠️ [#{label}]: Series name may have changed, handle manually? " +
"#{record.series_name.inspect} -> #{new_series_name.inspect}"
puts "✅ [#{label}]: Series name is now #{new_series_name.inspect}"
elsif record.series_name != new_series_name
if ENV['FORCE'] == '1'
puts "❗ [#{label}]: Series name forcibly changed: " +
"#{record.series_name.inspect} -> #{new_series_name.inspect}"
record.series_name = new_series_name
else
puts "⚠️ [#{label}]: Series name may have changed, handle manually? " +
"#{record.series_name.inspect} -> #{new_series_name.inspect}"
end
end
else
puts "⚠️ [#{label}]: Unable to detect series name, handle manually? " +
"#{record.pet_name.inspect} <-> #{style[:name].inspect}"
end
else
puts "⚠️ [#{label}]: Unable to detect series name, handle manually? " +
"#{record.pet_name.inspect} <-> #{style[:name].inspect}"
if record.changed?
counts[:changed] += 1
else
counts[:unchanged] += 1
end
record.save!
end
if record.changed?
counts[:changed] += 1
else
counts[:unchanged] += 1
end
record.save!
puts "#{species.human_name}: #{counts[:changed]} changed, " +
"#{counts[:unchanged]} unchanged, #{counts[:skipped]} skipped"
end
puts "#{species.human_name}: #{counts[:changed]} changed, " +
"#{counts[:unchanged]} unchanged, #{counts[:skipped]} skipped"
rescue => e
puts "Failed to import Styling Studio data: #{e.message}"
puts e.backtrace.join("\n")
Sentry.capture_exception(e, tags: { task: "neopets:import:styling_studio" })
raise
end
end
end