OpenNeo/impress
OpenNeo/impress
1
0
Fork 1
Code Pull requests Activity

Add more importing to cron

Browse source 
We're gonna try saving a neologin cookie in the environment variables, and see how long-lived it is.
This commit is contained in:
Emi Matchu 2025-11-02 06:00:50 +00:00
parent 3dca3fe05a
commit 59da1fa04d
5 changed files with 245 additions and 201 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
deploy/setup.yml
View file

@ -449,12 +449,12 @@
minute: "*/10" minute: "*/10"
job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails nc_mall:sync'" job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails nc_mall:sync'"
- name: Create 10min cron job to run `rails neopets:import:nc_mall` - name: Create 10min cron job to run `rails neopets:import`
become_user: impress become_user: impress
cron: cron:
name: "Impress: import NC Mall data" name: "Impress: import Neopets data"
minute: "*/10" minute: "*/10"
job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails neopets:import:nc_mall'" job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails neopets:import'"
- name: Create weekly cron job to run `rails public_data:commit` - name: Create weekly cron job to run `rails public_data:commit`
become_user: impress become_user: impress

19
lib/tasks/neopets/import.rake
View file

@ -22,9 +22,26 @@ namespace :neopets do
] ]
namespace :import do namespace :import do
# Gets the neologin cookie, either from ENV['NEOLOGIN_COOKIE'] or by prompting.
# The neologin cookie is required for authenticated Neopets requests (Rainbow Pool,
# Styling Studio). It's generally long-lived (~1 year), so it can be stored in the
# environment and rotated manually when it expires.
#
# To extract the cookie:
# 1. Log into Neopets.com in your browser
# 2. Open browser DevTools > Application/Storage > Cookies
# 3. Find the "neologin" cookie value
# 4. Set NEOLOGIN_COOKIE environment variable to that value
# 5. Update production.env and redeploy when the cookie expires
task :neologin do task :neologin do
unless Neologin.cookie? unless Neologin.cookie?
Neologin.cookie = STDIN.getpass("Neologin cookie: ") # Try environment variable first (for automated cron jobs)
if ENV['NEOLOGIN_COOKIE'].present?
Neologin.cookie = ENV['NEOLOGIN_COOKIE']
else
# Fall back to interactive prompt (for local development)
Neologin.cookie = STDIN.getpass("Neologin cookie: ")
end
end end
end end
end end

127
lib/tasks/neopets/import/nc_mall.rake
View file

@ -1,76 +1,83 @@
namespace "neopets:import" do namespace "neopets:import" do
desc "Sync our NCMallRecord table with the live NC Mall" desc "Sync our NCMallRecord table with the live NC Mall"
task :nc_mall => :environment do task :nc_mall => :environment do
# Log to STDOUT. begin
Rails.logger = Logger.new(STDOUT) # Log to STDOUT.
Rails.logger = Logger.new(STDOUT)
puts "Importing from NC Mall…" puts "Importing from NC Mall…"
# First, load all records of what's being sold in the live NC Mall. We load # First, load all records of what's being sold in the live NC Mall. We load
# all categories from the menu and fetch all items from each. (We also # all categories from the menu and fetch all items from each. (We also
# de-duplicate the items, which is important because the same item can # de-duplicate the items, which is important because the same item can
# appear in multiple categories!) # appear in multiple categories!)
live_item_records = load_all_nc_mall_items.uniq { |item| item[:id] } live_item_records = load_all_nc_mall_items.uniq { |item| item[:id] }
# Then, get the existing NC Mall records in our database. (We include the # Then, get the existing NC Mall records in our database. (We include the
# items, to be able to output the item name during logging.) # items, to be able to output the item name during logging.)
existing_records = NCMallRecord.includes(:item).all existing_records = NCMallRecord.includes(:item).all
existing_records_by_item_id = existing_records.to_h { |r| [r.item_id, r] } existing_records_by_item_id = existing_records.to_h { |r| [r.item_id, r] }
# Additionally, check which of the item IDs in the live records are items # Additionally, check which of the item IDs in the live records are items
# we've seen before. (We'll skip records for items we don't know.) # we've seen before. (We'll skip records for items we don't know.)
live_item_ids = live_item_records.map { |r| r[:id] } live_item_ids = live_item_records.map { |r| r[:id] }
recognized_item_ids = Item.where(id: live_item_ids).pluck(:id).to_set recognized_item_ids = Item.where(id: live_item_ids).pluck(:id).to_set
Rails.logger.debug "We found #{live_item_records.size} items, and we " + Rails.logger.debug "We found #{live_item_records.size} items, and we " +
"recognize #{recognized_item_ids.size} of them." "recognize #{recognized_item_ids.size} of them."
# For each record in the live NC Mall, check if there's an existing record. # For each record in the live NC Mall, check if there's an existing record.
# If so, update it, and remove it from the existing records hash. If not, # If so, update it, and remove it from the existing records hash. If not,
# create it. # create it.
live_item_records.each do |record_data| live_item_records.each do |record_data|
# If we don't recognize this item ID in our database already, skip it. # If we don't recognize this item ID in our database already, skip it.
next unless recognized_item_ids.include?(record_data[:id]) next unless recognized_item_ids.include?(record_data[:id])
record = existing_records_by_item_id.delete(record_data[:id]) || record = existing_records_by_item_id.delete(record_data[:id]) ||
NCMallRecord.new NCMallRecord.new
record.item_id = record_data[:id] record.item_id = record_data[:id]
record.price = record_data[:price] record.price = record_data[:price]
record.discount_price = record_data.dig(:discount, :price) record.discount_price = record_data.dig(:discount, :price)
record.discount_begins_at = record_data.dig(:discount, :begins_at) record.discount_begins_at = record_data.dig(:discount, :begins_at)
record.discount_ends_at = record_data.dig(:discount, :ends_at) record.discount_ends_at = record_data.dig(:discount, :ends_at)
if !record.changed? if !record.changed?
Rails.logger.info "Skipping record for item #{record_data[:name]} " + Rails.logger.info "Skipping record for item #{record_data[:name]} " +
"(unchanged)" "(unchanged)"
next next
end
if record.save
if record.previously_new_record?
Rails.logger.info "Created record for item #{record_data[:name]}"
else
Rails.logger.info "Updated record for item #{record_data[:name]}"
end end
else
Rails.logger.error "Failed to save record for item " +
"#{record_data[:name]}: " +
"#{record.errors.full_messages.join("; ")}: " +
"#{record.inspect}"
end
end
# For each existing record remaining in the existing records hash, this if record.save
# means there was no live record corresponding to it during this sync. if record.previously_new_record?
# Delete it! Rails.logger.info "Created record for item #{record_data[:name]}"
existing_records_by_item_id.values.each do |record| else
item_name = record.item&.name || "<item not found>" Rails.logger.info "Updated record for item #{record_data[:name]}"
if record.destroy end
Rails.logger.info "Destroyed record #{record.id} for item " + else
"#{item_name}" Rails.logger.error "Failed to save record for item " +
else "#{record_data[:name]}: " +
Rails.logger.error "Failed to destroy record #{record.id} for " + "#{record.errors.full_messages.join("; ")}: " +
"item #{item_name}: #{record.inspect}" "#{record.inspect}"
end
end end
# For each existing record remaining in the existing records hash, this
# means there was no live record corresponding to it during this sync.
# Delete it!
existing_records_by_item_id.values.each do |record|
item_name = record.item&.name || "<item not found>"
if record.destroy
Rails.logger.info "Destroyed record #{record.id} for item " +
"#{item_name}"
else
Rails.logger.error "Failed to destroy record #{record.id} for " +
"item #{item_name}: #{record.inspect}"
end
end
rescue => e
Rails.logger.error "Failed to import NC Mall data: #{e.message}"
Rails.logger.error e.backtrace.join("\n")
Sentry.capture_exception(e, tags: { task: "neopets:import:nc_mall" })
raise
end end
end end
end end

126
lib/tasks/neopets/import/rainbow_pool.rake
View file

@ -3,71 +3,81 @@ require "addressable/template"
namespace "neopets:import" do namespace "neopets:import" do
desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes" desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes"
task :rainbow_pool => ["neopets:import:neologin", :environment] do task :rainbow_pool => ["neopets:import:neologin", :environment] do
puts "Importing from Rainbow Pool…" begin
puts "Importing from Rainbow Pool…"
all_species = Species.order(:name).to_a all_species = Species.order(:name).to_a
all_pet_types = PetType.all.to_a all_pet_types = PetType.all.to_a
all_pet_types_by_species_id_and_color_id = all_pet_types. all_pet_types_by_species_id_and_color_id = all_pet_types.
to_h { |pt| [[pt.species_id, pt.color_id], pt] } to_h { |pt| [[pt.species_id, pt.color_id], pt] }
all_colors_by_name = Color.all.to_h { |c| [c.human_name.downcase, c] } all_colors_by_name = Color.all.to_h { |c| [c.human_name.downcase, c] }
hashes_by_color_name_by_species_id = {} hashes_by_color_name_by_species_id = {}
DTIRequests.load_many(max_at_once: 10) do |task| DTIRequests.load_many(max_at_once: 10) do |task|
num_loaded = 0 num_loaded = 0
num_total = all_species.size num_total = all_species.size
print "0/#{num_total} species loaded" print "0/#{num_total} species loaded"
all_species.each do |species|
task.async do
begin
hashes_by_color_name_by_species_id[species.id] =
RainbowPool.load_hashes_for_species(species.id, Neologin.cookie)
rescue => error
puts "Failed to load #{species.name} page, skipping: #{error.message}"
Sentry.capture_exception(error,
tags: { task: "neopets:import:rainbow_pool" },
contexts: { species: { name: species.name, id: species.id } })
end
num_loaded += 1
print "\r#{num_loaded}/#{num_total} species loaded"
end
end
end
all_species.each do |species| all_species.each do |species|
task.async do hashes_by_color_name = hashes_by_color_name_by_species_id[species.id]
begin next if hashes_by_color_name.nil?
hashes_by_color_name_by_species_id[species.id] =
RainbowPool.load_hashes_for_species(species.id, Neologin.cookie) changed_pet_types = []
rescue => error
puts "Failed to load #{species.name} page, skipping: #{error.message}" hashes_by_color_name.each do |color_name, image_hash|
color = all_colors_by_name[color_name.downcase]
if color.nil?
puts "Skipping unrecognized color name: #{color_name}"
next
end
pet_type = all_pet_types_by_species_id_and_color_id[
[species.id, color.id]]
if pet_type.nil?
puts "Skipping unrecognized pet type: " +
"#{color_name} #{species.human_name}"
next
end
if pet_type.basic_image_hash.nil?
puts "Found new image hash: #{image_hash} (#{pet_type.human_name})"
pet_type.basic_image_hash = image_hash
changed_pet_types << pet_type
elsif pet_type.basic_image_hash != image_hash
puts "Updating image hash: #{image_hash} ({#{pet_type.human_name})"
pet_type.basic_image_hash = image_hash
changed_pet_types << pet_type
else
# No need to do anything with image hashes that match!
end end
num_loaded += 1
print "\r#{num_loaded}/#{num_total} species loaded"
end end
PetType.transaction { changed_pet_types.each(&:save!) }
puts "Saved #{changed_pet_types.size} image hashes for " +
"#{species.human_name}"
end end
end rescue => e
puts "Failed to import Rainbow Pool data: #{e.message}"
all_species.each do |species| puts e.backtrace.join("\n")
hashes_by_color_name = hashes_by_color_name_by_species_id[species.id] Sentry.capture_exception(e, tags: { task: "neopets:import:rainbow_pool" })
next if hashes_by_color_name.nil? raise
changed_pet_types = []
hashes_by_color_name.each do |color_name, image_hash|
color = all_colors_by_name[color_name.downcase]
if color.nil?
puts "Skipping unrecognized color name: #{color_name}"
next
end
pet_type = all_pet_types_by_species_id_and_color_id[
[species.id, color.id]]
if pet_type.nil?
puts "Skipping unrecognized pet type: " +
"#{color_name} #{species.human_name}"
next
end
if pet_type.basic_image_hash.nil?
puts "Found new image hash: #{image_hash} (#{pet_type.human_name})"
pet_type.basic_image_hash = image_hash
changed_pet_types << pet_type
elsif pet_type.basic_image_hash != image_hash
puts "Updating image hash: #{image_hash} ({#{pet_type.human_name})"
pet_type.basic_image_hash = image_hash
changed_pet_types << pet_type
else
# No need to do anything with image hashes that match!
end
end
PetType.transaction { changed_pet_types.each(&:save!) }
puts "Saved #{changed_pet_types.size} image hashes for " +
"#{species.human_name}"
end end
end end
end end

168
lib/tasks/neopets/import/styling_studio.rake
View file

@ -1,99 +1,109 @@
namespace "neopets:import" do namespace "neopets:import" do
desc "Import alt style info from the NC Styling Studio" desc "Import alt style info from the NC Styling Studio"
task :styling_studio => ["neopets:import:neologin", :environment] do task :styling_studio => ["neopets:import:neologin", :environment] do
puts "Importing from Styling Studio…" begin
puts "Importing from Styling Studio…"
all_species = Species.order(:name).to_a all_species = Species.order(:name).to_a
# Load 10 species pages from the NC Mall at a time. # Load 10 species pages from the NC Mall at a time.
styles_by_species_id = {} styles_by_species_id = {}
DTIRequests.load_many(max_at_once: 10) do |task| DTIRequests.load_many(max_at_once: 10) do |task|
num_loaded = 0 num_loaded = 0
num_total = all_species.size num_total = all_species.size
print "0/#{num_total} species loaded" print "0/#{num_total} species loaded"
all_species.each do |species|
task.async {
begin
styles_by_species_id[species.id] = Neopets::NCMall.load_styles(
species_id: species.id,
neologin: Neologin.cookie,
)
rescue => error
puts "\n⚠️ Error loading for #{species.human_name}, skipping: #{error.message}"
Sentry.capture_exception(error,
tags: { task: "neopets:import:styling_studio" },
contexts: { species: { name: species.human_name, id: species.id } })
end
num_loaded += 1
print "\r#{num_loaded}/#{num_total} species loaded"
}
end
end
print "\n"
style_ids = styles_by_species_id.values.flatten(1).map { |s| s[:oii] }
style_records_by_id =
AltStyle.where(id: style_ids).to_h { |as| [as.id, as] }
all_species.each do |species| all_species.each do |species|
task.async { styles = styles_by_species_id[species.id]
begin next if styles.nil?
styles_by_species_id[species.id] = Neopets::NCMall.load_styles(
species_id: species.id, counts = {changed: 0, unchanged: 0, skipped: 0}
neologin: Neologin.cookie, styles.each do |style|
) record = style_records_by_id[style[:oii]]
rescue => error label = "#{style[:name]} (#{style[:oii]})"
puts "\n⚠️ Error loading for #{species.human_name}, skipping: #{error.message}" if record.nil?
puts "❔ [#{label}]: Not modeled yet, skipping"
counts[:skipped] += 1
next
end end
num_loaded += 1
print "\r#{num_loaded}/#{num_total} species loaded"
}
end
end
print "\n"
style_ids = styles_by_species_id.values.flatten(1).map { |s| s[:oii] } if !record.real_full_name?
style_records_by_id = record.full_name = style[:name]
AltStyle.where(id: style_ids).to_h { |as| [as.id, as] } puts "✅ [#{label}]: Full name is now #{style[:name].inspect}"
elsif record.full_name != style[:name]
puts "⚠️ [#{label}: Full name may have changed, handle manually? " +
"#{record.full_name.inspect} -> #{style[:name].inspect}"
end
all_species.each do |species| if !record.real_thumbnail_url?
styles = styles_by_species_id[species.id] record.thumbnail_url = style[:image]
next if styles.nil? puts "✅ [#{label}]: Thumbnail URL is now #{style[:image].inspect}"
elsif record.thumbnail_url != style[:image]
puts "⚠️ [#{label}: Thumbnail URL may have changed, handle manually? " +
"#{record.thumbnail_url.inspect} -> #{style[:image].inspect}"
end
counts = {changed: 0, unchanged: 0, skipped: 0} if style[:name].end_with?(record.pet_name)
styles.each do |style| new_series_name = style[:name].split(record.pet_name).first.strip
record = style_records_by_id[style[:oii]] if !record.real_series_name?
label = "#{style[:name]} (#{style[:oii]})"
if record.nil?
puts "❔ [#{label}]: Not modeled yet, skipping"
counts[:skipped] += 1
next
end
if !record.real_full_name?
record.full_name = style[:name]
puts "✅ [#{label}]: Full name is now #{style[:name].inspect}"
elsif record.full_name != style[:name]
puts "⚠️ [#{label}: Full name may have changed, handle manually? " +
"#{record.full_name.inspect} -> #{style[:name].inspect}"
end
if !record.real_thumbnail_url?
record.thumbnail_url = style[:image]
puts "✅ [#{label}]: Thumbnail URL is now #{style[:image].inspect}"
elsif record.thumbnail_url != style[:image]
puts "⚠️ [#{label}: Thumbnail URL may have changed, handle manually? " +
"#{record.thumbnail_url.inspect} -> #{style[:image].inspect}"
end
if style[:name].end_with?(record.pet_name)
new_series_name = style[:name].split(record.pet_name).first.strip
if !record.real_series_name?
record.series_name = new_series_name
puts "✅ [#{label}]: Series name is now #{new_series_name.inspect}"
elsif record.series_name != new_series_name
if ENV['FORCE'] == '1'
puts "❗ [#{label}]: Series name forcibly changed: " +
"#{record.series_name.inspect} -> #{new_series_name.inspect}"
record.series_name = new_series_name record.series_name = new_series_name
else puts "✅ [#{label}]: Series name is now #{new_series_name.inspect}"
puts "⚠️ [#{label}]: Series name may have changed, handle manually? " + elsif record.series_name != new_series_name
"#{record.series_name.inspect} -> #{new_series_name.inspect}" if ENV['FORCE'] == '1'
puts "❗ [#{label}]: Series name forcibly changed: " +
"#{record.series_name.inspect} -> #{new_series_name.inspect}"
record.series_name = new_series_name
else
puts "⚠️ [#{label}]: Series name may have changed, handle manually? " +
"#{record.series_name.inspect} -> #{new_series_name.inspect}"
end
end end
else
puts "⚠️ [#{label}]: Unable to detect series name, handle manually? " +
"#{record.pet_name.inspect} <-> #{style[:name].inspect}"
end end
else
puts "⚠️ [#{label}]: Unable to detect series name, handle manually? " + if record.changed?
"#{record.pet_name.inspect} <-> #{style[:name].inspect}" counts[:changed] += 1
else
counts[:unchanged] += 1
end
record.save!
end end
if record.changed? puts "#{species.human_name}: #{counts[:changed]} changed, " +
counts[:changed] += 1 "#{counts[:unchanged]} unchanged, #{counts[:skipped]} skipped"
else
counts[:unchanged] += 1
end
record.save!
end end
rescue => e
puts "#{species.human_name}: #{counts[:changed]} changed, " + puts "Failed to import Styling Studio data: #{e.message}"
"#{counts[:unchanged]} unchanged, #{counts[:skipped]} skipped" puts e.backtrace.join("\n")
Sentry.capture_exception(e, tags: { task: "neopets:import:styling_studio" })
raise
end end
end end
end end