diff --git a/deploy/setup.yml b/deploy/setup.yml index 26d872b3..d6b59878 100644 --- a/deploy/setup.yml +++ b/deploy/setup.yml @@ -449,12 +449,12 @@ minute: "*/10" job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails nc_mall:sync'" - - name: Create 10min cron job to run `rails neopets:import:nc_mall` + - name: Create 10min cron job to run `rails neopets:import` become_user: impress cron: - name: "Impress: import NC Mall data" + name: "Impress: import Neopets data" minute: "*/10" - job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails neopets:import:nc_mall'" + job: "bash -c 'source /etc/profile && source ~/.bash_profile && cd /srv/impress/current && bin/rails neopets:import'" - name: Create weekly cron job to run `rails public_data:commit` become_user: impress diff --git a/lib/tasks/neopets/import.rake b/lib/tasks/neopets/import.rake index e4fd5efd..3196fbcc 100644 --- a/lib/tasks/neopets/import.rake +++ b/lib/tasks/neopets/import.rake @@ -22,9 +22,26 @@ namespace :neopets do ] namespace :import do + # Gets the neologin cookie, either from ENV['NEOLOGIN_COOKIE'] or by prompting. + # The neologin cookie is required for authenticated Neopets requests (Rainbow Pool, + # Styling Studio). It's generally long-lived (~1 year), so it can be stored in the + # environment and rotated manually when it expires. + # + # To extract the cookie: + # 1. Log into Neopets.com in your browser + # 2. Open browser DevTools > Application/Storage > Cookies + # 3. Find the "neologin" cookie value + # 4. Set NEOLOGIN_COOKIE environment variable to that value + # 5. Update production.env and redeploy when the cookie expires task :neologin do unless Neologin.cookie? - Neologin.cookie = STDIN.getpass("Neologin cookie: ") + # Try environment variable first (for automated cron jobs) + if ENV['NEOLOGIN_COOKIE'].present? + Neologin.cookie = ENV['NEOLOGIN_COOKIE'] + else + # Fall back to interactive prompt (for local development) + Neologin.cookie = STDIN.getpass("Neologin cookie: ") + end end end end diff --git a/lib/tasks/neopets/import/nc_mall.rake b/lib/tasks/neopets/import/nc_mall.rake index 08d9f207..be34ef52 100644 --- a/lib/tasks/neopets/import/nc_mall.rake +++ b/lib/tasks/neopets/import/nc_mall.rake @@ -1,76 +1,83 @@ namespace "neopets:import" do desc "Sync our NCMallRecord table with the live NC Mall" task :nc_mall => :environment do - # Log to STDOUT. - Rails.logger = Logger.new(STDOUT) + begin + # Log to STDOUT. + Rails.logger = Logger.new(STDOUT) - puts "Importing from NC Mall…" + puts "Importing from NC Mall…" - # First, load all records of what's being sold in the live NC Mall. We load - # all categories from the menu and fetch all items from each. (We also - # de-duplicate the items, which is important because the same item can - # appear in multiple categories!) - live_item_records = load_all_nc_mall_items.uniq { |item| item[:id] } + # First, load all records of what's being sold in the live NC Mall. We load + # all categories from the menu and fetch all items from each. (We also + # de-duplicate the items, which is important because the same item can + # appear in multiple categories!) + live_item_records = load_all_nc_mall_items.uniq { |item| item[:id] } - # Then, get the existing NC Mall records in our database. (We include the - # items, to be able to output the item name during logging.) - existing_records = NCMallRecord.includes(:item).all - existing_records_by_item_id = existing_records.to_h { |r| [r.item_id, r] } + # Then, get the existing NC Mall records in our database. (We include the + # items, to be able to output the item name during logging.) + existing_records = NCMallRecord.includes(:item).all + existing_records_by_item_id = existing_records.to_h { |r| [r.item_id, r] } - # Additionally, check which of the item IDs in the live records are items - # we've seen before. (We'll skip records for items we don't know.) - live_item_ids = live_item_records.map { |r| r[:id] } - recognized_item_ids = Item.where(id: live_item_ids).pluck(:id).to_set - Rails.logger.debug "We found #{live_item_records.size} items, and we " + - "recognize #{recognized_item_ids.size} of them." + # Additionally, check which of the item IDs in the live records are items + # we've seen before. (We'll skip records for items we don't know.) + live_item_ids = live_item_records.map { |r| r[:id] } + recognized_item_ids = Item.where(id: live_item_ids).pluck(:id).to_set + Rails.logger.debug "We found #{live_item_records.size} items, and we " + + "recognize #{recognized_item_ids.size} of them." - # For each record in the live NC Mall, check if there's an existing record. - # If so, update it, and remove it from the existing records hash. If not, - # create it. - live_item_records.each do |record_data| - # If we don't recognize this item ID in our database already, skip it. - next unless recognized_item_ids.include?(record_data[:id]) + # For each record in the live NC Mall, check if there's an existing record. + # If so, update it, and remove it from the existing records hash. If not, + # create it. + live_item_records.each do |record_data| + # If we don't recognize this item ID in our database already, skip it. + next unless recognized_item_ids.include?(record_data[:id]) - record = existing_records_by_item_id.delete(record_data[:id]) || - NCMallRecord.new - record.item_id = record_data[:id] - record.price = record_data[:price] - record.discount_price = record_data.dig(:discount, :price) - record.discount_begins_at = record_data.dig(:discount, :begins_at) - record.discount_ends_at = record_data.dig(:discount, :ends_at) + record = existing_records_by_item_id.delete(record_data[:id]) || + NCMallRecord.new + record.item_id = record_data[:id] + record.price = record_data[:price] + record.discount_price = record_data.dig(:discount, :price) + record.discount_begins_at = record_data.dig(:discount, :begins_at) + record.discount_ends_at = record_data.dig(:discount, :ends_at) - if !record.changed? - Rails.logger.info "Skipping record for item #{record_data[:name]} " + - "(unchanged)" - next - end - - if record.save - if record.previously_new_record? - Rails.logger.info "Created record for item #{record_data[:name]}" - else - Rails.logger.info "Updated record for item #{record_data[:name]}" + if !record.changed? + Rails.logger.info "Skipping record for item #{record_data[:name]} " + + "(unchanged)" + next end - else - Rails.logger.error "Failed to save record for item " + - "#{record_data[:name]}: " + - "#{record.errors.full_messages.join("; ")}: " + - "#{record.inspect}" - end - end - # For each existing record remaining in the existing records hash, this - # means there was no live record corresponding to it during this sync. - # Delete it! - existing_records_by_item_id.values.each do |record| - item_name = record.item&.name || "" - if record.destroy - Rails.logger.info "Destroyed record #{record.id} for item " + - "#{item_name}" - else - Rails.logger.error "Failed to destroy record #{record.id} for " + - "item #{item_name}: #{record.inspect}" + if record.save + if record.previously_new_record? + Rails.logger.info "Created record for item #{record_data[:name]}" + else + Rails.logger.info "Updated record for item #{record_data[:name]}" + end + else + Rails.logger.error "Failed to save record for item " + + "#{record_data[:name]}: " + + "#{record.errors.full_messages.join("; ")}: " + + "#{record.inspect}" + end end + + # For each existing record remaining in the existing records hash, this + # means there was no live record corresponding to it during this sync. + # Delete it! + existing_records_by_item_id.values.each do |record| + item_name = record.item&.name || "" + if record.destroy + Rails.logger.info "Destroyed record #{record.id} for item " + + "#{item_name}" + else + Rails.logger.error "Failed to destroy record #{record.id} for " + + "item #{item_name}: #{record.inspect}" + end + end + rescue => e + Rails.logger.error "Failed to import NC Mall data: #{e.message}" + Rails.logger.error e.backtrace.join("\n") + Sentry.capture_exception(e, tags: { task: "neopets:import:nc_mall" }) + raise end end end diff --git a/lib/tasks/neopets/import/rainbow_pool.rake b/lib/tasks/neopets/import/rainbow_pool.rake index ae8d5de8..a5d48060 100644 --- a/lib/tasks/neopets/import/rainbow_pool.rake +++ b/lib/tasks/neopets/import/rainbow_pool.rake @@ -3,71 +3,81 @@ require "addressable/template" namespace "neopets:import" do desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes" task :rainbow_pool => ["neopets:import:neologin", :environment] do - puts "Importing from Rainbow Pool…" + begin + puts "Importing from Rainbow Pool…" - all_species = Species.order(:name).to_a - all_pet_types = PetType.all.to_a - all_pet_types_by_species_id_and_color_id = all_pet_types. - to_h { |pt| [[pt.species_id, pt.color_id], pt] } - all_colors_by_name = Color.all.to_h { |c| [c.human_name.downcase, c] } + all_species = Species.order(:name).to_a + all_pet_types = PetType.all.to_a + all_pet_types_by_species_id_and_color_id = all_pet_types. + to_h { |pt| [[pt.species_id, pt.color_id], pt] } + all_colors_by_name = Color.all.to_h { |c| [c.human_name.downcase, c] } - hashes_by_color_name_by_species_id = {} - DTIRequests.load_many(max_at_once: 10) do |task| - num_loaded = 0 - num_total = all_species.size - print "0/#{num_total} species loaded" + hashes_by_color_name_by_species_id = {} + DTIRequests.load_many(max_at_once: 10) do |task| + num_loaded = 0 + num_total = all_species.size + print "0/#{num_total} species loaded" + + all_species.each do |species| + task.async do + begin + hashes_by_color_name_by_species_id[species.id] = + RainbowPool.load_hashes_for_species(species.id, Neologin.cookie) + rescue => error + puts "Failed to load #{species.name} page, skipping: #{error.message}" + Sentry.capture_exception(error, + tags: { task: "neopets:import:rainbow_pool" }, + contexts: { species: { name: species.name, id: species.id } }) + end + num_loaded += 1 + print "\r#{num_loaded}/#{num_total} species loaded" + end + end + end all_species.each do |species| - task.async do - begin - hashes_by_color_name_by_species_id[species.id] = - RainbowPool.load_hashes_for_species(species.id, Neologin.cookie) - rescue => error - puts "Failed to load #{species.name} page, skipping: #{error.message}" + hashes_by_color_name = hashes_by_color_name_by_species_id[species.id] + next if hashes_by_color_name.nil? + + changed_pet_types = [] + + hashes_by_color_name.each do |color_name, image_hash| + color = all_colors_by_name[color_name.downcase] + if color.nil? + puts "Skipping unrecognized color name: #{color_name}" + next + end + + pet_type = all_pet_types_by_species_id_and_color_id[ + [species.id, color.id]] + if pet_type.nil? + puts "Skipping unrecognized pet type: " + + "#{color_name} #{species.human_name}" + next + end + + if pet_type.basic_image_hash.nil? + puts "Found new image hash: #{image_hash} (#{pet_type.human_name})" + pet_type.basic_image_hash = image_hash + changed_pet_types << pet_type + elsif pet_type.basic_image_hash != image_hash + puts "Updating image hash: #{image_hash} ({#{pet_type.human_name})" + pet_type.basic_image_hash = image_hash + changed_pet_types << pet_type + else + # No need to do anything with image hashes that match! end - num_loaded += 1 - print "\r#{num_loaded}/#{num_total} species loaded" end + + PetType.transaction { changed_pet_types.each(&:save!) } + puts "Saved #{changed_pet_types.size} image hashes for " + + "#{species.human_name}" end - end - - all_species.each do |species| - hashes_by_color_name = hashes_by_color_name_by_species_id[species.id] - next if hashes_by_color_name.nil? - - changed_pet_types = [] - - hashes_by_color_name.each do |color_name, image_hash| - color = all_colors_by_name[color_name.downcase] - if color.nil? - puts "Skipping unrecognized color name: #{color_name}" - next - end - - pet_type = all_pet_types_by_species_id_and_color_id[ - [species.id, color.id]] - if pet_type.nil? - puts "Skipping unrecognized pet type: " + - "#{color_name} #{species.human_name}" - next - end - - if pet_type.basic_image_hash.nil? - puts "Found new image hash: #{image_hash} (#{pet_type.human_name})" - pet_type.basic_image_hash = image_hash - changed_pet_types << pet_type - elsif pet_type.basic_image_hash != image_hash - puts "Updating image hash: #{image_hash} ({#{pet_type.human_name})" - pet_type.basic_image_hash = image_hash - changed_pet_types << pet_type - else - # No need to do anything with image hashes that match! - end - end - - PetType.transaction { changed_pet_types.each(&:save!) } - puts "Saved #{changed_pet_types.size} image hashes for " + - "#{species.human_name}" + rescue => e + puts "Failed to import Rainbow Pool data: #{e.message}" + puts e.backtrace.join("\n") + Sentry.capture_exception(e, tags: { task: "neopets:import:rainbow_pool" }) + raise end end end diff --git a/lib/tasks/neopets/import/styling_studio.rake b/lib/tasks/neopets/import/styling_studio.rake index 322eaa72..cd81319b 100644 --- a/lib/tasks/neopets/import/styling_studio.rake +++ b/lib/tasks/neopets/import/styling_studio.rake @@ -1,99 +1,109 @@ namespace "neopets:import" do desc "Import alt style info from the NC Styling Studio" task :styling_studio => ["neopets:import:neologin", :environment] do - puts "Importing from Styling Studio…" + begin + puts "Importing from Styling Studio…" - all_species = Species.order(:name).to_a + all_species = Species.order(:name).to_a - # Load 10 species pages from the NC Mall at a time. - styles_by_species_id = {} - DTIRequests.load_many(max_at_once: 10) do |task| - num_loaded = 0 - num_total = all_species.size - print "0/#{num_total} species loaded" + # Load 10 species pages from the NC Mall at a time. + styles_by_species_id = {} + DTIRequests.load_many(max_at_once: 10) do |task| + num_loaded = 0 + num_total = all_species.size + print "0/#{num_total} species loaded" + + all_species.each do |species| + task.async { + begin + styles_by_species_id[species.id] = Neopets::NCMall.load_styles( + species_id: species.id, + neologin: Neologin.cookie, + ) + rescue => error + puts "\n⚠️ Error loading for #{species.human_name}, skipping: #{error.message}" + Sentry.capture_exception(error, + tags: { task: "neopets:import:styling_studio" }, + contexts: { species: { name: species.human_name, id: species.id } }) + end + num_loaded += 1 + print "\r#{num_loaded}/#{num_total} species loaded" + } + end + end + print "\n" + + style_ids = styles_by_species_id.values.flatten(1).map { |s| s[:oii] } + style_records_by_id = + AltStyle.where(id: style_ids).to_h { |as| [as.id, as] } all_species.each do |species| - task.async { - begin - styles_by_species_id[species.id] = Neopets::NCMall.load_styles( - species_id: species.id, - neologin: Neologin.cookie, - ) - rescue => error - puts "\n⚠️ Error loading for #{species.human_name}, skipping: #{error.message}" + styles = styles_by_species_id[species.id] + next if styles.nil? + + counts = {changed: 0, unchanged: 0, skipped: 0} + styles.each do |style| + record = style_records_by_id[style[:oii]] + label = "#{style[:name]} (#{style[:oii]})" + if record.nil? + puts "❔ [#{label}]: Not modeled yet, skipping" + counts[:skipped] += 1 + next end - num_loaded += 1 - print "\r#{num_loaded}/#{num_total} species loaded" - } - end - end - print "\n" - style_ids = styles_by_species_id.values.flatten(1).map { |s| s[:oii] } - style_records_by_id = - AltStyle.where(id: style_ids).to_h { |as| [as.id, as] } + if !record.real_full_name? + record.full_name = style[:name] + puts "✅ [#{label}]: Full name is now #{style[:name].inspect}" + elsif record.full_name != style[:name] + puts "⚠️ [#{label}: Full name may have changed, handle manually? " + + "#{record.full_name.inspect} -> #{style[:name].inspect}" + end - all_species.each do |species| - styles = styles_by_species_id[species.id] - next if styles.nil? + if !record.real_thumbnail_url? + record.thumbnail_url = style[:image] + puts "✅ [#{label}]: Thumbnail URL is now #{style[:image].inspect}" + elsif record.thumbnail_url != style[:image] + puts "⚠️ [#{label}: Thumbnail URL may have changed, handle manually? " + + "#{record.thumbnail_url.inspect} -> #{style[:image].inspect}" + end - counts = {changed: 0, unchanged: 0, skipped: 0} - styles.each do |style| - record = style_records_by_id[style[:oii]] - label = "#{style[:name]} (#{style[:oii]})" - if record.nil? - puts "❔ [#{label}]: Not modeled yet, skipping" - counts[:skipped] += 1 - next - end - - if !record.real_full_name? - record.full_name = style[:name] - puts "✅ [#{label}]: Full name is now #{style[:name].inspect}" - elsif record.full_name != style[:name] - puts "⚠️ [#{label}: Full name may have changed, handle manually? " + - "#{record.full_name.inspect} -> #{style[:name].inspect}" - end - - if !record.real_thumbnail_url? - record.thumbnail_url = style[:image] - puts "✅ [#{label}]: Thumbnail URL is now #{style[:image].inspect}" - elsif record.thumbnail_url != style[:image] - puts "⚠️ [#{label}: Thumbnail URL may have changed, handle manually? " + - "#{record.thumbnail_url.inspect} -> #{style[:image].inspect}" - end - - if style[:name].end_with?(record.pet_name) - new_series_name = style[:name].split(record.pet_name).first.strip - if !record.real_series_name? - record.series_name = new_series_name - puts "✅ [#{label}]: Series name is now #{new_series_name.inspect}" - elsif record.series_name != new_series_name - if ENV['FORCE'] == '1' - puts "❗ [#{label}]: Series name forcibly changed: " + - "#{record.series_name.inspect} -> #{new_series_name.inspect}" + if style[:name].end_with?(record.pet_name) + new_series_name = style[:name].split(record.pet_name).first.strip + if !record.real_series_name? record.series_name = new_series_name - else - puts "⚠️ [#{label}]: Series name may have changed, handle manually? " + - "#{record.series_name.inspect} -> #{new_series_name.inspect}" + puts "✅ [#{label}]: Series name is now #{new_series_name.inspect}" + elsif record.series_name != new_series_name + if ENV['FORCE'] == '1' + puts "❗ [#{label}]: Series name forcibly changed: " + + "#{record.series_name.inspect} -> #{new_series_name.inspect}" + record.series_name = new_series_name + else + puts "⚠️ [#{label}]: Series name may have changed, handle manually? " + + "#{record.series_name.inspect} -> #{new_series_name.inspect}" + end end + else + puts "⚠️ [#{label}]: Unable to detect series name, handle manually? " + + "#{record.pet_name.inspect} <-> #{style[:name].inspect}" end - else - puts "⚠️ [#{label}]: Unable to detect series name, handle manually? " + - "#{record.pet_name.inspect} <-> #{style[:name].inspect}" + + if record.changed? + counts[:changed] += 1 + else + counts[:unchanged] += 1 + end + + record.save! end - if record.changed? - counts[:changed] += 1 - else - counts[:unchanged] += 1 - end - - record.save! + puts "#{species.human_name}: #{counts[:changed]} changed, " + + "#{counts[:unchanged]} unchanged, #{counts[:skipped]} skipped" end - - puts "#{species.human_name}: #{counts[:changed]} changed, " + - "#{counts[:unchanged]} unchanged, #{counts[:skipped]} skipped" + rescue => e + puts "Failed to import Styling Studio data: #{e.message}" + puts e.backtrace.join("\n") + Sentry.capture_exception(e, tags: { task: "neopets:import:styling_studio" }) + raise end end end