From 2b8fe68387982aff0fa60bb5a6cf60f4db209a24 Mon Sep 17 00:00:00 2001 From: Emi Matchu Date: Mon, 16 Dec 2024 14:12:38 -0800 Subject: [PATCH] Refactor `rails neopets:import:rainbow_pool` to run in parallel We're using our new helpers to make it easier, yay! --- lib/tasks/neopets/import/rainbow_pool.rake | 32 ++++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/lib/tasks/neopets/import/rainbow_pool.rake b/lib/tasks/neopets/import/rainbow_pool.rake index 8e66592d..ae8d5de8 100644 --- a/lib/tasks/neopets/import/rainbow_pool.rake +++ b/lib/tasks/neopets/import/rainbow_pool.rake @@ -5,21 +5,35 @@ namespace "neopets:import" do task :rainbow_pool => ["neopets:import:neologin", :environment] do puts "Importing from Rainbow Pool…" + all_species = Species.order(:name).to_a all_pet_types = PetType.all.to_a all_pet_types_by_species_id_and_color_id = all_pet_types. to_h { |pt| [[pt.species_id, pt.color_id], pt] } all_colors_by_name = Color.all.to_h { |c| [c.human_name.downcase, c] } - # TODO: Do these in parallel? I set up the HTTP requests to be able to - # handle it, and just didn't set up the rest of the code for it, lol - Species.order(:name).each do |species| - begin - hashes_by_color_name = RainbowPool.load_hashes_for_species( - species.id, Neologin.cookie) - rescue => error - puts "Failed to load #{species.name} page, skipping: #{error.message}" - next + hashes_by_color_name_by_species_id = {} + DTIRequests.load_many(max_at_once: 10) do |task| + num_loaded = 0 + num_total = all_species.size + print "0/#{num_total} species loaded" + + all_species.each do |species| + task.async do + begin + hashes_by_color_name_by_species_id[species.id] = + RainbowPool.load_hashes_for_species(species.id, Neologin.cookie) + rescue => error + puts "Failed to load #{species.name} page, skipping: #{error.message}" + end + num_loaded += 1 + print "\r#{num_loaded}/#{num_total} species loaded" + end end + end + + all_species.each do |species| + hashes_by_color_name = hashes_by_color_name_by_species_id[species.id] + next if hashes_by_color_name.nil? changed_pet_types = []