diff --git a/app/services/neopets/nc_mall.rb b/app/services/neopets/nc_mall.rb index 5f9cbec8..3916e683 100644 --- a/app/services/neopets/nc_mall.rb +++ b/app/services/neopets/nc_mall.rb @@ -31,7 +31,7 @@ module Neopets::NCMall ]) do |response| if response.status != 200 raise ResponseNotOK.new(response.status), - "expected status 200 but got #{response.status} (#{url})" + "expected status 200 but got #{response.status} (#{ROOT_DOCUMENT_URL})" end response.read diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index 33e25628..066ab74e 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -16,13 +16,15 @@ # end ActiveSupport::Inflector.inflections(:en) do |inflect| - # Teach Zeitwerk that `RocketAMF` is what to expect in `lib/rocketamf`. + # `lib/rocketamf` => `RocketAMF` inflect.acronym "RocketAMF" - # Teach Zeitwerk that `NeoPass` is what to expect in `neopass.rb`. + # `neopass.rb` => `NeoPass` inflect.acronym "NeoPass" - # Teach Zeitwerk that "NCMall" is what to expect in `nc_mall.rb`. - # (We do this by teaching it the word "NC".) + # `nc_mall.rb` => `NCMall` inflect.acronym "NC" + + # `dti_requests.rb` => `DTIRequests` + inflect.acronym "DTI" end diff --git a/lib/dti_requests.rb b/lib/dti_requests.rb new file mode 100644 index 00000000..17347f0b --- /dev/null +++ b/lib/dti_requests.rb @@ -0,0 +1,19 @@ +require "async" +require "async/barrier" + +module DTIRequests + class << self + def load_many(max_at_once: 10) + barrier = Async::Barrier.new + semaphore = Async::Semaphore.new(max_at_once, parent: barrier) + + Sync do + block_return_value = yield semaphore + barrier.wait # Load all the subtasks. + block_return_value + ensure + barrier.stop # If any subtasks failed, cancel the rest. + end + end + end +end diff --git a/lib/tasks/neopets/import/nc_mall.rake b/lib/tasks/neopets/import/nc_mall.rake index fb110714..60c0fb09 100644 --- a/lib/tasks/neopets/import/nc_mall.rake +++ b/lib/tasks/neopets/import/nc_mall.rake @@ -85,15 +85,10 @@ def load_all_nc_mall_pages links = Neopets::NCMall.load_page_links # Next, load the linked pages, 10 at a time. - barrier = Async::Barrier.new - semaphore = Async::Semaphore.new(10, parent: barrier) - begin - linked_page_tasks = links.map do |link| - semaphore.async { Neopets::NCMall.load_page link[:type], link[:cat] } + linked_page_tasks = DTIRequests.load_many(max_at_once: 10) do |task| + links.map do |link| + task.async { Neopets::NCMall.load_page link[:type], link[:cat] } end - barrier.wait # Load all the pages. - ensure - barrier.stop # If any pages failed, cancel the rest. end # Finally, return all the pages: the homepage, and the linked pages. diff --git a/lib/tasks/neopets/import/styling_studio.rake b/lib/tasks/neopets/import/styling_studio.rake index 516f5bf4..3d3277b4 100644 --- a/lib/tasks/neopets/import/styling_studio.rake +++ b/lib/tasks/neopets/import/styling_studio.rake @@ -6,16 +6,14 @@ namespace "neopets:import" do all_species = Species.order(:name).to_a # Load 10 species pages from the NC Mall at a time. - barrier = Async::Barrier.new - semaphore = Async::Semaphore.new(10, parent: barrier) styles_by_species_id = {} - Sync do + DTIRequests.load_many(max_at_once: 10) do |task| num_loaded = 0 num_total = all_species.size print "0/#{num_total} species loaded" all_species.each do |species| - semaphore.async { + task.async { begin styles_by_species_id[species.id] = Neopets::NCMall.load_styles( species_id: species.id, @@ -28,11 +26,6 @@ namespace "neopets:import" do print "\r#{num_loaded}/#{num_total} species loaded" } end - - # Wait until all tasks are done. - barrier.wait - ensure - barrier.stop # If something goes wrong, clean up all tasks. end print "\n"