Create new DTIRequests.{get,post} helpers for HTTP requests

Instead of using `Async::HTTP::Internet` directly, and always applying
our `User-Agent` header manually, let's build a helper for it!

app/services/neopets/nc_mall.rb

@@ -1,11 +1,6 @@
 require "addressable/template"
-require "async/http/internet/instance"
 
 module Neopets::NCMall
-	# Share a pool of persistent connections, rather than reconnecting on
-	# each request. (This library does that automatically!)
-	INTERNET = Async::HTTP::Internet.instance
-
 	# Load the NC Mall home page content area, and return its useful data.
 	HOME_PAGE_URL = "https://ncmall.neopets.com/mall/ajax/home_page.phtml"
 	def self.load_home_page
@@ -26,9 +21,7 @@ module Neopets::NCMall
 	PAGE_LINK_PATTERN = /load_items_pane\(['"](.+?)['"], ([0-9]+)\).+?>(.+?)</
 	def self.load_page_links
 		html = Sync do
-			INTERNET.get(ROOT_DOCUMENT_URL, [
+			DTIRequests.get(ROOT_DOCUMENT_URL) do |response|
-				["User-Agent", Rails.configuration.user_agent_for_neopets],
-			]) do |response|
 				if response.status != 200
 					raise ResponseNotOK.new(response.status),
 						"expected status 200 but got #{response.status} (#{ROOT_DOCUMENT_URL})"
@@ -48,15 +41,14 @@ module Neopets::NCMall
 	STYLING_STUDIO_URL = "https://www.neopets.com/np-templates/ajax/stylingstudio/studio.php"
 	def self.load_styles(species_id:, neologin:)
 		Sync do
-			INTERNET.post(
+			DTIRequests.post(
 				STYLING_STUDIO_URL,
-				headers: [
+				[
-					["User-Agent", Rails.configuration.user_agent_for_neopets],
 					["Content-Type", "application/x-www-form-urlencoded"],
 					["Cookie", "neologin=#{neologin}"],
 					["X-Requested-With", "XMLHttpRequest"],
 				],
-				body: {tab: 1, mode: "getStyles", species: species_id}.to_query,
+				{tab: 1, mode: "getStyles", species: species_id}.to_query,
 			) do |response|
 				if response.status != 200
 					raise ResponseNotOK.new(response.status),
@@ -80,9 +72,7 @@ module Neopets::NCMall
 
 	def self.load_page_by_url(url)
 		Sync do
-			INTERNET.get(url, [
+			DTIRequests.get(url) do |response|
-				["User-Agent", Rails.configuration.user_agent_for_neopets],
-			]) do |response|
 				if response.status != 200
 					raise ResponseNotOK.new(response.status),
 						"expected status 200 but got #{response.status} (#{url})"

app/services/neopets/neopass.rb

@@ -1,12 +1,6 @@
-require "async/http/internet/instance"
-
 # While most of our NeoPass logic is built into Devise -> OmniAuth -> OIDC
 # OmniAuth plugin, NeoPass also offers some supplemental APIs that we use here.
 module Neopets::NeoPass
-	# Share a pool of persistent connections, rather than reconnecting on
-	# each request. (This library does that automatically!)
-	INTERNET = Async::HTTP::Internet.instance
-
 	def self.load_main_neopets_username(access_token)
 		linkages = load_linkages(access_token)
 
@@ -32,10 +26,10 @@ module Neopets::NeoPass
 	LINKAGE_URL = "https://oidc.neopets.com/linkage/all"
 	def self.load_linkages(access_token)
 		linkages_str = Sync do
-			INTERNET.get(LINKAGE_URL, [
+			DTIRequests.get(
-				["User-Agent", Rails.configuration.user_agent_for_neopets],
+				LINKAGE_URL,
-				["Authorization", "Bearer #{access_token}"],
+				[["Authorization", "Bearer #{access_token}"]],
-			]) do |response|
+			) do |response|
 				if response.status != 200
 					raise ResponseNotOK.new(response.status),
 						"expected status 200 but got #{response.status} (#{LINKAGE_URL})"

app/services/neopets_media_archive.rb

@@ -1,5 +1,4 @@
 require "addressable/uri"
-require "async/http/internet/instance"
 require "json"
 
 # The Neopets Media Archive is a service that mirrors images.neopets.com files
@@ -11,10 +10,6 @@ require "json"
 # long-term archive, not dependent on their services having 100% uptime in
 # order for us to operate. We never discard old files, we just keep going!
 module NeopetsMediaArchive
-  # Share a pool of persistent connections, rather than reconnecting on
-  # each request. (This library does that automatically!)
-  INTERNET = Async::HTTP::Internet.instance
-
   ROOT_PATH = Pathname.new(Rails.configuration.neopets_media_archive_root)
 
   # Load the file from the given `images.neopets.com` URI.
@@ -72,9 +67,7 @@ module NeopetsMediaArchive
     # We use this in the `swf_assets:manifests:load` task to perform many
     # requests in parallel!
     Sync do
-      INTERNET.get(uri, [
+      DTIRequests.get(uri) do |response|
-        ["User-Agent", Rails.configuration.user_agent_for_neopets],
-      ]) do |response|
         if response.status != 200
           raise ResponseNotOK.new(response.status),
             "expected status 200 but got #{response.status} (#{uri})"

lib/dti_requests.rb

@@ -1,8 +1,17 @@
 require "async"
 require "async/barrier"
+require "async/http/internet/instance"
 
 module DTIRequests
 	class << self
+		def get(url, headers = [], &block)
+			Async::HTTP::Internet.get(url, add_headers(headers), &block)
+		end
+
+		def post(url, headers = [], body = nil, &block)
+			Async::HTTP::Internet.post(url, add_headers(headers), body, &block)
+		end
+
 		def load_many(max_at_once: 10)
 			barrier = Async::Barrier.new
 			semaphore = Async::Semaphore.new(max_at_once, parent: barrier)
@@ -15,5 +24,14 @@ module DTIRequests
 				barrier.stop # If any subtasks failed, cancel the rest.
 			end
 		end
+
+		private
+
+		def add_headers(headers)
+			if headers.none? { |(k, v)| k.downcase == "user-agent" }
+				headers += [["User-Agent", Rails.configuration.user_agent_for_neopets]]
+			end
+			headers
+		end
 	end
 end

lib/tasks/neopets/import/rainbow_pool.rake

@@ -1,5 +1,4 @@
 require "addressable/template"
-require "async/http/internet/instance"
 
 namespace "neopets:import" do
 	desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes"
@@ -60,10 +59,6 @@ namespace "neopets:import" do
 end
 
 module RainbowPool
-	# Share a pool of persistent connections, rather than reconnecting on
-	# each request. (This library does that automatically!)
-	INTERNET = Async::HTTP::Internet.instance
-
 	class << self
 		SPECIES_PAGE_URL_TEMPLATE = Addressable::Template.new(
 			"https://www.neopets.com/pool/all_pb.phtml{?f_species_id}"
@@ -71,10 +66,10 @@ module RainbowPool
 		def load_hashes_for_species(species_id, neologin)
 			Sync do
 				url = SPECIES_PAGE_URL_TEMPLATE.expand(f_species_id: species_id)
-				INTERNET.get(url, [
+				DTIRequests.get(
-					["User-Agent", Rails.configuration.user_agent_for_neopets],
+					url,
-					["Cookie", "neologin=#{neologin}"],
+					[["Cookie", "neologin=#{neologin}"]],
-				]) do |response|
+				) do |response|
 					if response.status != 200
 						raise "expected status 200 but got #{response.status} (#{url})"
 					end

spec/services/nc_mall_spec.rb

@@ -9,7 +9,8 @@ RSpec.describe Neopets::NCMall, type: :model do
 					headers: {
 						"Content-Type": "application/x-www-form-urlencoded",
 						"X-Requested-With": "XMLHttpRequest",
-						"Cookie": "neologin=STUB_NEOLOGIN"
+						"Cookie": "neologin=STUB_NEOLOGIN",
+						"User-Agent": Rails.configuration.user_agent_for_neopets,
 					},
 					body: "mode=getStyles&species=2&tab=1",
 				)