Upgrade async and related gems, and fix async-http response handling

When playing with a Rainbow Pool syncing task, I noticed that error
handling wasn't working correctly for requests using `async-http`: if
the block raised an error, the `Sync` block would never return.

My suspicion is that this is because we were never reading or releasing
the request body.

In this change, I upgrade all the relevant gems for good measure, and
switch to using the response object yielded by the _block_, so we can
know it's being resource-managed correctly. Now, failures raise errors
as expected!

(I tested all these relevant service calls, too!)

Gemfile

@@ -4,7 +4,7 @@ ruby '3.3.4'
 gem 'rails', '~> 7.1', '>= 7.1.3.4'
 
 # The HTTP server running the Rails instance.
-gem 'falcon', '~> 0.43.0'
+gem 'falcon', '~> 0.48.0'
 
 # Our database is MySQL, in both development and production.
 gem 'mysql2', '~> 0.5.5'
@@ -61,8 +61,8 @@ gem "httparty", "~> 0.22.0"
 gem "addressable", "~> 2.8"
 
 # For advanced batching of many HTTP requests.
-gem "async", "~> 2.6", require: false
-gem "async-http", "~> 0.61.0", require: false
+gem "async", "~> 2.17", require: false
+gem "async-http", "~> 0.75.0", require: false
 gem "thread-local", "~> 1.1", require: false
 
 # For debugging.

Gemfile.lock

@@ -81,29 +81,30 @@ GEM
       public_suffix (>= 2.0.2, < 7.0)
     aes_key_wrap (1.1.0)
     ast (2.4.2)
-    async (2.16.1)
+    async (2.17.0)
       console (~> 1.26)
       fiber-annotation
       io-event (~> 1.6, >= 1.6.5)
-    async-container (0.16.13)
-      async
-      async-io
-    async-http (0.61.0)
-      async (>= 1.25)
-      async-io (>= 1.28)
-      async-pool (>= 0.2)
-      protocol-http (~> 0.25.0)
-      protocol-http1 (~> 0.16.0)
-      protocol-http2 (~> 0.15.0)
-      traces (>= 0.10.0)
+    async-container (0.18.3)
+      async (~> 2.10)
+    async-http (0.75.0)
+      async (>= 2.10.2)
+      async-pool (~> 0.7)
+      io-endpoint (~> 0.11)
+      io-stream (~> 0.4)
+      protocol-http (~> 0.30)
+      protocol-http1 (~> 0.20)
+      protocol-http2 (~> 0.18)
+      traces (>= 0.10)
     async-http-cache (0.4.4)
       async-http (~> 0.56)
-    async-io (1.43.2)
-      async
     async-pool (0.8.1)
       async (>= 1.25)
       metrics
       traces
+    async-service (0.12.0)
+      async
+      async-container (~> 0.16)
     attr_required (1.0.2)
     babel-source (5.8.35)
     babel-transpiler (0.7.0)
@@ -118,7 +119,6 @@ GEM
     bindex (0.8.1)
     bootsnap (1.18.4)
       msgpack (~> 1.2)
-    build-environment (1.13.0)
     builder (3.3.0)
     childprocess (5.1.0)
       logger (~> 1.5)
@@ -150,19 +150,19 @@ GEM
       activemodel
     erubi (1.13.0)
     execjs (2.9.1)
-    falcon (0.43.0)
+    falcon (0.48.0)
       async
-      async-container (~> 0.16.0)
-      async-http (~> 0.57)
-      async-http-cache (~> 0.4.0)
-      async-io (~> 1.22)
-      build-environment (~> 1.13)
+      async-container (~> 0.18)
+      async-http (~> 0.75)
+      async-http-cache (~> 0.4)
+      async-service (~> 0.10)
       bundler
       localhost (~> 1.1)
       openssl (~> 3.0)
-      process-metrics (~> 0.2.0)
-      protocol-rack (~> 0.1)
-      samovar (~> 2.1)
+      process-metrics (~> 0.2)
+      protocol-http (~> 0.31)
+      protocol-rack (~> 0.7)
+      samovar (~> 2.3)
     faraday (2.11.0)
       faraday-net_http (>= 2.0, < 3.4)
       logger
@@ -190,7 +190,9 @@ GEM
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     io-console (0.7.2)
+    io-endpoint (0.13.1)
     io-event (1.6.5)
+    io-stream (0.4.0)
     irb (1.14.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
@@ -280,18 +282,19 @@ GEM
     parser (3.3.4.2)
       ast (~> 2.4.1)
       racc
-    process-metrics (0.2.1)
+    process-metrics (0.3.0)
       console (~> 1.8)
+      json (~> 2)
       samovar (~> 2.1)
     protocol-hpack (1.5.0)
-    protocol-http (0.25.0)
-    protocol-http1 (0.16.1)
+    protocol-http (0.33.0)
+    protocol-http1 (0.22.0)
       protocol-http (~> 0.22)
-    protocol-http2 (0.15.1)
+    protocol-http2 (0.18.0)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.18)
-    protocol-rack (0.6.0)
-      protocol-http (~> 0.23)
+    protocol-rack (0.7.0)
+      protocol-http (~> 0.27)
       rack (>= 1.0)
     psych (5.1.2)
       stringio
@@ -495,13 +498,13 @@ PLATFORMS
 DEPENDENCIES
   RocketAMF!
   addressable (~> 2.8)
-  async (~> 2.6)
-  async-http (~> 0.61.0)
+  async (~> 2.17)
+  async-http (~> 0.75.0)
   bootsnap (~> 1.16)
   devise (~> 4.9, >= 4.9.2)
   devise-encryptable (~> 0.2.0)
   dotenv-rails (~> 2.8, >= 2.8.1)
-  falcon (~> 0.43.0)
+  falcon (~> 0.48.0)
   haml (~> 6.1, >= 6.1.1)
   http_accept_language (~> 2.1, >= 2.1.1)
   httparty (~> 0.22.0)

app/services/nc_mall.rb

@@ -25,39 +25,40 @@ module NCMall
 	ROOT_DOCUMENT_URL = "https://ncmall.neopets.com/mall/shop.phtml"
 	PAGE_LINK_PATTERN = /load_items_pane\(['"](.+?)['"], ([0-9]+)\).+?>(.+?)</
 	def self.load_page_links
-		Sync do
-			response = INTERNET.get(ROOT_DOCUMENT_URL, [
+		html = Sync do
+			INTERNET.get(ROOT_DOCUMENT_URL, [
 				["User-Agent", Rails.configuration.user_agent_for_neopets],
-			])
+			]) do |response|
+				if response.status != 200
+					raise ResponseNotOK.new(response.status),
+						"expected status 200 but got #{response.status} (#{url})"
+				end
 
-			if response.status != 200
-				raise ResponseNotOK.new(response.status),
-					"expected status 200 but got #{response.status} (#{url})"
+				response.read
 			end
-
-			# Extract `load_items_pane` calls from the root document's HTML. (We use
-			# a very simplified regex, rather than actually parsing the full HTML!)
-			html = response.read
-			html.scan(PAGE_LINK_PATTERN).
-				map { |type, cat, label| {type:, cat:, label:} }.
-				uniq
 		end
+
+		# Extract `load_items_pane` calls from the root document's HTML. (We use
+		# a very simplified regex, rather than actually parsing the full HTML!)
+		html.scan(PAGE_LINK_PATTERN).
+			map { |type, cat, label| {type:, cat:, label:} }.
+			uniq
 	end
 
 	private
 
 	def self.load_page_by_url(url)
 		Sync do
-			response = INTERNET.get(url, [
+			INTERNET.get(url, [
 				["User-Agent", Rails.configuration.user_agent_for_neopets],
-			])
+			]) do |response|
+				if response.status != 200
+					raise ResponseNotOK.new(response.status),
+						"expected status 200 but got #{response.status} (#{url})"
+				end
 
-			if response.status != 200
-				raise ResponseNotOK.new(response.status),
-					"expected status 200 but got #{response.status} (#{url})"
+				parse_nc_page response.read
 			end
-
-			parse_nc_page response.read
 		end
 	end
 

app/services/neopass.rb

@@ -31,20 +31,20 @@ module NeoPass
 
 	LINKAGE_URL = "https://oidc.neopets.com/linkage/all"
 	def self.load_linkages(access_token)
-		response = Sync do
-			response = INTERNET.get(LINKAGE_URL, [
+		linkages_str = Sync do
+			INTERNET.get(LINKAGE_URL, [
 				["User-Agent", Rails.configuration.user_agent_for_neopets],
 				["Authorization", "Bearer #{access_token}"],
-			])
-		end
+			]) do |response|
+				if response.status != 200
+					raise ResponseNotOK.new(response.status),
+						"expected status 200 but got #{response.status} (#{LINKAGE_URL})"
+				end
 
-		if response.status != 200
-			raise ResponseNotOK.new(response.status),
-				"expected status 200 but got #{response.status} (#{LINKAGE_URL})"
+				response.read
+			end
 		end
 
-		linkages_str = response.body.read
-
 		begin
 			linkages = JSON.parse(linkages_str)
 		rescue JSON::ParserError

app/services/neopets_media_archive.rb

@@ -72,14 +72,15 @@ module NeopetsMediaArchive
     # We use this in the `swf_assets:manifests:load` task to perform many
     # requests in parallel!
     Sync do
-      response = INTERNET.get(uri, [
+      INTERNET.get(uri, [
         ["User-Agent", Rails.configuration.user_agent_for_neopets],
-      ])
-      if response.status != 200
-        raise ResponseNotOK.new(response.status),
-          "expected status 200 but got #{response.status} (#{uri})"
+      ]) do |response|
+        if response.status != 200
+          raise ResponseNotOK.new(response.status),
+            "expected status 200 but got #{response.status} (#{uri})"
+        end
+        response.read
       end
-      response.body.read
     end
   end