Upgrade async and related gems, and fix async-http response handling

When playing with a Rainbow Pool syncing task, I noticed that error
handling wasn't working correctly for requests using `async-http`: if
the block raised an error, the `Sync` block would never return.

My suspicion is that this is because we were never reading or releasing
the request body.

In this change, I upgrade all the relevant gems for good measure, and
switch to using the response object yielded by the _block_, so we can
know it's being resource-managed correctly. Now, failures raise errors
as expected!

(I tested all these relevant service calls, too!)
This commit is contained in:
Emi Matchu 2024-09-07 12:14:12 -07:00
parent c9f2d660bc
commit be560e4595
28 changed files with 76 additions and 71 deletions

View file

@ -4,7 +4,7 @@ ruby '3.3.4'
gem 'rails', '~> 7.1', '>= 7.1.3.4'
# The HTTP server running the Rails instance.
gem 'falcon', '~> 0.43.0'
gem 'falcon', '~> 0.48.0'
# Our database is MySQL, in both development and production.
gem 'mysql2', '~> 0.5.5'
@ -61,8 +61,8 @@ gem "httparty", "~> 0.22.0"
gem "addressable", "~> 2.8"
# For advanced batching of many HTTP requests.
gem "async", "~> 2.6", require: false
gem "async-http", "~> 0.61.0", require: false
gem "async", "~> 2.17", require: false
gem "async-http", "~> 0.75.0", require: false
gem "thread-local", "~> 1.1", require: false
# For debugging.

View file

@ -81,29 +81,30 @@ GEM
public_suffix (>= 2.0.2, < 7.0)
aes_key_wrap (1.1.0)
ast (2.4.2)
async (2.16.1)
async (2.17.0)
console (~> 1.26)
fiber-annotation
io-event (~> 1.6, >= 1.6.5)
async-container (0.16.13)
async
async-io
async-http (0.61.0)
async (>= 1.25)
async-io (>= 1.28)
async-pool (>= 0.2)
protocol-http (~> 0.25.0)
protocol-http1 (~> 0.16.0)
protocol-http2 (~> 0.15.0)
traces (>= 0.10.0)
async-container (0.18.3)
async (~> 2.10)
async-http (0.75.0)
async (>= 2.10.2)
async-pool (~> 0.7)
io-endpoint (~> 0.11)
io-stream (~> 0.4)
protocol-http (~> 0.30)
protocol-http1 (~> 0.20)
protocol-http2 (~> 0.18)
traces (>= 0.10)
async-http-cache (0.4.4)
async-http (~> 0.56)
async-io (1.43.2)
async
async-pool (0.8.1)
async (>= 1.25)
metrics
traces
async-service (0.12.0)
async
async-container (~> 0.16)
attr_required (1.0.2)
babel-source (5.8.35)
babel-transpiler (0.7.0)
@ -118,7 +119,6 @@ GEM
bindex (0.8.1)
bootsnap (1.18.4)
msgpack (~> 1.2)
build-environment (1.13.0)
builder (3.3.0)
childprocess (5.1.0)
logger (~> 1.5)
@ -150,19 +150,19 @@ GEM
activemodel
erubi (1.13.0)
execjs (2.9.1)
falcon (0.43.0)
falcon (0.48.0)
async
async-container (~> 0.16.0)
async-http (~> 0.57)
async-http-cache (~> 0.4.0)
async-io (~> 1.22)
build-environment (~> 1.13)
async-container (~> 0.18)
async-http (~> 0.75)
async-http-cache (~> 0.4)
async-service (~> 0.10)
bundler
localhost (~> 1.1)
openssl (~> 3.0)
process-metrics (~> 0.2.0)
protocol-rack (~> 0.1)
samovar (~> 2.1)
process-metrics (~> 0.2)
protocol-http (~> 0.31)
protocol-rack (~> 0.7)
samovar (~> 2.3)
faraday (2.11.0)
faraday-net_http (>= 2.0, < 3.4)
logger
@ -190,7 +190,9 @@ GEM
i18n (1.14.5)
concurrent-ruby (~> 1.0)
io-console (0.7.2)
io-endpoint (0.13.1)
io-event (1.6.5)
io-stream (0.4.0)
irb (1.14.0)
rdoc (>= 4.0.0)
reline (>= 0.4.2)
@ -280,18 +282,19 @@ GEM
parser (3.3.4.2)
ast (~> 2.4.1)
racc
process-metrics (0.2.1)
process-metrics (0.3.0)
console (~> 1.8)
json (~> 2)
samovar (~> 2.1)
protocol-hpack (1.5.0)
protocol-http (0.25.0)
protocol-http1 (0.16.1)
protocol-http (0.33.0)
protocol-http1 (0.22.0)
protocol-http (~> 0.22)
protocol-http2 (0.15.1)
protocol-http2 (0.18.0)
protocol-hpack (~> 1.4)
protocol-http (~> 0.18)
protocol-rack (0.6.0)
protocol-http (~> 0.23)
protocol-rack (0.7.0)
protocol-http (~> 0.27)
rack (>= 1.0)
psych (5.1.2)
stringio
@ -495,13 +498,13 @@ PLATFORMS
DEPENDENCIES
RocketAMF!
addressable (~> 2.8)
async (~> 2.6)
async-http (~> 0.61.0)
async (~> 2.17)
async-http (~> 0.75.0)
bootsnap (~> 1.16)
devise (~> 4.9, >= 4.9.2)
devise-encryptable (~> 0.2.0)
dotenv-rails (~> 2.8, >= 2.8.1)
falcon (~> 0.43.0)
falcon (~> 0.48.0)
haml (~> 6.1, >= 6.1.1)
http_accept_language (~> 2.1, >= 2.1.1)
httparty (~> 0.22.0)

View file

@ -25,39 +25,40 @@ module NCMall
ROOT_DOCUMENT_URL = "https://ncmall.neopets.com/mall/shop.phtml"
PAGE_LINK_PATTERN = /load_items_pane\(['"](.+?)['"], ([0-9]+)\).+?>(.+?)</
def self.load_page_links
Sync do
response = INTERNET.get(ROOT_DOCUMENT_URL, [
html = Sync do
INTERNET.get(ROOT_DOCUMENT_URL, [
["User-Agent", Rails.configuration.user_agent_for_neopets],
])
]) do |response|
if response.status != 200
raise ResponseNotOK.new(response.status),
"expected status 200 but got #{response.status} (#{url})"
end
if response.status != 200
raise ResponseNotOK.new(response.status),
"expected status 200 but got #{response.status} (#{url})"
response.read
end
# Extract `load_items_pane` calls from the root document's HTML. (We use
# a very simplified regex, rather than actually parsing the full HTML!)
html = response.read
html.scan(PAGE_LINK_PATTERN).
map { |type, cat, label| {type:, cat:, label:} }.
uniq
end
# Extract `load_items_pane` calls from the root document's HTML. (We use
# a very simplified regex, rather than actually parsing the full HTML!)
html.scan(PAGE_LINK_PATTERN).
map { |type, cat, label| {type:, cat:, label:} }.
uniq
end
private
def self.load_page_by_url(url)
Sync do
response = INTERNET.get(url, [
INTERNET.get(url, [
["User-Agent", Rails.configuration.user_agent_for_neopets],
])
]) do |response|
if response.status != 200
raise ResponseNotOK.new(response.status),
"expected status 200 but got #{response.status} (#{url})"
end
if response.status != 200
raise ResponseNotOK.new(response.status),
"expected status 200 but got #{response.status} (#{url})"
parse_nc_page response.read
end
parse_nc_page response.read
end
end

View file

@ -31,20 +31,20 @@ module NeoPass
LINKAGE_URL = "https://oidc.neopets.com/linkage/all"
def self.load_linkages(access_token)
response = Sync do
response = INTERNET.get(LINKAGE_URL, [
linkages_str = Sync do
INTERNET.get(LINKAGE_URL, [
["User-Agent", Rails.configuration.user_agent_for_neopets],
["Authorization", "Bearer #{access_token}"],
])
end
]) do |response|
if response.status != 200
raise ResponseNotOK.new(response.status),
"expected status 200 but got #{response.status} (#{LINKAGE_URL})"
end
if response.status != 200
raise ResponseNotOK.new(response.status),
"expected status 200 but got #{response.status} (#{LINKAGE_URL})"
response.read
end
end
linkages_str = response.body.read
begin
linkages = JSON.parse(linkages_str)
rescue JSON::ParserError

View file

@ -72,14 +72,15 @@ module NeopetsMediaArchive
# We use this in the `swf_assets:manifests:load` task to perform many
# requests in parallel!
Sync do
response = INTERNET.get(uri, [
INTERNET.get(uri, [
["User-Agent", Rails.configuration.user_agent_for_neopets],
])
if response.status != 200
raise ResponseNotOK.new(response.status),
"expected status 200 but got #{response.status} (#{uri})"
]) do |response|
if response.status != 200
raise ResponseNotOK.new(response.status),
"expected status 200 but got #{response.status} (#{uri})"
end
response.read
end
response.body.read
end
end

Binary file not shown.

BIN
vendor/cache/async-2.17.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/async-container-0.18.3.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/async-http-0.75.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/async-service-0.12.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/falcon-0.48.0.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/io-endpoint-0.13.1.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/io-stream-0.4.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/process-metrics-0.3.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/protocol-http-0.33.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/protocol-http1-0.22.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/protocol-http2-0.18.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/protocol-rack-0.7.0.gem vendored Normal file

Binary file not shown.