From 86205c5e44ea3f07f0306ce13dcb16118b83ded5 Mon Sep 17 00:00:00 2001 From: Emi Matchu Date: Mon, 16 Dec 2024 14:04:22 -0800 Subject: [PATCH] Create new `DTIRequests.{get,post}` helpers for HTTP requests Instead of using `Async::HTTP::Internet` directly, and always applying our `User-Agent` header manually, let's build a helper for it! --- app/services/neopets/nc_mall.rb | 20 +++++--------------- app/services/neopets/neopass.rb | 14 ++++---------- app/services/neopets_media_archive.rb | 9 +-------- lib/dti_requests.rb | 18 ++++++++++++++++++ lib/tasks/neopets/import/rainbow_pool.rake | 13 ++++--------- spec/services/nc_mall_spec.rb | 3 ++- 6 files changed, 34 insertions(+), 43 deletions(-) diff --git a/app/services/neopets/nc_mall.rb b/app/services/neopets/nc_mall.rb index 3916e683..c11fea2e 100644 --- a/app/services/neopets/nc_mall.rb +++ b/app/services/neopets/nc_mall.rb @@ -1,11 +1,6 @@ require "addressable/template" -require "async/http/internet/instance" module Neopets::NCMall - # Share a pool of persistent connections, rather than reconnecting on - # each request. (This library does that automatically!) - INTERNET = Async::HTTP::Internet.instance - # Load the NC Mall home page content area, and return its useful data. HOME_PAGE_URL = "https://ncmall.neopets.com/mall/ajax/home_page.phtml" def self.load_home_page @@ -26,9 +21,7 @@ module Neopets::NCMall PAGE_LINK_PATTERN = /load_items_pane\(['"](.+?)['"], ([0-9]+)\).+?>(.+?) OmniAuth -> OIDC # OmniAuth plugin, NeoPass also offers some supplemental APIs that we use here. module Neopets::NeoPass - # Share a pool of persistent connections, rather than reconnecting on - # each request. (This library does that automatically!) - INTERNET = Async::HTTP::Internet.instance - def self.load_main_neopets_username(access_token) linkages = load_linkages(access_token) @@ -32,10 +26,10 @@ module Neopets::NeoPass LINKAGE_URL = "https://oidc.neopets.com/linkage/all" def self.load_linkages(access_token) linkages_str = Sync do - INTERNET.get(LINKAGE_URL, [ - ["User-Agent", Rails.configuration.user_agent_for_neopets], - ["Authorization", "Bearer #{access_token}"], - ]) do |response| + DTIRequests.get( + LINKAGE_URL, + [["Authorization", "Bearer #{access_token}"]], + ) do |response| if response.status != 200 raise ResponseNotOK.new(response.status), "expected status 200 but got #{response.status} (#{LINKAGE_URL})" diff --git a/app/services/neopets_media_archive.rb b/app/services/neopets_media_archive.rb index c49affe6..88258093 100644 --- a/app/services/neopets_media_archive.rb +++ b/app/services/neopets_media_archive.rb @@ -1,5 +1,4 @@ require "addressable/uri" -require "async/http/internet/instance" require "json" # The Neopets Media Archive is a service that mirrors images.neopets.com files @@ -11,10 +10,6 @@ require "json" # long-term archive, not dependent on their services having 100% uptime in # order for us to operate. We never discard old files, we just keep going! module NeopetsMediaArchive - # Share a pool of persistent connections, rather than reconnecting on - # each request. (This library does that automatically!) - INTERNET = Async::HTTP::Internet.instance - ROOT_PATH = Pathname.new(Rails.configuration.neopets_media_archive_root) # Load the file from the given `images.neopets.com` URI. @@ -72,9 +67,7 @@ module NeopetsMediaArchive # We use this in the `swf_assets:manifests:load` task to perform many # requests in parallel! Sync do - INTERNET.get(uri, [ - ["User-Agent", Rails.configuration.user_agent_for_neopets], - ]) do |response| + DTIRequests.get(uri) do |response| if response.status != 200 raise ResponseNotOK.new(response.status), "expected status 200 but got #{response.status} (#{uri})" diff --git a/lib/dti_requests.rb b/lib/dti_requests.rb index 17347f0b..58a4a898 100644 --- a/lib/dti_requests.rb +++ b/lib/dti_requests.rb @@ -1,8 +1,17 @@ require "async" require "async/barrier" +require "async/http/internet/instance" module DTIRequests class << self + def get(url, headers = [], &block) + Async::HTTP::Internet.get(url, add_headers(headers), &block) + end + + def post(url, headers = [], body = nil, &block) + Async::HTTP::Internet.post(url, add_headers(headers), body, &block) + end + def load_many(max_at_once: 10) barrier = Async::Barrier.new semaphore = Async::Semaphore.new(max_at_once, parent: barrier) @@ -15,5 +24,14 @@ module DTIRequests barrier.stop # If any subtasks failed, cancel the rest. end end + + private + + def add_headers(headers) + if headers.none? { |(k, v)| k.downcase == "user-agent" } + headers += [["User-Agent", Rails.configuration.user_agent_for_neopets]] + end + headers + end end end diff --git a/lib/tasks/neopets/import/rainbow_pool.rake b/lib/tasks/neopets/import/rainbow_pool.rake index e0b16bc6..8e66592d 100644 --- a/lib/tasks/neopets/import/rainbow_pool.rake +++ b/lib/tasks/neopets/import/rainbow_pool.rake @@ -1,5 +1,4 @@ require "addressable/template" -require "async/http/internet/instance" namespace "neopets:import" do desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes" @@ -60,10 +59,6 @@ namespace "neopets:import" do end module RainbowPool - # Share a pool of persistent connections, rather than reconnecting on - # each request. (This library does that automatically!) - INTERNET = Async::HTTP::Internet.instance - class << self SPECIES_PAGE_URL_TEMPLATE = Addressable::Template.new( "https://www.neopets.com/pool/all_pb.phtml{?f_species_id}" @@ -71,10 +66,10 @@ module RainbowPool def load_hashes_for_species(species_id, neologin) Sync do url = SPECIES_PAGE_URL_TEMPLATE.expand(f_species_id: species_id) - INTERNET.get(url, [ - ["User-Agent", Rails.configuration.user_agent_for_neopets], - ["Cookie", "neologin=#{neologin}"], - ]) do |response| + DTIRequests.get( + url, + [["Cookie", "neologin=#{neologin}"]], + ) do |response| if response.status != 200 raise "expected status 200 but got #{response.status} (#{url})" end diff --git a/spec/services/nc_mall_spec.rb b/spec/services/nc_mall_spec.rb index 584c863e..96c7ab35 100644 --- a/spec/services/nc_mall_spec.rb +++ b/spec/services/nc_mall_spec.rb @@ -9,7 +9,8 @@ RSpec.describe Neopets::NCMall, type: :model do headers: { "Content-Type": "application/x-www-form-urlencoded", "X-Requested-With": "XMLHttpRequest", - "Cookie": "neologin=STUB_NEOLOGIN" + "Cookie": "neologin=STUB_NEOLOGIN", + "User-Agent": Rails.configuration.user_agent_for_neopets, }, body: "mode=getStyles&species=2&tab=1", )