From c48b2b14aa6d7213c187c54e7291f9f0f7167f43 Mon Sep 17 00:00:00 2001 From: Emi Matchu Date: Sat, 29 Mar 2025 14:09:50 -0700 Subject: [PATCH] Add workarounds for new Neopets.com security rules Neopets.com recently added some new security rules that, if not satisfied, cause the request to return 403 Forbidden. We figured these out through trial and error, and added them to the `DTIRequests` library, so they would apply to all requests we make. We also updated our AMFPHP library to use `DTIRequests` as well, as an easy way to get the same security rules to apply to those requests. This change was motivated by pet loading being down for the past day or so, because all pet loading requests were returning 403 Forbidden! Now, we've fixed it, hooray! --- app/services/neopets/custom_pets.rb | 4 +- config/application.rb | 5 +- lib/dti_requests.rb | 28 +++++++++-- .../remote_gateway/request.rb | 49 +++++++------------ 4 files changed, 46 insertions(+), 40 deletions(-) diff --git a/app/services/neopets/custom_pets.rb b/app/services/neopets/custom_pets.rb index 9e38b545..5b90bf99 100644 --- a/app/services/neopets/custom_pets.rb +++ b/app/services/neopets/custom_pets.rb @@ -49,9 +49,7 @@ module Neopets::CustomPets # Return the response body as a `HashWithIndifferentAccess`. def send_amfphp_request(request, timeout: 10) begin - response_data = request.post(timeout: timeout, headers: { - "User-Agent" => Rails.configuration.user_agent_for_neopets, - }) + response_data = request.post(timeout: timeout) rescue RocketAMFExtensions::RemoteGateway::AMFError => e raise DownloadError, e.message rescue RocketAMFExtensions::RemoteGateway::ConnectionError => e diff --git a/config/application.rb b/config/application.rb index a1c0b745..1d919e8c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -64,7 +64,10 @@ module OpenneoImpressItems # symbols? So I can't provide anything helpful like a URL, email address, # version number, etc. So let's only send this to Neopets systems, where it # should hopefully be clear who we are from context! - config.user_agent_for_neopets = "Dress to Impress" + # + # NOTE: To be able to access Neopets.com, the User-Agent string must contain + # a slash character. + config.user_agent_for_neopets = "Dress to Impress (https://impress.openneo.net)" # Use the usual Neopets.com, unless we have an override. (At times, we've # used this in collaboration with TNT to address the server directly, diff --git a/lib/dti_requests.rb b/lib/dti_requests.rb index 58a4a898..f7876709 100644 --- a/lib/dti_requests.rb +++ b/lib/dti_requests.rb @@ -5,11 +5,11 @@ require "async/http/internet/instance" module DTIRequests class << self def get(url, headers = [], &block) - Async::HTTP::Internet.get(url, add_headers(headers), &block) + Async::HTTP::Internet.get(url, ensure_headers(headers), &block) end def post(url, headers = [], body = nil, &block) - Async::HTTP::Internet.post(url, add_headers(headers), body, &block) + Async::HTTP::Internet.post(url, ensure_headers(headers), body, &block) end def load_many(max_at_once: 10) @@ -27,10 +27,28 @@ module DTIRequests private - def add_headers(headers) - if headers.none? { |(k, v)| k.downcase == "user-agent" } - headers += [["User-Agent", Rails.configuration.user_agent_for_neopets]] + def ensure_headers(headers) + # To access Neopets.com, requests must have a User-Agent header that + # contains a slash. + headers = ensure_header(headers, "User-Agent", Rails.configuration.user_agent_for_neopets) + + # To access Neopets.com, requests must have the following headers + # present, even with the most basic value possible. + headers = ensure_header(headers, "Accept", "*/*") + headers = ensure_header(headers, "Accept-Language", "*") + headers = ensure_header(headers, "Cookie", " ") + + # NOTE: An Accept-Encoding header is also required, but the underlying + # library already manages this. Don't mess with it! + + headers + end + + def ensure_header(headers, name, value) + if headers.none? { |(k, v)| k.downcase == name.downcase } + headers += [[name, value]] end + headers end end diff --git a/lib/rocketamf_extensions/remote_gateway/request.rb b/lib/rocketamf_extensions/remote_gateway/request.rb index 9adde85a..81a7e484 100644 --- a/lib/rocketamf_extensions/remote_gateway/request.rb +++ b/lib/rocketamf_extensions/remote_gateway/request.rb @@ -11,36 +11,12 @@ module RocketAMFExtensions end def post(options={}) - uri = @action.service.gateway.uri - data = envelope.serialize - - req = Net::HTTP::Post.new(uri.request_uri) - req.body = data - headers = options[:headers] || {} - headers.each do |key, value| - req[key] = value - end - - res = nil - - if options[:timeout] + response_body = if options[:timeout] Timeout.timeout(options[:timeout], ConnectionError) do - res = send_request(uri, req) + send_request(options) end else - res = send_request(uri, req) - end - - if res.is_a?(Net::HTTPSuccess) - response_body = res.body - else - error = nil - begin - res.error! - rescue Exception => scoped_error - error = scoped_error - end - raise ConnectionError, error.message + send_request(options) end begin @@ -95,11 +71,22 @@ module RocketAMFExtensions message end - def send_request(uri, req) + def send_request(options={}) + url = @action.service.gateway.uri + headers = options.fetch(:headers, []).to_a + body = envelope.serialize + begin - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true if uri.instance_of? URI::HTTPS - return http.request(req) + Sync do + DTIRequests.post(url, headers, body) do |response| + if response.status != 200 + raise ConnectionError, + "expected status 200 but got #{response.status} (#{url})" + end + + response.read + end + end rescue Exception => e raise ConnectionError, e.message end