Add workarounds for new Neopets.com security rules

Neopets.com recently added some new security rules that, if not
satisfied, cause the request to return 403 Forbidden.

We figured these out through trial and error, and added them to the
`DTIRequests` library, so they would apply to all requests we make.

We also updated our AMFPHP library to use `DTIRequests` as well, as an
easy way to get the same security rules to apply to those requests.

This change was motivated by pet loading being down for the past day or
so, because all pet loading requests were returning 403 Forbidden! Now,
we've fixed it, hooray!

app/services/neopets/custom_pets.rb

@@ -49,9 +49,7 @@ module Neopets::CustomPets
 		# Return the response body as a `HashWithIndifferentAccess`.
 		def send_amfphp_request(request, timeout: 10)
 			begin
-				response_data = request.post(timeout: timeout, headers: {
+				response_data = request.post(timeout: timeout)
-					"User-Agent" => Rails.configuration.user_agent_for_neopets,
-				})
 			rescue RocketAMFExtensions::RemoteGateway::AMFError => e
 				raise DownloadError, e.message
 			rescue RocketAMFExtensions::RemoteGateway::ConnectionError => e

config/application.rb

@@ -64,7 +64,10 @@ module OpenneoImpressItems
     # symbols? So I can't provide anything helpful like a URL, email address,
     # version number, etc. So let's only send this to Neopets systems, where it
     # should hopefully be clear who we are from context!
-    config.user_agent_for_neopets = "Dress to Impress"
+    #
+    # NOTE: To be able to access Neopets.com, the User-Agent string must contain
+    #       a slash character.
+    config.user_agent_for_neopets = "Dress to Impress (https://impress.openneo.net)"
 
     # Use the usual Neopets.com, unless we have an override. (At times, we've
     # used this in collaboration with TNT to address the server directly,

lib/dti_requests.rb

@@ -5,11 +5,11 @@ require "async/http/internet/instance"
 module DTIRequests
 	class << self
 		def get(url, headers = [], &block)
-			Async::HTTP::Internet.get(url, add_headers(headers), &block)
+			Async::HTTP::Internet.get(url, ensure_headers(headers), &block)
 		end
 
 		def post(url, headers = [], body = nil, &block)
-			Async::HTTP::Internet.post(url, add_headers(headers), body, &block)
+			Async::HTTP::Internet.post(url, ensure_headers(headers), body, &block)
 		end
 
 		def load_many(max_at_once: 10)
@@ -27,10 +27,28 @@ module DTIRequests
 
 		private
 
-		def add_headers(headers)
+		def ensure_headers(headers)
-			if headers.none? { |(k, v)| k.downcase == "user-agent" }
+			# To access Neopets.com, requests must have a User-Agent header that
-				headers += [["User-Agent", Rails.configuration.user_agent_for_neopets]]
+			# contains a slash.
+			headers = ensure_header(headers, "User-Agent", Rails.configuration.user_agent_for_neopets)
+
+			# To access Neopets.com, requests must have the following headers
+			# present, even with the most basic value possible.
+			headers = ensure_header(headers, "Accept", "*/*")
+			headers = ensure_header(headers, "Accept-Language", "*")
+			headers = ensure_header(headers, "Cookie", " ")
+
+			# NOTE: An Accept-Encoding header is also required, but the underlying
+			#       library already manages this. Don't mess with it!
+
+			headers
+		end
+
+		def ensure_header(headers, name, value)
+			if headers.none? { |(k, v)| k.downcase == name.downcase }
+				headers += [[name, value]]
 			end
+
 			headers
 		end
 	end

lib/rocketamf_extensions/remote_gateway/request.rb

@@ -11,36 +11,12 @@ module RocketAMFExtensions
       end
       
       def post(options={})
-        uri = @action.service.gateway.uri
+        response_body = if options[:timeout]
-        data = envelope.serialize
-
-        req = Net::HTTP::Post.new(uri.request_uri)
-        req.body = data
-        headers = options[:headers] || {}
-        headers.each do |key, value|
-          req[key] = value
-        end
-        
-        res = nil
-        
-        if options[:timeout]
           Timeout.timeout(options[:timeout], ConnectionError) do
-            res = send_request(uri, req)
+            send_request(options)
           end
         else
-          res = send_request(uri, req)
+          send_request(options)
-        end
-        
-        if res.is_a?(Net::HTTPSuccess)
-          response_body = res.body
-        else
-          error = nil
-          begin
-            res.error!
-          rescue Exception => scoped_error
-            error = scoped_error
-          end
-          raise ConnectionError, error.message
         end
         
         begin
@@ -95,11 +71,22 @@ module RocketAMFExtensions
         message
       end
       
-      def send_request(uri, req)
+      def send_request(options={})
+        url = @action.service.gateway.uri
+        headers = options.fetch(:headers, []).to_a
+        body = envelope.serialize
+
         begin
-          http = Net::HTTP.new(uri.host, uri.port)
+          Sync do
-          http.use_ssl = true if uri.instance_of? URI::HTTPS
+            DTIRequests.post(url, headers, body) do |response|
-          return http.request(req)
+              if response.status != 200
+                raise ConnectionError,
+                  "expected status 200 but got #{response.status} (#{url})"
+              end
+
+              response.read
+            end
+          end
         rescue Exception => e
           raise ConnectionError, e.message
         end