Add workarounds for new Neopets.com security rules

Neopets.com recently added some new security rules that, if not
satisfied, cause the request to return 403 Forbidden.

We figured these out through trial and error, and added them to the
`DTIRequests` library, so they would apply to all requests we make.

We also updated our AMFPHP library to use `DTIRequests` as well, as an
easy way to get the same security rules to apply to those requests.

This change was motivated by pet loading being down for the past day or
so, because all pet loading requests were returning 403 Forbidden! Now,
we've fixed it, hooray!

.ruby-version

@@ -1 +1 @@
-3.3.6
+3.3.7

Gemfile

@@ -1,5 +1,5 @@
 source 'https://rubygems.org'
-ruby '3.3.6'
+ruby '3.3.7'
 
 gem 'rails', '~> 8.0', '>= 8.0.1'
 

Gemfile.lock

@@ -571,7 +571,7 @@ DEPENDENCIES
   will_paginate (~> 4.0)
 
 RUBY VERSION
-   ruby 3.3.6p108
+   ruby 3.3.7p123
 
 BUNDLED WITH
    2.5.18

app/services/neopets/custom_pets.rb

@@ -49,9 +49,7 @@ module Neopets::CustomPets
 		# Return the response body as a `HashWithIndifferentAccess`.
 		def send_amfphp_request(request, timeout: 10)
 			begin
-				response_data = request.post(timeout: timeout, headers: {
-					"User-Agent" => Rails.configuration.user_agent_for_neopets,
-				})
+				response_data = request.post(timeout: timeout)
 			rescue RocketAMFExtensions::RemoteGateway::AMFError => e
 				raise DownloadError, e.message
 			rescue RocketAMFExtensions::RemoteGateway::ConnectionError => e

config/application.rb

@@ -64,7 +64,10 @@ module OpenneoImpressItems
     # symbols? So I can't provide anything helpful like a URL, email address,
     # version number, etc. So let's only send this to Neopets systems, where it
     # should hopefully be clear who we are from context!
-    config.user_agent_for_neopets = "Dress to Impress"
+    #
+    # NOTE: To be able to access Neopets.com, the User-Agent string must contain
+    #       a slash character.
+    config.user_agent_for_neopets = "Dress to Impress / impress.openneo.net"
 
     # Use the usual Neopets.com, unless we have an override. (At times, we've
     # used this in collaboration with TNT to address the server directly,

deploy/deploy.yml

@@ -6,7 +6,7 @@
   vars:
     local_app_root: "{{ playbook_dir  }}/.."
     remote_project_root: "/srv/impress"
-    ruby_version: "3.3.6"
+    ruby_version: "3.3.7"
 
     # deploy:setup should have added us to the "impress-deployers" group, so we
     # should be able to become the "impress" user without a password.

deploy/files/impress.service.j2

@@ -5,7 +5,7 @@ Description=Dress to Impress webapp
 User=impress
 Restart=always
 WorkingDirectory=/srv/impress/current
-ExecStart=/opt/ruby-{{ ruby_version }}/bin/bundle exec falcon host
+ExecStart=/opt/ruby-{{ ruby_version_live }}/bin/bundle exec falcon host
 Environment="RAILS_ENV=production"
 ; Set EXECJS_RUNTIME to save us from needing to install Node
 Environment="EXECJS_RUNTIME=Disabled"

deploy/setup.yml

@@ -6,7 +6,10 @@
   vars:
     email_address: "emi@matchu.dev" # TODO: Extract this to personal config?
     impress_hostname: impress.openneo.net
-    ruby_version: "3.3.6"
+
+    # These two values should usually be the same, unless you're in the middle of an upgrade!
+    ruby_version_latest: "3.3.7" # The latest Ruby to install.
+    ruby_version_live: "3.3.7" # The Ruby to run the latest version of the app with.
   vars_files:
     # mysql_root_password, mysql_user_password, mysql_user_password_2020,
     # dev_ips
@@ -171,21 +174,21 @@
       git:
         repo: https://github.com/rbenv/ruby-build.git
         dest: /opt/ruby-build
-        version: v20241225.2
+        version: v20250326
 
-    - name: Check if Ruby {{ ruby_version }} is already installed
+    - name: Check if Ruby {{ ruby_version_latest }} is already installed
       stat:
-        path: /opt/ruby-{{ ruby_version }}
+        path: /opt/ruby-{{ ruby_version_latest }}
       register: ruby_dir
 
-    - name: Install Ruby {{ ruby_version }}
-      command: "/opt/ruby-build/bin/ruby-build {{ ruby_version }} /opt/ruby-{{ ruby_version }}"
+    - name: Install Ruby {{ ruby_version_latest }}
+      command: "/opt/ruby-build/bin/ruby-build {{ ruby_version_latest }} /opt/ruby-{{ ruby_version_latest }}"
       when: not ruby_dir.stat.exists
 
-    - name: Add Ruby {{ ruby_version }} to the global PATH, for developer convenience
+    - name: Add Ruby {{ ruby_version_latest }} to the global PATH, for developer convenience
       copy:
         dest: /etc/profile.d/ruby_path.sh
-        content: PATH="/opt/ruby-{{ ruby_version }}/bin:$PATH"
+        content: PATH="/opt/ruby-{{ ruby_version_latest }}/bin:$PATH"
 
     - name: Install system dependencies for impress's Ruby gems
       apt:
@@ -249,14 +252,14 @@
       become_user: impress
       command:
         chdir: /srv/impress/versions/initial-placeholder
-        cmd: /opt/ruby-{{ ruby_version }}/bin/bundle config set --local deployment true
+        cmd: /opt/ruby-{{ ruby_version_live }}/bin/bundle config set --local deployment true
       when: not current_app_version.stat.exists
 
     - name: Install the placeholder app's dependencies
       become_user: impress
       command:
         chdir: /srv/impress/versions/initial-placeholder
-        cmd: /opt/ruby-{{ ruby_version }}/bin/bundle install
+        cmd: /opt/ruby-{{ ruby_version_live }}/bin/bundle install
       when: not current_app_version.stat.exists
 
     - name: Set the placeholder app as the current version

lib/dti_requests.rb

@@ -5,11 +5,11 @@ require "async/http/internet/instance"
 module DTIRequests
 	class << self
 		def get(url, headers = [], &block)
-			Async::HTTP::Internet.get(url, add_headers(headers), &block)
+			Async::HTTP::Internet.get(url, ensure_headers(headers), &block)
 		end
 
 		def post(url, headers = [], body = nil, &block)
-			Async::HTTP::Internet.post(url, add_headers(headers), body, &block)
+			Async::HTTP::Internet.post(url, ensure_headers(headers), body, &block)
 		end
 
 		def load_many(max_at_once: 10)
@@ -27,10 +27,29 @@ module DTIRequests
 
 		private
 
-		def add_headers(headers)
-			if headers.none? { |(k, v)| k.downcase == "user-agent" }
-				headers += [["User-Agent", Rails.configuration.user_agent_for_neopets]]
+		def ensure_headers(headers)
+			# To access Neopets.com, requests must have a User-Agent header that
+			# contains a slash.
+			headers = ensure_header(headers, "User-Agent", Rails.configuration.user_agent_for_neopets)
+
+			# To access Neopets.com, requests must have the following headers
+			# present, even with the most basic value possible.
+			headers = ensure_header(headers, "Accept", "*/*")
+			headers = ensure_header(headers, "Accept-Language", "*")
+			headers = ensure_header(headers, "Accept-Language", "*")
+			headers = ensure_header(headers, "Cookie", " ")
+
+			# NOTE: An Accept-Encoding header is also required, but the underlying
+			#       library already manages this. Don't mess with it!
+
+			headers
+		end
+
+		def ensure_header(headers, name, value)
+			if headers.none? { |(k, v)| k.downcase == name.downcase }
+				headers += [[name, value]]
 			end
+
 			headers
 		end
 	end

lib/rocketamf_extensions/remote_gateway/request.rb

@@ -11,36 +11,12 @@ module RocketAMFExtensions
       end
       
       def post(options={})
-        uri = @action.service.gateway.uri
-        data = envelope.serialize
-
-        req = Net::HTTP::Post.new(uri.request_uri)
-        req.body = data
-        headers = options[:headers] || {}
-        headers.each do |key, value|
-          req[key] = value
-        end
-        
-        res = nil
-        
-        if options[:timeout]
+        response_body = if options[:timeout]
           Timeout.timeout(options[:timeout], ConnectionError) do
-            res = send_request(uri, req)
+            send_request(options)
           end
         else
-          res = send_request(uri, req)
-        end
-        
-        if res.is_a?(Net::HTTPSuccess)
-          response_body = res.body
-        else
-          error = nil
-          begin
-            res.error!
-          rescue Exception => scoped_error
-            error = scoped_error
-          end
-          raise ConnectionError, error.message
+          send_request(options)
         end
         
         begin
@@ -95,11 +71,22 @@ module RocketAMFExtensions
         message
       end
       
-      def send_request(uri, req)
+      def send_request(options={})
+        url = @action.service.gateway.uri
+        headers = options.fetch(:headers, []).to_a
+        body = envelope.serialize
+
         begin
-          http = Net::HTTP.new(uri.host, uri.port)
-          http.use_ssl = true if uri.instance_of? URI::HTTPS
-          return http.request(req)
+          Sync do
+            DTIRequests.post(url, headers, body) do |response|
+              if response.status != 200
+                raise ConnectionError,
+                  "expected status 200 but got #{response.status} (#{url})"
+              end
+
+              response.read
+            end
+          end
         rescue Exception => e
           raise ConnectionError, e.message
         end