Add rails rainbow_pool:import task, to get clean image hashes for pets

Used to have something like this long ago, now here's the latest
version!

This task can't run autonomously, it needs the human user to provide a
neologin cookie value. So, no cron for us! But we're cleaning up *years*
of lil guys in one swoop now :3

Gemfile

@@ -4,7 +4,7 @@ ruby '3.3.4'
 gem 'rails', '~> 7.1', '>= 7.1.3.4'
 
 # The HTTP server running the Rails instance.
-gem 'falcon', '~> 0.43.0'
+gem 'falcon', '~> 0.48.0'
 
 # Our database is MySQL, in both development and production.
 gem 'mysql2', '~> 0.5.5'
@@ -61,8 +61,8 @@ gem "httparty", "~> 0.22.0"
 gem "addressable", "~> 2.8"
 
 # For advanced batching of many HTTP requests.
-gem "async", "~> 2.6", require: false
-gem "async-http", "~> 0.61.0", require: false
+gem "async", "~> 2.17", require: false
+gem "async-http", "~> 0.75.0", require: false
 gem "thread-local", "~> 1.1", require: false
 
 # For debugging.

Gemfile.lock

@@ -81,29 +81,30 @@ GEM
       public_suffix (>= 2.0.2, < 7.0)
     aes_key_wrap (1.1.0)
     ast (2.4.2)
-    async (2.16.1)
+    async (2.17.0)
       console (~> 1.26)
       fiber-annotation
       io-event (~> 1.6, >= 1.6.5)
-    async-container (0.16.13)
-      async
-      async-io
-    async-http (0.61.0)
-      async (>= 1.25)
-      async-io (>= 1.28)
-      async-pool (>= 0.2)
-      protocol-http (~> 0.25.0)
-      protocol-http1 (~> 0.16.0)
-      protocol-http2 (~> 0.15.0)
-      traces (>= 0.10.0)
+    async-container (0.18.3)
+      async (~> 2.10)
+    async-http (0.75.0)
+      async (>= 2.10.2)
+      async-pool (~> 0.7)
+      io-endpoint (~> 0.11)
+      io-stream (~> 0.4)
+      protocol-http (~> 0.30)
+      protocol-http1 (~> 0.20)
+      protocol-http2 (~> 0.18)
+      traces (>= 0.10)
     async-http-cache (0.4.4)
       async-http (~> 0.56)
-    async-io (1.43.2)
-      async
     async-pool (0.8.1)
       async (>= 1.25)
       metrics
       traces
+    async-service (0.12.0)
+      async
+      async-container (~> 0.16)
     attr_required (1.0.2)
     babel-source (5.8.35)
     babel-transpiler (0.7.0)
@@ -118,7 +119,6 @@ GEM
     bindex (0.8.1)
     bootsnap (1.18.4)
       msgpack (~> 1.2)
-    build-environment (1.13.0)
     builder (3.3.0)
     childprocess (5.1.0)
       logger (~> 1.5)
@@ -150,19 +150,19 @@ GEM
       activemodel
     erubi (1.13.0)
     execjs (2.9.1)
-    falcon (0.43.0)
+    falcon (0.48.0)
       async
-      async-container (~> 0.16.0)
-      async-http (~> 0.57)
-      async-http-cache (~> 0.4.0)
-      async-io (~> 1.22)
-      build-environment (~> 1.13)
+      async-container (~> 0.18)
+      async-http (~> 0.75)
+      async-http-cache (~> 0.4)
+      async-service (~> 0.10)
       bundler
       localhost (~> 1.1)
       openssl (~> 3.0)
-      process-metrics (~> 0.2.0)
-      protocol-rack (~> 0.1)
-      samovar (~> 2.1)
+      process-metrics (~> 0.2)
+      protocol-http (~> 0.31)
+      protocol-rack (~> 0.7)
+      samovar (~> 2.3)
     faraday (2.11.0)
       faraday-net_http (>= 2.0, < 3.4)
       logger
@@ -190,7 +190,9 @@ GEM
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     io-console (0.7.2)
+    io-endpoint (0.13.1)
     io-event (1.6.5)
+    io-stream (0.4.0)
     irb (1.14.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
@@ -280,18 +282,19 @@ GEM
     parser (3.3.4.2)
       ast (~> 2.4.1)
       racc
-    process-metrics (0.2.1)
+    process-metrics (0.3.0)
       console (~> 1.8)
+      json (~> 2)
       samovar (~> 2.1)
     protocol-hpack (1.5.0)
-    protocol-http (0.25.0)
-    protocol-http1 (0.16.1)
+    protocol-http (0.33.0)
+    protocol-http1 (0.22.0)
       protocol-http (~> 0.22)
-    protocol-http2 (0.15.1)
+    protocol-http2 (0.18.0)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.18)
-    protocol-rack (0.6.0)
-      protocol-http (~> 0.23)
+    protocol-rack (0.7.0)
+      protocol-http (~> 0.27)
       rack (>= 1.0)
     psych (5.1.2)
       stringio
@@ -495,13 +498,13 @@ PLATFORMS
 DEPENDENCIES
   RocketAMF!
   addressable (~> 2.8)
-  async (~> 2.6)
-  async-http (~> 0.61.0)
+  async (~> 2.17)
+  async-http (~> 0.75.0)
   bootsnap (~> 1.16)
   devise (~> 4.9, >= 4.9.2)
   devise-encryptable (~> 0.2.0)
   dotenv-rails (~> 2.8, >= 2.8.1)
-  falcon (~> 0.43.0)
+  falcon (~> 0.48.0)
   haml (~> 6.1, >= 6.1.1)
   http_accept_language (~> 2.1, >= 2.1.1)
   httparty (~> 0.22.0)

app/assets/javascripts/outfit-viewer.js

@@ -112,9 +112,7 @@ class OutfitLayer extends HTMLElement {
 				this.#setStatus("error"),
 			);
 		} else {
-			throw new Error(
-				`<outfit-layer> must contain an <img> or <iframe> tag`,
-			);
+			console.warn(`<outfit-layer> contained no image or iframe: `, this);
 		}
 	}
 

app/services/nc_mall.rb

@@ -25,39 +25,40 @@ module NCMall
 	ROOT_DOCUMENT_URL = "https://ncmall.neopets.com/mall/shop.phtml"
 	PAGE_LINK_PATTERN = /load_items_pane\(['"](.+?)['"], ([0-9]+)\).+?>(.+?)</
 	def self.load_page_links
-		Sync do
-			response = INTERNET.get(ROOT_DOCUMENT_URL, [
+		html = Sync do
+			INTERNET.get(ROOT_DOCUMENT_URL, [
 				["User-Agent", Rails.configuration.user_agent_for_neopets],
-			])
+			]) do |response|
+				if response.status != 200
+					raise ResponseNotOK.new(response.status),
+						"expected status 200 but got #{response.status} (#{url})"
+				end
 
-			if response.status != 200
-				raise ResponseNotOK.new(response.status),
-					"expected status 200 but got #{response.status} (#{url})"
+				response.read
 			end
-
-			# Extract `load_items_pane` calls from the root document's HTML. (We use
-			# a very simplified regex, rather than actually parsing the full HTML!)
-			html = response.read
-			html.scan(PAGE_LINK_PATTERN).
-				map { |type, cat, label| {type:, cat:, label:} }.
-				uniq
 		end
+
+		# Extract `load_items_pane` calls from the root document's HTML. (We use
+		# a very simplified regex, rather than actually parsing the full HTML!)
+		html.scan(PAGE_LINK_PATTERN).
+			map { |type, cat, label| {type:, cat:, label:} }.
+			uniq
 	end
 
 	private
 
 	def self.load_page_by_url(url)
 		Sync do
-			response = INTERNET.get(url, [
+			INTERNET.get(url, [
 				["User-Agent", Rails.configuration.user_agent_for_neopets],
-			])
+			]) do |response|
+				if response.status != 200
+					raise ResponseNotOK.new(response.status),
+						"expected status 200 but got #{response.status} (#{url})"
+				end
 
-			if response.status != 200
-				raise ResponseNotOK.new(response.status),
-					"expected status 200 but got #{response.status} (#{url})"
+				parse_nc_page response.read
 			end
-
-			parse_nc_page response.read
 		end
 	end
 

app/services/neopass.rb

@@ -31,20 +31,20 @@ module NeoPass
 
 	LINKAGE_URL = "https://oidc.neopets.com/linkage/all"
 	def self.load_linkages(access_token)
-		response = Sync do
-			response = INTERNET.get(LINKAGE_URL, [
+		linkages_str = Sync do
+			INTERNET.get(LINKAGE_URL, [
 				["User-Agent", Rails.configuration.user_agent_for_neopets],
 				["Authorization", "Bearer #{access_token}"],
-			])
-		end
+			]) do |response|
+				if response.status != 200
+					raise ResponseNotOK.new(response.status),
+						"expected status 200 but got #{response.status} (#{LINKAGE_URL})"
+				end
 
-		if response.status != 200
-			raise ResponseNotOK.new(response.status),
-				"expected status 200 but got #{response.status} (#{LINKAGE_URL})"
+				response.read
+			end
 		end
 
-		linkages_str = response.body.read
-
 		begin
 			linkages = JSON.parse(linkages_str)
 		rescue JSON::ParserError

app/services/neopets_media_archive.rb

@@ -72,14 +72,15 @@ module NeopetsMediaArchive
     # We use this in the `swf_assets:manifests:load` task to perform many
     # requests in parallel!
     Sync do
-      response = INTERNET.get(uri, [
+      INTERNET.get(uri, [
         ["User-Agent", Rails.configuration.user_agent_for_neopets],
-      ])
-      if response.status != 200
-        raise ResponseNotOK.new(response.status),
-          "expected status 200 but got #{response.status} (#{uri})"
+      ]) do |response|
+        if response.status != 200
+          raise ResponseNotOK.new(response.status),
+            "expected status 200 but got #{response.status} (#{uri})"
+        end
+        response.read
       end
-      response.body.read
     end
   end
 

app/views/items/_outfit_viewer.html.haml

@@ -20,5 +20,7 @@
     }
       - if swf_asset.canvas_movie?
         %iframe{src: swf_asset_path(swf_asset, playing: outfit_viewer_is_playing ? true : nil)}
-      - else
+      - elsif swf_asset.image_url.present?
         = image_tag swf_asset.image_url, alt: ""
+      - else
+        / No movie or image available for SWF asset: #{swf_asset.url}

lib/tasks/rainbow_pool.rake

@@ -0,0 +1,102 @@
+require "addressable/template"
+require "async/http/internet/instance"
+
+namespace :rainbow_pool do
+	desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes"
+	task :import => :environment do
+		neologin = STDIN.getpass("Neologin cookie: ")
+
+		all_pet_types = PetType.all.to_a
+		all_pet_types_by_species_id_and_color_id = all_pet_types.
+			to_h { |pt| [[pt.species_id, pt.color_id], pt] }
+		all_colors_by_name = Color.all.to_h { |c| [c.human_name.downcase, c] }
+
+		# TODO: Do these in parallel? I set up the HTTP requests to be able to
+		#       handle it, and just didn't set up the rest of the code for it, lol
+		Species.order(:name).each do |species|
+			begin
+				hashes_by_color_name = RainbowPool.load_hashes_for_species(
+					species.id, neologin)
+			rescue => error
+				puts "Failed to load #{species.name} page, skipping: #{error.message}"
+				next
+			end
+
+			changed_pet_types = []
+
+			hashes_by_color_name.each do |color_name, image_hash|
+				color = all_colors_by_name[color_name.downcase]
+				if color.nil?
+					puts "Skipping unrecognized color name: #{color_name}"
+					next
+				end
+
+				pet_type = all_pet_types_by_species_id_and_color_id[
+					[species.id, color.id]]
+				if pet_type.nil?
+					puts "Skipping unrecognized pet type: " +
+							 "#{color_name} #{species.human_name}"
+					next
+				end
+
+				if pet_type.basic_image_hash.nil?
+					puts "Found new image hash: #{image_hash} (#{pet_type.human_name})"
+					pet_type.basic_image_hash = image_hash
+					changed_pet_types << pet_type
+				elsif pet_type.basic_image_hash != image_hash
+					puts "Updating image hash: #{image_hash} ({#{pet_type.human_name})"
+					pet_type.basic_image_hash = image_hash
+					changed_pet_types << pet_type
+				else
+					# No need to do anything with image hashes that match!
+				end
+			end
+
+			PetType.transaction { changed_pet_types.each(&:save!) }
+			puts "Saved #{changed_pet_types.size} image hashes for " +
+			     "#{species.human_name}"
+		end
+	end
+end
+
+module RainbowPool
+	# Share a pool of persistent connections, rather than reconnecting on
+	# each request. (This library does that automatically!)
+	INTERNET = Async::HTTP::Internet.instance
+
+	class << self
+		SPECIES_PAGE_URL_TEMPLATE = Addressable::Template.new(
+			"https://www.neopets.com/pool/all_pb.phtml{?f_species_id}"
+		)
+		def load_hashes_for_species(species_id, neologin)
+			Sync do
+				url = SPECIES_PAGE_URL_TEMPLATE.expand(f_species_id: species_id)
+				INTERNET.get(url, [
+					["User-Agent", Rails.configuration.user_agent_for_neopets],
+					["Cookie", "neologin=#{neologin}"],
+				]) do |response|
+					if response.status != 200
+						raise "expected status 200 but got #{response.status} (#{url})"
+					end
+
+					parse_hashes_from_page response.read
+				end
+			end
+		end
+
+		private
+
+		IMAGE_HASH_PATTERN = %r{
+			set_pet_img\(
+				'https?://pets\.neopets\.com/cp/(?<hash>[0-9a-z]+)/[0-9]+/[0-9]+\.png',
+				\s*
+				'(?<color_name>.+?)'
+			\)
+		}x
+		def parse_hashes_from_page(html)
+			html.scan(IMAGE_HASH_PATTERN).to_h do |(image_hash, color_name)|
+				[color_name, image_hash]
+			end
+		end
+	end
+end