Create swf_assets:manifests:load task to save all manifest files

Doing that sweet, sweet backfill!! It's not exactly *fast*, since
there's about 570k records to work through, but it's pretty good all
things considered! Thanks, surprisingly-reusable async code!

app/controllers/alt_styles_controller.rb

@@ -8,6 +8,10 @@ class AltStylesController < ApplicationController
 			@alt_styles = @alt_styles.merge(@species.alt_styles)
 		end
 
+		# We're going to link to the HTML5 image URL, so make sure we have all the
+		# manifests ready!
+		SwfAsset.preload_manifests @alt_styles.map(&:swf_assets).flatten
+
 		respond_to do |format|
 			format.html { render }
 			format.json {

app/models/swf_asset.rb

@@ -1,3 +1,6 @@
+require 'async'
+require 'async/barrier'
+require 'async/semaphore'
 require 'fileutils'
 require 'uri'
 
@@ -113,31 +116,39 @@ class SwfAsset < ApplicationRecord
     }
   end
 
-  MANIFEST_PATTERN = %r{^https://images.neopets.com/(?<prefix>.+)/(?<id>[0-9]+)(?<hash_part>_[^/]+)?/manifest\.json}
+  def manifest
+    raise "manifest_url is blank" if manifest_url.blank?
+    NeopetsMediaArchive.load_json(manifest_url)
+  end
+
+  def preload_manifest
+    raise "manifest_url is blank" if manifest_url.blank?
+    NeopetsMediaArchive.preload_file(manifest_url)
+  end
+
+  MANIFEST_BASE_URL = Addressable::URI.parse("https://images.neopets.com")
+  def manifest_asset_urls
+    return {} if manifest_url.nil?
+
+    begin
+      # Organize the asset URLs by file extension, grab the ones we want, and
+      # convert them from paths to full URLs.
+      manifest["cpmanifest"]["assets"][0]["asset_data"].
+        to_h { |a| [a["file_ext"].to_sym, a] }.
+        slice(:png, :svg, :js)
+        .transform_values { |a| (MANIFEST_BASE_URL + a["url"]).to_s }
+    rescue StandardError => error
+      Rails.logger.error "Could not read URLs from manifest: #{error.full_message}"
+      return {}
+    end
+  end
+
   def html5_image_url
-    return nil if manifest_url.nil?
-
-    # HACK: Just assuming all of these were well-formed by the same process,
-    # and infer the image URL from the manifest URL! But strictly speaking we
-    # should be reading the manifest to check!
-    match = manifest_url.match(MANIFEST_PATTERN)
-    return nil if match.nil?
-    
-    "https://images.neopets.com/#{match[:prefix]}/" +
-      "#{match[:id]}#{match[:hash_part]}/#{match[:id]}.png"
+    manifest_asset_urls[:png]
   end
 
   def html5_svg_url
-    return nil if manifest_url.nil?
-
-    # HACK: Just assuming all of these were well-formed by the same process,
-    # and infer the image URL from the manifest URL! But strictly speaking we
-    # should be reading the manifest to check!
-    match = manifest_url.match(MANIFEST_PATTERN)
-    return nil if match.nil?
-    
-    "https://images.neopets.com/#{match[:prefix]}/" +
-      "#{match[:id]}#{match[:hash_part]}/#{match[:id]}.svg"
+    manifest_asset_urls[:svg]
   end
 
   def known_glitches
@@ -227,6 +238,38 @@ class SwfAsset < ApplicationRecord
     ))
   end
 
+  # Given a list of SWF assets, ensure all of their manifests are loaded, with
+  # fast concurrent execution!
+  def self.preload_manifests(swf_assets)
+    # Blocks all tasks beneath it.
+    barrier = Async::Barrier.new
+
+    Sync do
+      # Only allow 10 manifests to be loaded at a time.
+      semaphore = Async::Semaphore.new(10, parent: barrier)
+
+      # Load all the manifests in async tasks. This will load them 10 at a time
+      # rather than all at once (because of the semaphore), and the
+      # NeopetsMediaArchive will share a pool of persistent connections for
+      # them.
+      swf_assets.map do |swf_asset|
+        semaphore.async do
+          begin
+            swf_asset.preload_manifest
+          rescue StandardError => error
+            Rails.logger.error "Could not preload manifest for asset " + 
+              "#{swf_asset.id} (#{swf_asset.manifest_url}): #{error.message}"
+          end
+        end
+      end
+
+      # Wait until all tasks are done.
+      barrier.wait
+    ensure
+      barrier.stop # If something goes wrong, clean up all tasks.
+    end
+  end
+
   before_save do
     # If an asset body ID changes, that means more than one body ID has been
     # linked to it, meaning that it's probably wearable by all bodies.

app/services/neopets_media_archive.rb

@@ -0,0 +1,120 @@
+require "addressable/uri"
+require "async/http/internet/instance"
+require "json"
+
+# The Neopets Media Archive is a service that mirrors images.neopets.com files
+# locally. You can request a file from it, and we'll serve it from disk if we
+# have it, or request and save it if not.
+#
+# This is a bit different than a cache, because the intention is not just
+# optimization but that we *want* to be saving images.neopets.com as a
+# long-term archive, not dependent on their services having 100% uptime in
+# order for us to operate. We never discard old files, we just keep going!
+module NeopetsMediaArchive
+  # Share a pool of persistent connections, rather than reconnecting on
+  # each request. (This library does that automatically!)
+  INTERNET = Async::HTTP::Internet.instance
+
+  ROOT_PATH = Pathname.new(Rails.configuration.neopets_media_archive_root)
+
+  # Load the file from the given `images.neopets.com` URI, as JSON.
+  def self.load_json(uri)
+    JSON.parse(load_file(uri))
+  end
+
+  # Load the file from the given `images.neopets.com` URI.
+  def self.load_file(uri, return_content: true)
+    local_path = local_file_path(uri)
+
+    # Read the file locally if we have it.
+    if return_content
+      begin
+        content = File.read(local_path)
+        debug "Loaded source file from filesystem: #{local_path}"
+        return content
+      rescue Errno::ENOENT
+        # If it doesn't exist, that's fine: just move on and download it.
+      end
+    else
+      # When we don't need the content, "loading" the file is just ensuring
+      # it exists. If it doesn't, we'll move on and load it from source.
+      # (We use this when preloading files, to save the cost of reading files
+      # we're not ready to use yet.)
+      if File.exist?(local_path)
+        debug "Source file is already loaded, skipping: #{local_path}"
+        return
+      end
+    end
+
+    # Download the file from the origin, then save a copy for next time.
+    content = load_file_from_origin(uri)
+    info "Loaded source file from origin: #{uri}"
+    local_path.dirname.mkpath
+    File.write(local_path, content)
+    info "Wrote source file to filesystem: #{local_path}"
+    
+    return_content ? content : nil
+  end
+
+  # Load the file from the given `images.neopets.com` URI, but don't return its
+  # content. This can be faster in cases where the file's content isn't
+  # relevant to us, and we just want to ensure it exists.
+  def self.preload_file(uri)
+    load_file(uri, return_content: false)
+  end
+
+  # Load the file from the given `images.neopets.com` URI, directly from the
+  # source, without checking the local filesystem.
+  def self.load_file_from_origin(uri)
+    unless Addressable::URI.parse(uri).origin == "https://images.neopets.com"
+      raise ArgumentError, "NeopetsMediaArchive can only load from " +
+        "https://images.neopets.com, but got #{uri}"
+    end
+
+    # By running this request in a `Sync` block, we make this method look
+    # synchronous to the caller—but if run in the context of an async task, it
+    # will pause execution and move onto other work until the request is done.
+    # We use this in the `swf_assets:manifests:load` task to perform many
+    # requests in parallel!
+    Sync do
+      response = INTERNET.get(uri)
+      if response.status == 404
+        raise NotFound, "origin server returned 404: #{uri}"
+      elsif response.status != 200
+        raise "expected status 200 but got #{response.status} (#{uri})"
+      end
+      response.body.read
+    end
+  end
+
+  def self.path_within_archive(uri)
+    uri = Addressable::URI.parse(uri)
+    path = uri.host + uri.path
+
+    # We include the query string as part of the file path, which is a bit odd!
+    # But Neopets often uses this for cache-busting, so we do need a mechanism
+    # for knowing whether we're holding the right version of the file. We could
+    # also consider storing the file by just its normal path, but with some
+    # metadata to track versioning information (e.g. a sqlite db, or a metadata
+    # file in the same directory).
+    path += "?" + uri.query if !uri.query.nil? && !uri.query.empty?
+
+    path
+  end
+
+  def self.local_file_path(uri)
+    ROOT_PATH + path_within_archive(uri)
+  end
+
+  class NotFound < StandardError; end
+
+  private
+
+  def self.info(message)
+    Rails.logger.info "[NeopetsMediaArchive] #{message}"
+  end
+
+  def self.debug(message)
+    Rails.logger.debug "[NeopetsMediaArchive] #{message}"
+  end
+end

config/environments/development.rb

@@ -107,4 +107,9 @@ Rails.application.configure do
   # override this with the IMPRESS_2020_ORIGIN environment variable!)
   config.impress_2020_origin = ENV.fetch("IMPRESS_2020_ORIGIN",
     "http://localhost:4000")
+
+  # Save the Neopets Media Archive in the local `tmp` folder. (In production,
+  # we keep this in a long-term location instead!)
+  config.neopets_media_archive_root = Rails.root / "tmp" /
+    "neopets_media_archive" / "development"
 end

config/environments/production.rb

@@ -126,4 +126,8 @@ Rails.application.configure do
   # IMPRESS_2020_ORIGIN environment variable!)
   config.impress_2020_origin = ENV.fetch("IMPRESS_2020_ORIGIN",
     "https://impress-2020.openneo.net")
+
+  # Save the Neopets Media Archive in `/var/lib/neopets-media-archive`, a
+  # long-term storage location.
+  config.neopets_media_archive_root = "/var/lib/neopets-media-archive"
 end

config/environments/test.rb

@@ -66,4 +66,9 @@ Rails.application.configure do
   # override this with the IMPRESS_2020_ORIGIN environment variable!)
   config.impress_2020_origin = ENV.fetch("IMPRESS_2020_ORIGIN",
     "http://localhost:4000")
+
+  # Save the Neopets Media Archive in the local `tmp` folder. (In production,
+  # we keep this in a long-term location instead!)
+  config.neopets_media_archive_root = Rails.root / "tmp" /
+    "neopets_media_archive" / "test"
 end

deploy/setup.yml

@@ -402,6 +402,14 @@
         password: "{{ mysql_user_password_2020 }}"
         priv: "openneo_impress.*:ALL,openneo_id.*:ALL"
 
+    - name: Create the Neopets Media Archive data directory
+      file:
+        path: /var/lib/neopets-media-archive
+        owner: impress
+        group: impress
+        mode: "755"
+        state: directory
+
   handlers:
     - name: Reload nginx
       systemd:

lib/tasks/swf_assets.rake

@@ -35,15 +35,43 @@ namespace :swf_assets do
 		end
 	end
 
+	namespace :manifests do
+		desc "Save all known manifests to the Neopets Media Archive"
+		task load: [:environment] do
+			# Log errors to STDOUT, but we don't need the info messages about
+			# successful saves.
+			Rails.logger = Logger.new(STDOUT, level: :error)
+
+			# Find all the manifests with known URLs. (We don't have a database
+			# filter for "do we already have the manifest downloaded", but that's
+			# okay, the preload method will quickly check that for us!)
+			swf_assets = SwfAsset.where.not(manifest_url: nil)
+			total_count = swf_assets.count
+			puts "Found #{total_count} assets with manifests"
+
+			# For each batch of 1000 assets, load their manifests concurrently.
+			# Wrap everything in a top-level sync, so keyboard interrupts will
+			# propagate all the way up to here, instead of just cancelling the
+			# current batch.
+			Sync do
+				saved_count = 0
+				swf_assets.find_in_batches(batch_size: 1000) do |swf_assets|
+					SwfAsset.preload_manifests(swf_assets)
+					saved_count += swf_assets.size
+					puts "Loaded #{saved_count} of #{total_count} manifests"
+				end
+			end
+		end
+
 		desc "Backfill manifest_url for SwfAsset models"
-	task manifests: [:environment] do
+		task urls: [:environment] do
 			timeout = ENV.fetch("TIMEOUT", "5").to_i
 
 			assets = SwfAsset.where(manifest_url: nil)
 			count = assets.count
 			puts "Found #{count} assets without manifests"
 
-		Async do
+			Sync do
 				# Share a pool of persistent connections, rather than reconnecting on
 				# each request. (This library does that automatically!)
 				internet = Async::HTTP::Internet.instance
@@ -109,6 +137,7 @@ namespace :swf_assets do
 			end
 		end
 	end
+end
 
 SWF_URL_PATTERN = %r{^(?:https?:)?//images\.neopets\.com/cp/(bio|items)/swf/(.+?)_([a-z0-9]+)\.swf$}
 def infer_manifest_url(swf_url, internet)