diff --git a/app/controllers/alt_styles_controller.rb b/app/controllers/alt_styles_controller.rb index d6a8fa03..33840614 100644 --- a/app/controllers/alt_styles_controller.rb +++ b/app/controllers/alt_styles_controller.rb @@ -8,6 +8,10 @@ class AltStylesController < ApplicationController @alt_styles = @alt_styles.merge(@species.alt_styles) end + # We're going to link to the HTML5 image URL, so make sure we have all the + # manifests ready! + SwfAsset.preload_manifests @alt_styles.map(&:swf_assets).flatten + respond_to do |format| format.html { render } format.json { diff --git a/app/models/swf_asset.rb b/app/models/swf_asset.rb index 3b6ecae9..2a56c5c3 100644 --- a/app/models/swf_asset.rb +++ b/app/models/swf_asset.rb @@ -1,3 +1,6 @@ +require 'async' +require 'async/barrier' +require 'async/semaphore' require 'fileutils' require 'uri' @@ -117,6 +120,10 @@ class SwfAsset < ApplicationRecord NeopetsMediaArchive.load_json(manifest_url) end + def preload_manifest + NeopetsMediaArchive.preload_file(manifest_url) + end + MANIFEST_BASE_URL = Addressable::URI.parse("https://images.neopets.com") def manifest_asset_urls return {} if manifest_url.nil? @@ -229,6 +236,31 @@ class SwfAsset < ApplicationRecord )) end + # Given a list of SWF assets, ensure all of their manifests are loaded, with + # fast concurrent execution! + def self.preload_manifests(swf_assets) + # Blocks all tasks beneath it. + barrier = Async::Barrier.new + + Sync do + # Only allow 10 manifests to be loaded at a time. + semaphore = Async::Semaphore.new(10, parent: barrier) + + # Load all the manifests in async tasks. This will load them 10 at a time + # rather than all at once (because of the semaphore), and the + # NeopetsMediaArchive will share a pool of persistent connections for + # them. + swf_assets.map do |swf_asset| + semaphore.async { swf_asset.preload_manifest } + end + + # Wait until all tasks are done. + barrier.wait + ensure + barrier.stop # If something goes wrong, clean up all tasks. + end + end + before_save do # If an asset body ID changes, that means more than one body ID has been # linked to it, meaning that it's probably wearable by all bodies. diff --git a/app/services/neopets_media_archive.rb b/app/services/neopets_media_archive.rb index e4350947..1b3ff17d 100644 --- a/app/services/neopets_media_archive.rb +++ b/app/services/neopets_media_archive.rb @@ -1,5 +1,5 @@ require "addressable/uri" -require "httparty" +require "async/http/internet/instance" require "json" # The Neopets Media Archive is a service that mirrors images.neopets.com files @@ -11,8 +11,9 @@ require "json" # long-term archive, not dependent on their services having 100% uptime in # order for us to operate. We never discard old files, we just keep going! module NeopetsMediaArchive - include HTTParty - base_uri "https://images.neopets.com/" + # Share a pool of persistent connections, rather than reconnecting on + # each request. (This library does that automatically!) + INTERNET = Async::HTTP::Internet.instance ROOT_PATH = Pathname.new(Rails.configuration.neopets_media_archive_root) @@ -46,9 +47,8 @@ module NeopetsMediaArchive end # Download the file from the origin, then save a copy for next time. - response = load_file_from_origin(uri) + content = load_file_from_origin(uri) info "Loaded source file from origin: #{uri}" - content = response.body local_path.dirname.mkpath File.write(local_path, content) info "Wrote source file to filesystem: #{local_path}" @@ -71,13 +71,20 @@ module NeopetsMediaArchive "https://images.neopets.com, but got #{uri}" end - response = get(uri) - if response.code == 404 - raise NotFound, "origin server returned 404: #{uri}" - elsif response.code != 200 - raise "expected status 200 but got #{response.code} (#{uri})" + # By running this request in a `Sync` block, we make this method look + # synchronous to the caller—but if run in the context of an async task, it + # will pause execution and move onto other work until the request is done. + # We use this in the `swf_assets:manifests:load` task to perform many + # requests in parallel! + Sync do + response = INTERNET.get(uri) + if response.status == 404 + raise NotFound, "origin server returned 404: #{uri}" + elsif response.status != 200 + raise "expected status 200 but got #{response.status} (#{uri})" + end + response.body.read end - response end def self.path_within_archive(uri)