Emi Matchu
1e99376449
I think the Rails query cache handled these anyway? But `SwfAsset` has a `before_save` hook that checks its zone's info, and `SwfAsset.preload_manifests` saves all the assets, on the assumption that saving is a no-op when the record didn't change anyway. And it basically is! But I figure that, now that I'm realizing hooks exist, simply not attempting to save unchanged records is probably a better representation of what we intend to do. So I'm fixing it like that! Another potential fix would be to preload the zones for these assets, but I think that confuses the intent too much; the method itself isn't using the zones, it's just a weird incidental thing that a save hook happens to use. (Would probably be better to refactor this old save hook into a different situation altogether, but that's for another time!)
352 lines
12 KiB
Ruby
352 lines
12 KiB
Ruby
require 'addressable/template'
|
|
require 'async'
|
|
require 'async/barrier'
|
|
require 'async/semaphore'
|
|
require 'fileutils'
|
|
require 'uri'
|
|
|
|
class SwfAsset < ApplicationRecord
|
|
# We use the `type` column to mean something other than what Rails means!
|
|
self.inheritance_column = nil
|
|
|
|
# Used in `item_is_body_specific?`. (TODO: Could we refactor this out?)
|
|
attr_accessor :item
|
|
|
|
belongs_to :zone
|
|
has_many :parent_swf_asset_relationships
|
|
has_one :contribution, :as => :contributed, :inverse_of => :contributed
|
|
has_many :parent_swf_asset_relationships
|
|
|
|
before_validation :normalize_manifest_url, if: :manifest_url?
|
|
|
|
delegate :depth, :to => :zone
|
|
|
|
scope :biology_assets, -> { where(:type => PetState::SwfAssetType) }
|
|
scope :object_assets, -> { where(:type => Item::SwfAssetType) }
|
|
|
|
CANVAS_MOVIE_IMAGE_URL_TEMPLATE = Addressable::Template.new(
|
|
Rails.configuration.impress_2020_origin +
|
|
"/api/assetImage{?libraryUrl,size}"
|
|
)
|
|
LEGACY_IMAGE_URL_TEMPLATE = Addressable::Template.new(
|
|
"https://aws.impress-asset-images.openneo.net/{type}" +
|
|
"/{id1}/{id2}/{id3}/{id}/{size}x{size}.png?v2-{time}"
|
|
)
|
|
|
|
def as_json(options={})
|
|
super({
|
|
only: [:id, :known_glitches],
|
|
methods: [:zone, :restricted_zones, :urls]
|
|
}.merge(options))
|
|
end
|
|
|
|
def urls
|
|
{
|
|
swf: url,
|
|
png: image_url,
|
|
svg: manifest_asset_urls[:svg],
|
|
canvas_library: manifest_asset_urls[:js],
|
|
manifest: manifest_url,
|
|
}
|
|
end
|
|
|
|
def manifest
|
|
@manifest ||= load_manifest
|
|
end
|
|
|
|
def preload_manifest(save_changes: true)
|
|
load_manifest(return_content: false, save_changes:)
|
|
end
|
|
|
|
def load_manifest(return_content: true, save_changes: true)
|
|
return nil if manifest_url.blank?
|
|
|
|
# If we recently tried loading the manifest and got a 4xx HTTP status code
|
|
# (e.g. a 404, there's a surprising amount of these!), don't try again. But
|
|
# after enough time passes, if this is called again, we will!
|
|
#
|
|
# (We always retry 5xx errors, on the assumption that they probably
|
|
# represent intermittent failures, whereas 4xx errors are not likely to
|
|
# succeed just by retrying.)
|
|
if manifest_loaded_at.present?
|
|
last_try_was_4xx =(400...500).include?(manifest_status_code)
|
|
last_try_was_recent = (Time.now - manifest_loaded_at) <= 1.day
|
|
if last_try_was_4xx and last_try_was_recent
|
|
Rails.logger.debug "Skipping loading manifest for asset #{id}: " +
|
|
"last try was status #{manifest_status_code} at #{manifest_loaded_at}"
|
|
return nil
|
|
end
|
|
end
|
|
|
|
# Try loading the manifest. If we fail, record that we failed and return.
|
|
begin
|
|
Sync do |task|
|
|
task.with_timeout(5) do
|
|
NeopetsMediaArchive.load_file(manifest_url, return_content:)
|
|
end
|
|
end => {content:, source:}
|
|
rescue Async::TimeoutError
|
|
# If the request times out, record nothing and return nothing! We'll try
|
|
# again sometime, on the assumption that this is intermittent.
|
|
Rails.logger.warn("Timed out loading manifest for asset #{id}")
|
|
return nil
|
|
rescue NeopetsMediaArchive::ResponseNotOK => error
|
|
Rails.logger.warn "Failed to load manifest for asset #{id}: " +
|
|
error.message
|
|
self.manifest_loaded_at = Time.now
|
|
self.manifest_status_code = error.status
|
|
save! if save_changes
|
|
return nil
|
|
end
|
|
|
|
# If this was a fresh load over the network (or for some reason we're
|
|
# missing the timestamp), record that we succeeded.
|
|
if source == "network" || manifest_loaded_at.blank?
|
|
self.manifest_loaded_at = Time.now
|
|
self.manifest_status_code = 200
|
|
save! if save_changes
|
|
end
|
|
|
|
return nil unless return_content # skip parsing if not needed!
|
|
|
|
# Parse the manifest as JSON, and return it!
|
|
begin
|
|
JSON.parse(content)
|
|
rescue JSON::ParserError => error
|
|
Rails.logger.warn "Failed to parse manifest for asset #{id}: " +
|
|
error.message
|
|
return nil
|
|
end
|
|
end
|
|
|
|
MANIFEST_BASE_URL = Addressable::URI.parse("https://images.neopets.com")
|
|
def manifest_asset_urls
|
|
return {} unless manifest.present?
|
|
|
|
begin
|
|
# Organize the asset URLs by file extension, convert them from paths to
|
|
# full URLs, and grab the ones we want.
|
|
assets_by_ext = manifest["cpmanifest"]["assets"][0]["asset_data"].
|
|
group_by { |a| a["file_ext"].to_sym }.
|
|
transform_values do |assets|
|
|
assets.map { |a| (MANIFEST_BASE_URL + a["url"]).to_s }
|
|
end
|
|
|
|
if assets_by_ext[:js].present?
|
|
# If a JS asset is present, assume any other assets are supporting
|
|
# assets, and skip them. (e.g. if there's a PNG, it's likely to be an
|
|
# "atlas" file used in the animation, rather than a thumbnail.)
|
|
#
|
|
# NOTE: We take the last one, because sometimes there are multiple JS
|
|
# assets in the same manifest, and earlier ones are broken and later
|
|
# ones are fixed. I don't know the logic exactly, but that's what we've
|
|
# seen!
|
|
{ js: assets_by_ext[:js].last }
|
|
else
|
|
# Otherwise, return the first PNG and the first SVG. (Unlike the JS
|
|
# case, it's important to choose the *first* PNG, because sometimes
|
|
# reference art is included in the manifest, like with the Stealthy
|
|
# Eyrie Shirt's asset 304486_b28cae0d76.)
|
|
{
|
|
png: assets_by_ext.fetch(:png, []).first,
|
|
svg: assets_by_ext.fetch(:svg, []).first,
|
|
}
|
|
end
|
|
rescue StandardError => error
|
|
Rails.logger.error "Could not read URLs from manifest: #{error.full_message}"
|
|
return {}
|
|
end
|
|
end
|
|
|
|
def image_url
|
|
# Use the PNG image from the manifest, if one exists.
|
|
return manifest_asset_urls[:png] if manifest_asset_urls[:png].present?
|
|
|
|
# Or, if this is a canvas movie, let Impress 2020 generate a PNG for us.
|
|
return canvas_movie_image_url if manifest_asset_urls[:js].present?
|
|
|
|
# Otherwise, if we don't have the manifest or it doesn't have the files we
|
|
# need, fall back to the Classic DTI image storage, which was generated
|
|
# from the SWFs via an old version of gnash (or sometimes manually
|
|
# overridden). It's less accurate, but well-tested to generally work okay,
|
|
# and it's the only image we have for assets not yet converted to HTML5.
|
|
#
|
|
# NOTE: We've stopped generating these images for new assets! This is
|
|
# mainly for old assets not yet converted to HTML5.
|
|
#
|
|
# NOTE: If you're modeling from a fresh development database, `has_image?`
|
|
# might be false even though we *do* have a saved copy of the image
|
|
# available in production. But if you're using the public modeling
|
|
# data exported from production, then this check should be fine!
|
|
#
|
|
# TODO: Rename `has_image?` to `has_legacy_image?`.
|
|
return legacy_image_url if has_image?
|
|
|
|
# Otherwise, there's no image URL.
|
|
nil
|
|
end
|
|
|
|
def canvas_movie_image_url
|
|
return nil unless manifest_asset_urls[:js]
|
|
|
|
CANVAS_MOVIE_IMAGE_URL_TEMPLATE.expand(
|
|
libraryUrl: manifest_asset_urls[:js],
|
|
size: 600,
|
|
).to_s
|
|
end
|
|
|
|
def legacy_image_url
|
|
return nil unless has_image?
|
|
|
|
padded_id = remote_id.to_s.rjust(12, "0")
|
|
LEGACY_IMAGE_URL_TEMPLATE.expand(
|
|
type: type,
|
|
id1: padded_id[0...3],
|
|
id2: padded_id[3...6],
|
|
id3: padded_id[6...9],
|
|
id: remote_id,
|
|
size: "600",
|
|
time: converted_at.to_i,
|
|
).to_s
|
|
end
|
|
|
|
def known_glitches
|
|
self[:known_glitches].split(',')
|
|
end
|
|
|
|
def known_glitches=(new_known_glitches)
|
|
if new_known_glitches.is_a? Array
|
|
new_known_glitches = new_known_glitches.join(',')
|
|
end
|
|
self[:known_glitches] = new_known_glitches
|
|
end
|
|
|
|
def restricted_zone_ids
|
|
[].tap do |ids|
|
|
zones_restrict.chars.each_with_index do |bit, index|
|
|
ids << index + 1 if bit == "1"
|
|
end
|
|
end
|
|
end
|
|
|
|
def restricted_zones
|
|
Zone.where(id: restricted_zone_ids)
|
|
end
|
|
|
|
def body_specific?
|
|
self.zone.type_id < 3 || item_is_body_specific?
|
|
end
|
|
|
|
def item_is_body_specific?
|
|
# Get items that we're already bound to in the database, and
|
|
# also the one passed to us from the current modeling operation,
|
|
# if any.
|
|
#
|
|
# NOTE: I know this has perf impact... it would be better for
|
|
# modeling to preload this probably? But oh well!
|
|
items = parent_swf_asset_relationships.includes(:parent).where(parent_type: "Item").map { |r| r.parent }
|
|
items << item if item
|
|
|
|
# Return whether any of them is known to be body-specific.
|
|
# This ensures that we always respect the explicitly_body_specific flag!
|
|
return items.any? { |i| i.body_specific? }
|
|
end
|
|
|
|
def origin_pet_type=(pet_type)
|
|
self.body_id = pet_type.body_id
|
|
end
|
|
|
|
def origin_biology_data=(data)
|
|
Rails.logger.debug("my biology data is: #{data.inspect}")
|
|
self.type = 'biology'
|
|
self.zone_id = data[:zone_id].to_i
|
|
self.url = data[:asset_url]
|
|
self.zones_restrict = data[:zones_restrict]
|
|
self.manifest_url = data[:manifest]
|
|
end
|
|
|
|
def origin_object_data=(data)
|
|
Rails.logger.debug("my object data is: #{data.inspect}")
|
|
self.type = 'object'
|
|
self.zone_id = data[:zone_id].to_i
|
|
self.url = data[:asset_url]
|
|
self.zones_restrict = ""
|
|
self.manifest_url = data[:manifest]
|
|
end
|
|
|
|
def normalize_manifest_url
|
|
parsed_manifest_url = Addressable::URI.parse(manifest_url)
|
|
parsed_manifest_url.scheme = "https"
|
|
self.manifest_url = parsed_manifest_url.to_s
|
|
end
|
|
|
|
# To manually change the body ID without triggering the usual change to 0,
|
|
# use this override method. (This is intended for use from the console.)
|
|
def override_body_id(new_body_id)
|
|
@body_id_overridden = true
|
|
self.body_id = new_body_id
|
|
end
|
|
|
|
def self.from_biology_data(body_id, data)
|
|
remote_id = data[:part_id].to_i
|
|
swf_asset = SwfAsset.find_or_initialize_by type: 'biology',
|
|
remote_id: remote_id
|
|
swf_asset.body_id = body_id
|
|
swf_asset.origin_biology_data = data
|
|
swf_asset
|
|
end
|
|
|
|
def self.from_wardrobe_link_params(ids)
|
|
where((
|
|
arel_table[:remote_id].in(ids[:biology]).and(arel_table[:type].eq('biology'))
|
|
).or(
|
|
arel_table[:remote_id].in(ids[:object]).and(arel_table[:type].eq('object'))
|
|
))
|
|
end
|
|
|
|
# Given a list of SWF assets, ensure all of their manifests are loaded, with
|
|
# fast concurrent execution!
|
|
def self.preload_manifests(swf_assets)
|
|
# Blocks all tasks beneath it.
|
|
barrier = Async::Barrier.new
|
|
|
|
Sync do
|
|
# Only allow 10 manifests to be loaded at a time.
|
|
semaphore = Async::Semaphore.new(10, parent: barrier)
|
|
|
|
# Load all the manifests in async tasks. This will load them 10 at a time
|
|
# rather than all at once (because of the semaphore), and the
|
|
# NeopetsMediaArchive will share a pool of persistent connections for
|
|
# them.
|
|
swf_assets.map do |swf_asset|
|
|
semaphore.async do
|
|
begin
|
|
# Don't save changes in this big async situation; we'll do it all
|
|
# in one batch after, to avoid too much database concurrency!
|
|
swf_asset.preload_manifest(save_changes: false)
|
|
rescue StandardError => error
|
|
Rails.logger.error "Could not preload manifest for asset " +
|
|
"#{swf_asset.id} (#{swf_asset.manifest_url}): #{error.message}"
|
|
end
|
|
end
|
|
end
|
|
|
|
# Wait until all tasks are done.
|
|
barrier.wait
|
|
ensure
|
|
barrier.stop # If something goes wrong, clean up all tasks.
|
|
end
|
|
|
|
SwfAsset.transaction do
|
|
swf_assets.select(&:changed?).each(&:save!)
|
|
end
|
|
end
|
|
|
|
before_save do
|
|
# If an asset body ID changes, that means more than one body ID has been
|
|
# linked to it, meaning that it's probably wearable by all bodies.
|
|
self.body_id = 0 if !@body_id_overridden && (!self.body_specific? || (!self.new_record? && self.body_id_changed?))
|
|
end
|
|
|
|
class DownloadError < Exception;end
|
|
end
|