First draft of auto-modeling

Run `rails items:auto_model`! We'll set this up on a cron if we continue to be satisfied with the results.

app/models/pet/auto_modeling.rb

@@ -0,0 +1,63 @@
+# Pet::AutoModeling provides utilities for automatically modeling items on pet
+# bodies using the NC Mall preview API. This allows us to fetch appearance data
+# for items without needing a real pet of that type.
+#
+# The workflow:
+# 1. Generate a combined "SCI" (Species/Color Image hash) using NC Mall's
+#    getPetData endpoint, which combines a pet type with items.
+# 2. Fetch the viewer data for that combined SCI using the CustomPets API.
+# 3. Process the viewer data to create SwfAsset records.
+module Pet::AutoModeling
+  extend self
+
+  # Model an item on a specific body ID. This fetches the appearance data from
+  # Neopets and creates/updates the SwfAsset records.
+  #
+  # @param item [Item] The item to model
+  # @param body_id [Integer] The body ID to model on
+  # @return [Symbol] Result status:
+  #   - :modeled - Successfully created SwfAsset records
+  #   - :not_compatible - Item is explicitly not compatible with this body
+  # @raise [NoPetTypeForBody] If no PetType exists for this body_id
+  # @raise [Neopets::NCMall::ResponseNotOK] On HTTP errors (transient for 5xx)
+  # @raise [Neopets::NCMall::UnexpectedResponseFormat] On invalid response
+  # @raise [Neopets::CustomPets::DownloadError] On AMF protocol errors
+  def model_item_on_body(item, body_id)
+    # Find a pet type with this body ID to use as a base
+    pet_type = PetType.find_by(body_id: body_id)
+    raise NoPetTypeForBody.new(body_id) if pet_type.nil?
+
+    # Fetch the viewer data for this item on this pet type
+    new_image_hash = Neopets::NCMall.fetch_pet_data_sci(pet_type.image_hash, [item.id])
+    viewer_data = Neopets::CustomPets.fetch_viewer_data("@#{new_image_hash}")
+
+    # If the item wasn't in the response, it's not compatible.
+    object_info = viewer_data[:object_info_registry]&.to_h&.[](item.id.to_s)
+    return :not_compatible if object_info.nil?
+
+    # Process the modeling data using the existing infrastructure
+    snapshot = Pet::ModelingSnapshot.new(viewer_data)
+
+    # Save the pet type (may update image hash, etc.)
+    snapshot.pet_type.save!
+
+    # Get the items from the snapshot and process them
+    modeled_items = snapshot.items
+    modeled_item = modeled_items.find { |i| i.id == item.id }
+
+    if modeled_item
+      modeled_item.save!
+      modeled_item.handle_assets!
+    end
+
+    :modeled
+  end
+
+  class NoPetTypeForBody < StandardError
+    attr_reader :body_id
+    def initialize(body_id)
+      @body_id = body_id
+      super("No PetType found for body_id=#{body_id}")
+    end
+  end
+end

app/services/neopets/nc_mall.rb

@@ -126,7 +126,7 @@ module Neopets::NCMall
 	# Use the returned hash with Neopets::CustomPets.fetch_viewer_data("@#{newsci}")
 	# to get the full appearance data.
 	PET_DATA_URL = "https://ncmall.neopets.com/mall/ajax/petview/getPetData.php"
-	def self.fetch_pet_data(pet_sci, item_ids = [])
+	def self.fetch_pet_data_sci(pet_sci, item_ids = [])
 		Sync do
 			params = {"selPetsci" => pet_sci}
 			item_ids.each { |id| params["itemsList[]"] = id.to_s }
@@ -153,7 +153,7 @@ module Neopets::NCMall
 						"missing or invalid field newsci in pet data response"
 				end
 
-				{newsci: data["newsci"]}
+				data["newsci"]
 			end
 		end
 	end

docs/customization-architecture.md

@@ -249,6 +249,28 @@ This crowdsourced approach is why DTI is "self-sustaining" - users passively con
 
 See `app/models/pet/modeling_snapshot.rb` for the full implementation.
 
+### Potential Upgrade: Auto-Modeling
+
+We are currently not making full use of a recently-discovered Neopets feature: **we no longer need a real pet to model
+the item**. There's an NC Mall feature that supports previews of *any* item, not just those sold in the Mall.
+
+See the `pets:load` task for implementation details.
+
+We still need users to show us new items in the first place, to learn their item IDs and what body types they might
+fit. But once we have that, we can proactively attempt to model the pet on all relevant body types.
+
+Let's pursue this in two steps:
+
+1. [x] Create a backfill Rake task to attempt to load any models we suspect we need, based on the same logic as the
+   `-is:modeled` item search filter. 
+   - Consider having this task auto-update the `modeling_status_hint` field on the item, if we've demonstrated that
+     the item is almost certainly completely modeled, despite our heuristic indicating it is not. This will keep the
+     `is:modeled` filter clean and approximately empty.
+   - As part of this, let's refactor the logic out of `pets:load`, to more simply construct an "image hash" ("sci")
+     from a pet type + items combination.
+2. [ ] Set this as a cron job to run very frequently, to quickly load in new items.
+   - If we're able to reliably keep `is:modeled` basically empty, this could even be safe to run every, say, 2–5min.
+
 ### Cached Fields
 
 To avoid expensive queries, several models cache computed data:

lib/tasks/items.rake

@@ -9,4 +9,90 @@ namespace :items do
 			end
 		end
 	end
+
+	desc "Auto-model items on missing body types using NC Mall preview API"
+	task :auto_model, [:limit] => :environment do |task, args|
+		limit = (args[:limit] || 100).to_i
+		dry_run = ENV["DRY_RUN"] == "1"
+		auto_hint = ENV["AUTO_HINT"] != "0"
+
+		puts "Auto-modeling up to #{limit} items#{dry_run ? ' (DRY RUN)' : ''}..."
+		puts "Auto-hint: #{auto_hint ? 'enabled' : 'disabled'}"
+		puts
+
+		# Find items that need modeling, newest first
+		items = Item.is_not_modeled.order(created_at: :desc).limit(limit)
+		puts "Found #{items.count} items to process"
+		puts
+
+		items.each_with_index do |item, index|
+			puts "[#{index + 1}/#{items.count}] Item ##{item.id}: #{item.name}"
+
+			missing_body_ids = item.predicted_missing_body_ids
+			if missing_body_ids.empty?
+				puts "  ⚠️  No missing body IDs (item may already be fully modeled)"
+				puts
+				next
+			end
+
+			puts "  Missing #{missing_body_ids.size} body IDs: #{missing_body_ids.join(', ')}"
+
+			# Track results for this item
+			results = {modeled: 0, not_compatible: 0, not_found: 0}
+			had_transient_error = false
+
+			missing_body_ids.each do |body_id|
+				if dry_run
+					puts "    Body #{body_id}: [DRY RUN] would attempt modeling"
+					next
+				end
+
+				begin
+					result = Pet::AutoModeling.model_item_on_body(item, body_id)
+					results[result] += 1
+
+					case result
+					when :modeled
+						puts "    Body #{body_id}: ✅ Modeled successfully"
+					when :not_compatible
+						puts "    Body #{body_id}: ❌ Not compatible (heuristic over-predicted)"
+					end
+				rescue Pet::AutoModeling::NoPetTypeForBody => e
+					puts "    Body #{body_id}: ⚠️  #{e.message}"
+				rescue Neopets::NCMall::ResponseNotOK => e
+					if e.status >= 500
+						puts "    Body #{body_id}: ⚠️  Server error (#{e.status}), will retry later"
+						had_transient_error = true
+					else
+						puts "    Body #{body_id}: ❌ HTTP error (#{e.status})"
+						Sentry.capture_exception(e)
+					end
+				rescue Neopets::NCMall::UnexpectedResponseFormat => e
+					puts "    Body #{body_id}: ❌ Unexpected response format: #{e.message}"
+					Sentry.capture_exception(e)
+				rescue Neopets::CustomPets::DownloadError => e
+					puts "    Body #{body_id}: ⚠️  AMF error: #{e.message}"
+					had_transient_error = true
+				end
+			end
+
+			unless dry_run
+				# Set hint if we've addressed all bodies without transient errors.
+				# That way, if the item is not compatible with some bodies, we'll stop
+				# trying to auto-model it.
+				if auto_hint && !had_transient_error
+					item.update!(modeling_status_hint: "done")
+					puts "  📋 Set modeling_status_hint = 'done'"
+				end
+			end
+
+			puts "  Summary: #{results[:modeled]} modeled, #{results[:not_compatible]} not compatible, #{results[:not_found]} not found"
+			puts
+
+			# Be nice to Neopets API
+			sleep 0.5 unless dry_run || index == items.count - 1
+		end
+
+		puts "Done!"
+	end
 end

lib/tasks/pets.rake

@@ -13,19 +13,16 @@ namespace :pets do
 			species_name = all_args[1]
 			item_ids = all_args[2..]
 
-			# Look up the PetType to get its image hash
+			# Look up the PetType to use for the preview
 			pet_type = PetType.matching_name(color_name, species_name).first!
-			pet_sci = pet_type.image_hash
 
-			# Fetch the new image hash for this pet+items combination
+			# Convert it to an image hash for direct lookup
-			response = Neopets::NCMall.fetch_pet_data(pet_sci, item_ids)
+			new_image_hash = Neopets::NCMall.fetch_pet_data_sci(pet_type.image_hash, item_ids)
-			new_sci = response[:newsci]
+			pet_name = '@' + new_image_hash
+			$stderr.puts "Loading pet #{pet_name}"
 
-			# Output the hash to stderr for debugging
+			# Load the image hash as if it were a pet
-			$stderr.puts "Generated image hash: #{new_sci}"
+			viewer_data = Neopets::CustomPets.fetch_viewer_data(pet_name)
-
-			# Load the full viewer data using the new image hash
-			viewer_data = Neopets::CustomPets.fetch_viewer_data("@#{new_sci}")
 		end
 
 		puts JSON.pretty_generate(viewer_data)

spec/models/pet/auto_modeling_spec.rb

@@ -0,0 +1,92 @@
+require_relative '../../rails_helper'
+require_relative '../../support/mocks/custom_pets'
+require_relative '../../support/mocks/nc_mall'
+
+RSpec.describe Pet::AutoModeling, type: :model do
+  fixtures :colors, :species, :zones
+
+  # Set up a Purple Chia pet type (body_id 212) for testing
+  let!(:pet_type) do
+    PetType.create!(
+      species_id: Species.find_by_name!("chia").id,
+      color_id: Color.find_by_name!("purple").id,
+      body_id: 212,
+      image_hash: "purpchia"
+    )
+  end
+
+  # A known compatible item for testing (exists in mock data)
+  let(:compatible_item) do
+    Item.create!(
+      id: 71706,
+      name: "On the Roof Background",
+      description: "Who is that on the roof?! Could it be...?",
+      thumbnail_url: "https://images.neopets.com/items/gif_roof_onthe_fg.gif",
+      rarity: "Special",
+      rarity_index: 101,
+      price: 0,
+      zones_restrict: "0000000000000000000000000000000000000000000000000000"
+    )
+  end
+
+  describe ".model_item_on_body" do
+    context "when item is compatible with the body" do
+      let(:item) { compatible_item }
+
+      it "returns :modeled" do
+        result = Pet::AutoModeling.model_item_on_body(item, 212)
+        expect(result).to eq :modeled
+      end
+
+      it "creates SwfAsset records for the item" do
+        expect {
+          Pet::AutoModeling.model_item_on_body(item, 212)
+        }.to change { SwfAsset.where(type: "object").count }.by(1)
+      end
+
+      it "associates the SwfAsset with the item" do
+        Pet::AutoModeling.model_item_on_body(item, 212)
+        item.reload
+
+        asset = item.swf_assets.find_by(remote_id: 410722)
+        expect(asset).to be_present
+        expect(asset.body_id).to eq 0 # This item fits all bodies
+        expect(asset.zone_id).to eq 3
+      end
+    end
+
+    context "when item is not in the response" do
+      let(:item) do
+        # Create an item that won't be in our mock response
+        Item.create!(
+          id: 99999,
+          name: "Nonexistent Item",
+          description: "This item doesn't exist in the mock",
+          thumbnail_url: "https://example.com/item.gif",
+          rarity: "Special",
+          rarity_index: 101,
+          price: 0,
+          zones_restrict: "0000000000000000000000000000000000000000000000000000"
+        )
+      end
+
+      it "returns :not_compatible" do
+        result = Pet::AutoModeling.model_item_on_body(item, 212)
+        expect(result).to eq :not_compatible
+      end
+    end
+
+    context "when no PetType exists for the body_id" do
+      let(:item) { compatible_item }
+
+      it "raises NoPetTypeForBody" do
+        expect {
+          Pet::AutoModeling.model_item_on_body(item, 99999)
+        }.to raise_error(Pet::AutoModeling::NoPetTypeForBody) do |error|
+          expect(error.body_id).to eq 99999
+          expect(error.message).to include "99999"
+        end
+      end
+    end
+  end
+end

spec/support/mocks/custom_pets.rb

@@ -3,9 +3,14 @@ module Neopets::CustomPets
   DATA_DIR = Pathname.new(__dir__) / "custom_pets"
 
   def self.fetch_viewer_data(pet_name, ...)
-    File.open(DATA_DIR / "#{pet_name}.json") do |file|
+    # NOTE: Windows doesn't support `@` in filenames, so we use a `scis` directory instead.
-      HashWithIndifferentAccess.new JSON.load(file)
+    path = if pet_name.start_with?('@')
+      DATA_DIR / "scis" / "#{pet_name[1..]}.json"
+    else
+      DATA_DIR / "#{pet_name}.json"
     end
+
+    File.open(path) { |f| HashWithIndifferentAccess.new JSON.load(f) }
   end
 
   def self.fetch_metadata(...)

spec/support/mocks/custom_pets/scis/purpchia-39552.json

@@ -0,0 +1,69 @@
+{
+  "custom_pet": {
+    "name": "@mock:m:thyass:39552",
+    "owner": "",
+    "slot": 1.0,
+    "scale": 0.5,
+    "muted": true,
+    "body_id": 212.0,
+    "species_id": 6.0,
+    "color_id": 61.0,
+    "alt_style": false,
+    "alt_color": 61.0,
+    "style_closet_id": null,
+    "biology_by_zone": {
+      "37": {
+        "part_id": 10083.0,
+        "zone_id": 37.0,
+        "asset_url": "https://images.neopets.com/cp/bio/swf/000/000/010/10083_8a1111a13f.swf",
+        "manifest": "https://images.neopets.com/cp/bio/data/000/000/010/10083_8a1111a13f/manifest.json",
+        "zones_restrict": "0000000000000000000000000000000000000000000000000000"
+      },
+      "15": {
+        "part_id": 11613.0,
+        "zone_id": 15.0,
+        "asset_url": "https://images.neopets.com/cp/bio/swf/000/000/011/11613_f7d8d377ab.swf",
+        "manifest": "https://images.neopets.com/cp/bio/data/000/000/011/11613_f7d8d377ab/manifest.json",
+        "zones_restrict": "0000000000000000000000000000000000000000000000000000"
+      },
+      "34": {
+        "part_id": 14187.0,
+        "zone_id": 34.0,
+        "asset_url": "https://images.neopets.com/cp/bio/swf/000/000/014/14187_0e65c2082f.swf",
+        "manifest": "https://images.neopets.com/cp/bio/data/000/000/014/14187_0e65c2082f/manifest.json",
+        "zones_restrict": "0000000000000000000000000000000000000000000000000000"
+      },
+      "33": {
+        "part_id": 14189.0,
+        "zone_id": 33.0,
+        "asset_url": "https://images.neopets.com/cp/bio/swf/000/000/014/14189_102e4991e9.swf",
+        "manifest": "https://images.neopets.com/cp/bio/data/000/000/014/14189_102e4991e9/manifest.json",
+        "zones_restrict": "0000000000000000000000000000000000000000000000000000"
+      }
+    },
+    "equipped_by_zone": {},
+    "original_biology": []
+  },
+  "closet_items": {},
+  "object_info_registry": {
+    "39552": {
+      "obj_info_id": 39552.0,
+      "assets_by_zone": {},
+      "zones_restrict": "0000000000000000000000000000000000000000000000000000",
+      "is_compatible": false,
+      "is_paid": true,
+      "thumbnail_url": "https://images.neopets.com/items/mall_springyeyeglasses.gif",
+      "name": "Springy Eye Glasses",
+      "description": "Hey, keep your eyes in your head!",
+      "category": "Clothes",
+      "type": "Clothes",
+      "rarity": "Artifact",
+      "rarity_index": 500.0,
+      "price": 0.0,
+      "weight_lbs": 1.0,
+      "species_support": [3.0],
+      "converted": true
+    }
+  },
+  "object_asset_registry": {}
+}

spec/support/mocks/custom_pets/scis/purpchia-71706.json

@@ -0,0 +1,85 @@
+{
+  "custom_pet": {
+    "name": "@mock:m:thyass:71706",
+    "owner": "",
+    "slot": 1.0,
+    "scale": 0.5,
+    "muted": true,
+    "body_id": 212.0,
+    "species_id": 6.0,
+    "color_id": 61.0,
+    "alt_style": false,
+    "alt_color": 61.0,
+    "style_closet_id": null,
+    "biology_by_zone": {
+      "37": {
+        "part_id": 10083.0,
+        "zone_id": 37.0,
+        "asset_url": "https://images.neopets.com/cp/bio/swf/000/000/010/10083_8a1111a13f.swf",
+        "manifest": "https://images.neopets.com/cp/bio/data/000/000/010/10083_8a1111a13f/manifest.json",
+        "zones_restrict": "0000000000000000000000000000000000000000000000000000"
+      },
+      "15": {
+        "part_id": 11613.0,
+        "zone_id": 15.0,
+        "asset_url": "https://images.neopets.com/cp/bio/swf/000/000/011/11613_f7d8d377ab.swf",
+        "manifest": "https://images.neopets.com/cp/bio/data/000/000/011/11613_f7d8d377ab/manifest.json",
+        "zones_restrict": "0000000000000000000000000000000000000000000000000000"
+      },
+      "34": {
+        "part_id": 14187.0,
+        "zone_id": 34.0,
+        "asset_url": "https://images.neopets.com/cp/bio/swf/000/000/014/14187_0e65c2082f.swf",
+        "manifest": "https://images.neopets.com/cp/bio/data/000/000/014/14187_0e65c2082f/manifest.json",
+        "zones_restrict": "0000000000000000000000000000000000000000000000000000"
+      },
+      "33": {
+        "part_id": 14189.0,
+        "zone_id": 33.0,
+        "asset_url": "https://images.neopets.com/cp/bio/swf/000/000/014/14189_102e4991e9.swf",
+        "manifest": "https://images.neopets.com/cp/bio/data/000/000/014/14189_102e4991e9/manifest.json",
+        "zones_restrict": "0000000000000000000000000000000000000000000000000000"
+      }
+    },
+    "equipped_by_zone": {
+      "3": {
+        "asset_id": 410722.0,
+        "zone_id": 3.0,
+        "closet_obj_id": 0.0
+      }
+    },
+    "original_biology": []
+  },
+  "closet_items": {},
+  "object_info_registry": {
+    "71706": {
+      "obj_info_id": 71706.0,
+      "assets_by_zone": {
+        "3": 410722.0
+      },
+      "zones_restrict": "0000000000000000000000000000000000000000000000000000",
+      "is_compatible": true,
+      "is_paid": false,
+      "thumbnail_url": "https://images.neopets.com/items/gif_roof_onthe_fg.gif",
+      "name": "On the Roof Background",
+      "description": "Who is that on the roof?! Could it be...?",
+      "category": "Special",
+      "type": "Mystical Surroundings",
+      "rarity": "Special",
+      "rarity_index": 101.0,
+      "price": 0.0,
+      "weight_lbs": 1.0,
+      "species_support": [],
+      "converted": true
+    }
+  },
+  "object_asset_registry": {
+    "410722": {
+      "asset_id": 410722.0,
+      "zone_id": 3.0,
+      "asset_url": "https://images.neopets.com/cp/items/swf/000/000/410/410722_3bcd2f5e11.swf",
+      "obj_info_id": 71706.0,
+      "manifest": "https://images.neopets.com/cp/items/data/000/000/410/410722_3bcd2f5e11/manifest.json?v=1706"
+    }
+  }
+}

spec/support/mocks/custom_pets/scis/purpchia-99999.json

@@ -0,0 +1,24 @@
+{
+  "custom_pet": {
+    "name": "@purpchia:99999",
+    "body_id": 212.0,
+    "species_id": 6.0,
+    "color_id": 61.0,
+    "alt_style": false,
+    "alt_color": 61.0,
+    "biology_by_zone": {
+      "15": {
+        "part_id": 11613.0,
+        "zone_id": 15.0,
+        "asset_url": "https://images.neopets.com/cp/bio/swf/000/000/011/11613_f7d8d377ab.swf",
+        "manifest": "https://images.neopets.com/cp/bio/data/000/000/011/11613_f7d8d377ab/manifest.json",
+        "zones_restrict": "0000000000000000000000000000000000000000000000000000"
+      }
+    },
+    "equipped_by_zone": [],
+    "original_biology": []
+  },
+  "closet_items": [],
+  "object_info_registry": [],
+  "object_asset_registry": []
+}

spec/support/mocks/nc_mall.rb

@@ -0,0 +1,8 @@
+# We replace Neopets::NCMall.fetch_pet_data_sci with a mocked implementation.
+module Neopets::NCMall
+  # Mock implementation that generates predictable SCI hashes for testing.
+  # The hash is derived from the pet_sci and item_ids to ensure consistency.
+  def self.fetch_pet_data_sci(pet_sci, item_ids = [])
+    "#{pet_sci}-#{item_ids.sort.join('-')}"
+  end
+end