first step in repairing mall spider

This commit is contained in:
Emi Matchu 2013-07-02 11:30:38 -07:00
parent 72c59f0b68
commit 9bd49aa85d
2 changed files with 41 additions and 40 deletions

View file

@ -459,30 +459,35 @@ class Item < ActiveRecord::Base
end
puts "#{items.size} items found"
all_item_ids = items.keys
# Find which of these already exist but aren't marked as sold_in_mall so
# we can update them as being sold
Item.not_sold_in_mall.where(:id => items.keys).select([:id]).each do |item|
items.delete(item.id)
item.sold_in_mall = true
item.save
puts "#{item.name} (#{item.id}) now in mall, updated"
end
# Find items marked as sold_in_mall so we can skip those we just found
# if they already are properly marked, and mark those that we didn't just
# find as no longer sold_in_mall
Item.sold_in_mall.select([:id, :name]).each do |item|
if all_item_ids.include?(item.id)
Item.transaction do
# Find which of these already exist but aren't marked as sold_in_mall so
# we can update them as being sold
items_added_to_mall = Item.not_sold_in_mall.includes(:translations).
where(:id => items.keys)
items_added_to_mall.each do |item|
items.delete(item.id)
else
item.sold_in_mall = false
item.sold_in_mall = true
item.save
puts "#{item.name} (#{item.id}) no longer in mall, removed sold_in_mall status"
puts "#{item.name} (#{item.id}) now in mall, updated"
end
# Find items marked as sold_in_mall so we can skip those we just found
# if they already are properly marked, and mark those that we didn't just
# find as no longer sold_in_mall
items_removed_from_mall = Item.sold_in_mall.includes(:translations)
items_removed_from_mall.each do |item|
if all_item_ids.include?(item.id)
items.delete(item.id)
else
item.sold_in_mall = false
item.save
puts "#{item.name} (#{item.id}) no longer in mall, removed sold_in_mall status"
end
end
puts "#{items.size} new items"
items.each do |item_id, item|
item.save
puts "Saved #{item.name} (#{item_id})"
end
end
puts "#{items.size} new items"
items.each do |item_id, item|
item.save
puts "Saved #{item.name} (#{item_id})"
end
items
end
@ -530,6 +535,7 @@ class Item < ActiveRecord::Base
def spider(item)
puts " - Using #{@name} strategy"
exit = false
Rails.logger.debug(@pet_types.inspect)
@pet_types.each do |pet_type|
swf_assets = load_for_pet_type(item, pet_type)
if swf_assets
@ -561,9 +567,15 @@ class Item < ActiveRecord::Base
private
def load_for_pet_type(item, pet_type, banned_pet_ids=[])
pet_id = pet_type.pet_id
pet_name = pet_type.pet_name
def load_for_pet_type(item, pet_type)
original_pet = Pet.select([:id, :name]).
where(pet_type_id: pet_type.id).first
if original_pet.nil?
puts " - We have no more pets of type \##{pet_type.id}. Skipping"
return nil
end
pet_id = original_pet.id
pet_name = original_pet.name
pet_valid = nil
begin
pet = Pet.load(pet_name)
@ -577,7 +589,7 @@ class Item < ActiveRecord::Base
rescue Pet::PetNotFound
pet_valid = false
puts " - Pet #{pet_name} no longer exists; destroying and loading new pet"
Pet.find_by_name(pet_name).destroy
original_pet.destroy
end
if pet_valid
swf_assets = load_for_pet_name(item, pet_type, pet_name)
@ -588,16 +600,7 @@ class Item < ActiveRecord::Base
end
return swf_assets
else
banned_pet_ids << pet_id
new_pet = pet_type.pets.select([:id, :name]).where(Pet.arel_table[:id].not_in(banned_pet_ids)).first
if new_pet
pet_type.pet_id = new_pet.id
pet_type.pet_name = new_pet.name
load_for_pet_type(item, pet_type, banned_pet_ids)
else
puts " - We have no more pets of type \##{pet_type.id}. Skipping"
return nil
end
load_for_pet_type(item, pet_type) # try again
end
end
@ -665,9 +668,7 @@ class Item < ActiveRecord::Base
if Strategies.empty?
pet_type_t = PetType.arel_table
require 'pet' # FIXME: console is whining when i don't do this
pet_t = Pet.arel_table
pet_types = PetType.select([pet_type_t[:id], pet_type_t[:body_id], "#{Pet.table_name}.id as pet_id, #{Pet.table_name}.name as pet_name"]).
joins(:pets).group(pet_type_t[:id])
pet_types = PetType.select([:id, :body_id])
remaining_standard_pet_types = pet_types.single_standard_color.order(:species_id)
first_standard_pet_type = [remaining_standard_pet_types.slice!(0)]

View file

@ -43,11 +43,11 @@ class Pet < ActiveRecord::Base
true
end
def fetch_viewer_data
def fetch_viewer_data(timeout=4)
begin
neopets_language_code = I18n.compatible_neopets_language_code_for(I18n.locale)
envelope = PET_VIEWER.request([name, 0]).post(
:timeout => 4,
:timeout => timeout,
:headers => {
'Cookie' => "lang=#{neopets_language_code}"
}