Emi Matchu
acb52cb870
Just a bit more clarity of grouping! I'm also thinking about extracting modeling APIs into a service file like this too, in which case I think this would help clarify what it is.
100 lines
3.7 KiB
Ruby
100 lines
3.7 KiB
Ruby
namespace :nc_mall do
|
|
desc "Sync our NCMallRecord table with the live NC Mall"
|
|
task :sync => :environment do
|
|
# Log to STDOUT.
|
|
Rails.logger = Logger.new(STDOUT)
|
|
|
|
# First, load all records of what's being sold in the live NC Mall. We load
|
|
# the homepage and all pages linked from the main document, and extract the
|
|
# items from each. (We also de-duplicate the items, which is important
|
|
# because the algorithm expects to only process each item once!)
|
|
pages = load_all_nc_mall_pages
|
|
live_item_records = pages.map { |p| p[:items] }.flatten.uniq
|
|
|
|
# Then, get the existing NC Mall records in our database. (We include the
|
|
# items, to be able to output the item name during logging.)
|
|
existing_records = NCMallRecord.includes(:item).all
|
|
existing_records_by_item_id = existing_records.to_h { |r| [r.item_id, r] }
|
|
|
|
# Additionally, check which of the item IDs in the live records are items
|
|
# we've seen before. (We'll skip records for items we don't know.)
|
|
live_item_ids = live_item_records.map { |r| r[:id] }
|
|
recognized_item_ids = Item.where(id: live_item_ids).pluck(:id).to_set
|
|
Rails.logger.debug "We found #{live_item_records.size} items, and we " +
|
|
"recognize #{recognized_item_ids.size} of them."
|
|
|
|
# For each record in the live NC Mall, check if there's an existing record.
|
|
# If so, update it, and remove it from the existing records hash. If not,
|
|
# create it.
|
|
live_item_records.each do |record_data|
|
|
# If we don't recognize this item ID in our database already, skip it.
|
|
next unless recognized_item_ids.include?(record_data[:id])
|
|
|
|
record = existing_records_by_item_id.delete(record_data[:id]) ||
|
|
NCMallRecord.new
|
|
record.item_id = record_data[:id]
|
|
record.price = record_data[:price]
|
|
record.discount_price = record_data.dig(:discount, :price)
|
|
record.discount_begins_at = record_data.dig(:discount, :begins_at)
|
|
record.discount_ends_at = record_data.dig(:discount, :ends_at)
|
|
|
|
if !record.changed?
|
|
Rails.logger.info "Skipping record for item #{record_data[:name]} " +
|
|
"(unchanged)"
|
|
next
|
|
end
|
|
|
|
if record.save
|
|
if record.previously_new_record?
|
|
Rails.logger.info "Created record for item #{record_data[:name]}"
|
|
else
|
|
Rails.logger.info "Updated record for item #{record_data[:name]}"
|
|
end
|
|
else
|
|
Rails.logger.error "Failed to save record for item " +
|
|
"#{record_data[:name]}: " +
|
|
"#{record.errors.full_messages.join("; ")}: " +
|
|
"#{record.inspect}"
|
|
end
|
|
end
|
|
|
|
# For each existing record remaining in the existing records hash, this
|
|
# means there was no live record corresponding to it during this sync.
|
|
# Delete it!
|
|
existing_records_by_item_id.values.each do |record|
|
|
item_name = record.item&.name || "<item not found>"
|
|
if record.destroy
|
|
Rails.logger.info "Destroyed record #{record.id} for item " +
|
|
"#{item_name}"
|
|
else
|
|
Rails.logger.error "Failed to destroy record #{record.id} for " +
|
|
"item #{item_name}: #{record.inspect}"
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def load_all_nc_mall_pages
|
|
Sync do
|
|
# First, start loading the homepage.
|
|
homepage_task = Async { Neopets::NCMall.load_home_page }
|
|
|
|
# Next, load the page links for different categories etc.
|
|
links = Neopets::NCMall.load_page_links
|
|
|
|
# Next, load the linked pages, 10 at a time.
|
|
barrier = Async::Barrier.new
|
|
semaphore = Async::Semaphore.new(10, parent: barrier)
|
|
begin
|
|
linked_page_tasks = links.map do |link|
|
|
semaphore.async { Neopets::NCMall.load_page link[:type], link[:cat] }
|
|
end
|
|
barrier.wait # Load all the pages.
|
|
ensure
|
|
barrier.stop # If any pages failed, cancel the rest.
|
|
end
|
|
|
|
# Finally, return all the pages: the homepage, and the linked pages.
|
|
[homepage_task.wait] + linked_page_tasks.map(&:wait)
|
|
end
|
|
end
|