Compare commits
No commits in common. "46d3325144fbc6fd78a030abd9274c6fb4372aaa" and "9733ceae250f5272fe1e10ac3fb7c71594553417" have entirely different histories.
46d3325144
...
9733ceae25
7 changed files with 1 additions and 260 deletions
|
|
@ -1,3 +0,0 @@
|
|||
class NCMallRecord < ApplicationRecord
|
||||
belongs_to :item
|
||||
end
|
||||
|
|
@ -1,116 +0,0 @@
|
|||
require "addressable/template"
|
||||
require "async/http/internet/instance"
|
||||
|
||||
module NCMall
|
||||
# Share a pool of persistent connections, rather than reconnecting on
|
||||
# each request. (This library does that automatically!)
|
||||
INTERNET = Async::HTTP::Internet.instance
|
||||
|
||||
# Load the NC Mall home page content area, and return its useful data.
|
||||
HOME_PAGE_URL = "https://ncmall.neopets.com/mall/ajax/home_page.phtml"
|
||||
def self.load_home_page
|
||||
load_page_by_url HOME_PAGE_URL
|
||||
end
|
||||
|
||||
# Load the NC Mall page for a specific type and category ID.
|
||||
CATEGORY_PAGE_URL_TEMPLATE = Addressable::Template.new(
|
||||
"https://ncmall.neopets.com/mall/ajax/load_page.phtml?lang=en{&type,cat}"
|
||||
)
|
||||
def self.load_page(type, cat)
|
||||
load_page_by_url CATEGORY_PAGE_URL_TEMPLATE.expand(type:, cat:)
|
||||
end
|
||||
|
||||
# Load the NC Mall root document HTML, and extract the list of links to
|
||||
# other pages ("New", "Popular", etc.)
|
||||
ROOT_DOCUMENT_URL = "https://ncmall.neopets.com/mall/shop.phtml"
|
||||
PAGE_LINK_PATTERN = /load_items_pane\(['"](.+?)['"], ([0-9]+)\).+?>(.+?)</
|
||||
def self.load_page_links
|
||||
Sync do
|
||||
response = INTERNET.get(ROOT_DOCUMENT_URL, [
|
||||
["User-Agent", Rails.configuration.user_agent_for_neopets],
|
||||
])
|
||||
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{url})"
|
||||
end
|
||||
|
||||
# Extract `load_items_pane` calls from the root document's HTML. (We use
|
||||
# a very simplified regex, rather than actually parsing the full HTML!)
|
||||
html = response.read
|
||||
html.scan(PAGE_LINK_PATTERN).
|
||||
map { |type, cat, label| {type:, cat:, label:} }.
|
||||
uniq
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def self.load_page_by_url(url)
|
||||
Sync do
|
||||
response = INTERNET.get(url, [
|
||||
["User-Agent", Rails.configuration.user_agent_for_neopets],
|
||||
])
|
||||
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{url})"
|
||||
end
|
||||
|
||||
parse_nc_page response.read
|
||||
end
|
||||
end
|
||||
|
||||
# Given a string of NC page data, parse the useful data out of it!
|
||||
def self.parse_nc_page(nc_page_str)
|
||||
begin
|
||||
nc_page = JSON.parse(nc_page_str)
|
||||
rescue JSON::ParserError
|
||||
Rails.logger.debug "Unexpected NC page response:\n#{nc_page_str}"
|
||||
raise UnexpectedResponseFormat,
|
||||
"failed to parse NC page response as JSON"
|
||||
end
|
||||
|
||||
unless nc_page.has_key? "object_data"
|
||||
raise UnexpectedResponseFormat, "missing field object_data in NC page"
|
||||
end
|
||||
|
||||
# NOTE: When there's no object data, it will be an empty array instead of
|
||||
# an empty hash. Weird API thing to work around!
|
||||
nc_page["object_data"] = {} if nc_page["object_data"] == []
|
||||
|
||||
items = nc_page["object_data"].values.map do |item_info|
|
||||
{
|
||||
id: item_info["id"],
|
||||
name: item_info["name"],
|
||||
description: item_info["description"],
|
||||
price: item_info["price"],
|
||||
discount: parse_item_discount(item_info),
|
||||
is_available: item_info["isAvailable"] == 1,
|
||||
}
|
||||
end
|
||||
|
||||
{items:}
|
||||
end
|
||||
|
||||
# Given item info, return a hash of discount-specific info, if any.
|
||||
def self.parse_item_discount(item_info)
|
||||
discount_price = item_info["discountPrice"]
|
||||
return nil unless discount_price.present? && discount_price > 0
|
||||
|
||||
{
|
||||
price: discount_price,
|
||||
begins_at: item_info["discountBegin"],
|
||||
ends_at: item_info["discountEnd"],
|
||||
}
|
||||
end
|
||||
|
||||
class ResponseNotOK < StandardError
|
||||
attr_reader :status
|
||||
def initialize(status)
|
||||
super
|
||||
@status = status
|
||||
end
|
||||
end
|
||||
class UnexpectedResponseFormat < StandardError;end
|
||||
end
|
||||
|
|
@ -21,8 +21,4 @@ ActiveSupport::Inflector.inflections(:en) do |inflect|
|
|||
|
||||
# Teach Zeitwerk that `NeoPass` is what to expect in `app/services/neopass.rb`.
|
||||
inflect.acronym "NeoPass"
|
||||
|
||||
# Teach Zeitwerk that "NCMall" is what to expect in `app/services/nc_mall.rb`.
|
||||
# (We do this by teaching it the word "NC".)
|
||||
inflect.acronym "NC"
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
class CreateNCMallRecords < ActiveRecord::Migration[7.1]
|
||||
def change
|
||||
create_table :nc_mall_records do |t|
|
||||
t.references :item, type: :integer, null: false, foreign_key: true
|
||||
t.integer :price, null: false
|
||||
t.integer :discount_price
|
||||
t.datetime :discount_begins_at
|
||||
t.datetime :discount_ends_at
|
||||
|
||||
t.timestamps
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
class AddUniqueIndexOnItemIdToNCMallRecords < ActiveRecord::Migration[7.1]
|
||||
def change
|
||||
# NOTE: We need to temporarily remove the foreign key, then add it back
|
||||
# once the index is in.
|
||||
remove_foreign_key :nc_mall_records, :items
|
||||
remove_index :nc_mall_records, :item_id
|
||||
|
||||
add_index :nc_mall_records, :item_id, unique: true
|
||||
add_foreign_key :nc_mall_records, :items
|
||||
end
|
||||
end
|
||||
14
db/schema.rb
14
db/schema.rb
|
|
@ -10,7 +10,7 @@
|
|||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[7.1].define(version: 2024_05_11_003019) do
|
||||
ActiveRecord::Schema[7.1].define(version: 2024_05_02_195157) do
|
||||
create_table "alt_styles", charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
|
||||
t.integer "species_id", null: false
|
||||
t.integer "color_id", null: false
|
||||
|
|
@ -154,17 +154,6 @@ ActiveRecord::Schema[7.1].define(version: 2024_05_11_003019) do
|
|||
t.string "pet_name", limit: 128, null: false
|
||||
end
|
||||
|
||||
create_table "nc_mall_records", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t|
|
||||
t.integer "item_id", null: false
|
||||
t.integer "price", null: false
|
||||
t.integer "discount_price"
|
||||
t.datetime "discount_begins_at"
|
||||
t.datetime "discount_ends_at"
|
||||
t.datetime "created_at", null: false
|
||||
t.datetime "updated_at", null: false
|
||||
t.index ["item_id"], name: "index_nc_mall_records_on_item_id", unique: true
|
||||
end
|
||||
|
||||
create_table "neopets_connections", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
|
||||
t.integer "user_id"
|
||||
t.string "neopets_username"
|
||||
|
|
@ -289,6 +278,5 @@ ActiveRecord::Schema[7.1].define(version: 2024_05_11_003019) do
|
|||
|
||||
add_foreign_key "alt_styles", "colors"
|
||||
add_foreign_key "alt_styles", "species"
|
||||
add_foreign_key "nc_mall_records", "items"
|
||||
add_foreign_key "outfits", "alt_styles"
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
namespace :nc_mall do
|
||||
desc "Sync our NCMallRecord table with the live NC Mall"
|
||||
task :sync => :environment do
|
||||
# Log to STDOUT.
|
||||
Rails.logger = Logger.new(STDOUT)
|
||||
|
||||
# First, load all records of what's being sold in the live NC Mall. We load
|
||||
# the homepage and all pages linked from the main document, and extract the
|
||||
# items from each. (We also de-duplicate the items, which is important
|
||||
# because the algorithm expects to only process each item once!)
|
||||
pages = load_all_nc_mall_pages
|
||||
live_item_records = pages.map { |p| p[:items] }.flatten.uniq
|
||||
|
||||
# Then, get the existing NC Mall records in our database. (We include the
|
||||
# items, to be able to output the item name during logging.)
|
||||
existing_records = NCMallRecord.includes(:item).all
|
||||
existing_records_by_item_id = existing_records.to_h { |r| [r.item_id, r] }
|
||||
|
||||
# Additionally, check which of the item IDs in the live records are items
|
||||
# we've seen before. (We'll skip records for items we don't know.)
|
||||
live_item_ids = live_item_records.map { |r| r[:id] }
|
||||
recognized_item_ids = Item.where(id: live_item_ids).pluck(:id).to_set
|
||||
Rails.logger.debug "We found #{live_item_records.size} items, and we " +
|
||||
"recognize #{recognized_item_ids.size} of them."
|
||||
|
||||
# For each record in the live NC Mall, check if there's an existing record.
|
||||
# If so, update it, and remove it from the existing records hash. If not,
|
||||
# create it.
|
||||
live_item_records.each do |record_data|
|
||||
# If we don't recognize this item ID in our database already, skip it.
|
||||
next unless recognized_item_ids.include?(record_data[:id])
|
||||
|
||||
record = existing_records_by_item_id.delete(record_data[:id]) ||
|
||||
NCMallRecord.new
|
||||
record.item_id = record_data[:id]
|
||||
record.price = record_data[:price]
|
||||
record.discount_price = record_data.dig(:discount, :price)
|
||||
record.discount_begins_at = record_data.dig(:discount, :begins_at)
|
||||
record.discount_ends_at = record_data.dig(:discount, :ends_at)
|
||||
|
||||
if !record.changed?
|
||||
Rails.logger.info "Skipping record for item #{record_data[:name]} " +
|
||||
"(unchanged)"
|
||||
next
|
||||
end
|
||||
|
||||
if record.save
|
||||
if record.previously_new_record?
|
||||
Rails.logger.info "Created record for item #{record_data[:name]}"
|
||||
else
|
||||
Rails.logger.info "Updated record for item #{record_data[:name]}"
|
||||
end
|
||||
else
|
||||
Rails.logger.error "Failed to save record for item " +
|
||||
"#{record_data[:name]}: " +
|
||||
"#{record.errors.full_messages.join("; ")}: " +
|
||||
"#{record.inspect}"
|
||||
end
|
||||
end
|
||||
|
||||
# For each existing record remaining in the existing records hash, this
|
||||
# means there was no live record corresponding to it during this sync.
|
||||
# Delete it!
|
||||
existing_records_by_item_id.values.each do |record|
|
||||
item_name = record.item&.name || "<item not found>"
|
||||
if record.destroy
|
||||
Rails.logger.info "Destroyed record #{record.id} for item " +
|
||||
"#{item_name}"
|
||||
else
|
||||
Rails.logger.error "Failed to destroy record #{record.id} for " +
|
||||
"item #{item_name}: #{record.inspect}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def load_all_nc_mall_pages
|
||||
Sync do
|
||||
# First, start loading the homepage.
|
||||
homepage_task = Async { NCMall.load_home_page }
|
||||
|
||||
# Next, load the page links for different categories etc.
|
||||
links = NCMall.load_page_links
|
||||
|
||||
# Next, load the linked pages, 10 at a time.
|
||||
barrier = Async::Barrier.new
|
||||
semaphore = Async::Semaphore.new(10, parent: barrier)
|
||||
begin
|
||||
linked_page_tasks = links.map do |link|
|
||||
semaphore.async { NCMall.load_page link[:type], link[:cat] }
|
||||
end
|
||||
barrier.wait # Load all the pages.
|
||||
ensure
|
||||
barrier.stop # If any pages failed, cancel the rest.
|
||||
end
|
||||
|
||||
# Finally, return all the pages: the homepage, and the linked pages.
|
||||
[homepage_task.wait] + linked_page_tasks.map(&:wait)
|
||||
end
|
||||
end
|
||||
Loading…
Reference in a new issue