Initial NC Mall scraper service
This doesn't connect to anything yet, I'm just doing the beginnings of loading NC Mall item data! My intent is to run this regularly to keep our own NC info in the database too, primarily for use in the Item Getting Guide. (Could be useful to surface in other places too though!) This will help us split items into those that can be one-click purchased with the NC Mall integration, vs NC items that need to be acquired by other means.
This commit is contained in:
parent
9733ceae25
commit
7b0b6b70d2
2 changed files with 79 additions and 0 deletions
75
app/services/nc_mall.rb
Normal file
75
app/services/nc_mall.rb
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
require "async/http/internet/instance"
|
||||||
|
|
||||||
|
module NCMall
|
||||||
|
# Share a pool of persistent connections, rather than reconnecting on
|
||||||
|
# each request. (This library does that automatically!)
|
||||||
|
INTERNET = Async::HTTP::Internet.instance
|
||||||
|
|
||||||
|
# Load the NC home page, and return its useful data.
|
||||||
|
HOME_PAGE_URL = "https://ncmall.neopets.com/mall/ajax/home_page.phtml"
|
||||||
|
def self.load_home_page
|
||||||
|
response = Sync do
|
||||||
|
INTERNET.get(HOME_PAGE_URL, [
|
||||||
|
["User-Agent", Rails.configuration.user_agent_for_neopets],
|
||||||
|
])
|
||||||
|
end
|
||||||
|
|
||||||
|
if response.status != 200
|
||||||
|
raise ResponseNotOK.new(response.status),
|
||||||
|
"expected status 200 but got #{response.status} (#{HOME_PAGE_URL})"
|
||||||
|
end
|
||||||
|
|
||||||
|
parse_nc_page response.body.read
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
# Given a string of NC page data, parse the useful data out of it!
|
||||||
|
def self.parse_nc_page(nc_page_str)
|
||||||
|
begin
|
||||||
|
nc_page = JSON.parse(nc_page_str)
|
||||||
|
rescue JSON::ParserError
|
||||||
|
Rails.logger.debug "Unexpected NC page response:\n#{nc_page_str}"
|
||||||
|
raise UnexpectedResponseFormat,
|
||||||
|
"failed to parse NC page response as JSON"
|
||||||
|
end
|
||||||
|
|
||||||
|
unless nc_page.has_key? "object_data"
|
||||||
|
raise UnexpectedResponseFormat, "missing field object_data in NC page"
|
||||||
|
end
|
||||||
|
|
||||||
|
items = nc_page["object_data"].values.map do |item_info|
|
||||||
|
{
|
||||||
|
id: item_info["id"],
|
||||||
|
name: item_info["name"],
|
||||||
|
description: item_info["description"],
|
||||||
|
price: item_info["price"],
|
||||||
|
discount: parse_item_discount(item_info),
|
||||||
|
is_available: item_info["isAvailable"] == 1,
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
{items:}
|
||||||
|
end
|
||||||
|
|
||||||
|
# Given item info, return a hash of discount-specific info, if any.
|
||||||
|
def self.parse_item_discount(item_info)
|
||||||
|
discount_price = item_info["discountPrice"]
|
||||||
|
return nil unless discount_price.present? && discount_price > 0
|
||||||
|
|
||||||
|
{
|
||||||
|
price: discount_price,
|
||||||
|
start: item_info["discountBegin"],
|
||||||
|
end: item_info["discountEnd"],
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
class ResponseNotOK < StandardError
|
||||||
|
attr_reader :status
|
||||||
|
def initialize(status)
|
||||||
|
super
|
||||||
|
@status = status
|
||||||
|
end
|
||||||
|
end
|
||||||
|
class UnexpectedResponseFormat < StandardError;end
|
||||||
|
end
|
|
@ -21,4 +21,8 @@ ActiveSupport::Inflector.inflections(:en) do |inflect|
|
||||||
|
|
||||||
# Teach Zeitwerk that `NeoPass` is what to expect in `app/services/neopass.rb`.
|
# Teach Zeitwerk that `NeoPass` is what to expect in `app/services/neopass.rb`.
|
||||||
inflect.acronym "NeoPass"
|
inflect.acronym "NeoPass"
|
||||||
|
|
||||||
|
# Teach Zeitwerk that "NCMall" is what to expect in `app/services/nc_mall.rb`.
|
||||||
|
# (We do this by teaching it the word "NC".)
|
||||||
|
inflect.acronym "NC"
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in a new issue