From 1f157b49dafa886492ab20968f76a02ce9ce8ef2 Mon Sep 17 00:00:00 2001 From: Matchu Date: Tue, 7 May 2024 17:38:48 -0700 Subject: [PATCH] Load additional pages via NC Mall scraper service This is for URLs like this! https://ncmall.neopets.com/mall/ajax/load_page.phtml?type=browse&cat=43&lang=en --- app/services/nc_mall.rb | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/app/services/nc_mall.rb b/app/services/nc_mall.rb index 8edf7448..a2216e58 100644 --- a/app/services/nc_mall.rb +++ b/app/services/nc_mall.rb @@ -1,3 +1,4 @@ +require "addressable/template" require "async/http/internet/instance" module NCMall @@ -8,22 +9,34 @@ module NCMall # Load the NC home page, and return its useful data. HOME_PAGE_URL = "https://ncmall.neopets.com/mall/ajax/home_page.phtml" def self.load_home_page - response = Sync do - INTERNET.get(HOME_PAGE_URL, [ - ["User-Agent", Rails.configuration.user_agent_for_neopets], - ]) - end + load_page_by_url HOME_PAGE_URL + end - if response.status != 200 - raise ResponseNotOK.new(response.status), - "expected status 200 but got #{response.status} (#{HOME_PAGE_URL})" - end - - parse_nc_page response.body.read + # Load the NC Mall page for a specific type and category ID. + CATEGORY_PAGE_URL_TEMPLATE = Addressable::Template.new( + "https://ncmall.neopets.com/mall/ajax/load_page.phtml?lang=en{&type,cat}" + ) + def self.load_page(type, cat) + load_page_by_url CATEGORY_PAGE_URL_TEMPLATE.expand(type:, cat:) end private + def self.load_page_by_url(url) + Sync do + response = INTERNET.get(url, [ + ["User-Agent", Rails.configuration.user_agent_for_neopets], + ]) + + if response.status != 200 + raise ResponseNotOK.new(response.status), + "expected status 200 but got #{response.status} (#{url})" + end + + parse_nc_page response.read + end + end + # Given a string of NC page data, parse the useful data out of it! def self.parse_nc_page(nc_page_str) begin