Compare commits
No commits in common. "620e59f3edb2625d1bd0779b59e3d377388e4b39" and "052c02f841f1f07456e3ed93bdf65ea38d556239" have entirely different histories.
620e59f3ed
...
052c02f841
31 changed files with 76 additions and 183 deletions
6
Gemfile
6
Gemfile
|
@ -4,7 +4,7 @@ ruby '3.3.4'
|
|||
gem 'rails', '~> 7.1', '>= 7.1.3.4'
|
||||
|
||||
# The HTTP server running the Rails instance.
|
||||
gem 'falcon', '~> 0.48.0'
|
||||
gem 'falcon', '~> 0.43.0'
|
||||
|
||||
# Our database is MySQL, in both development and production.
|
||||
gem 'mysql2', '~> 0.5.5'
|
||||
|
@ -61,8 +61,8 @@ gem "httparty", "~> 0.22.0"
|
|||
gem "addressable", "~> 2.8"
|
||||
|
||||
# For advanced batching of many HTTP requests.
|
||||
gem "async", "~> 2.17", require: false
|
||||
gem "async-http", "~> 0.75.0", require: false
|
||||
gem "async", "~> 2.6", require: false
|
||||
gem "async-http", "~> 0.61.0", require: false
|
||||
gem "thread-local", "~> 1.1", require: false
|
||||
|
||||
# For debugging.
|
||||
|
|
69
Gemfile.lock
69
Gemfile.lock
|
@ -81,30 +81,29 @@ GEM
|
|||
public_suffix (>= 2.0.2, < 7.0)
|
||||
aes_key_wrap (1.1.0)
|
||||
ast (2.4.2)
|
||||
async (2.17.0)
|
||||
async (2.16.1)
|
||||
console (~> 1.26)
|
||||
fiber-annotation
|
||||
io-event (~> 1.6, >= 1.6.5)
|
||||
async-container (0.18.3)
|
||||
async (~> 2.10)
|
||||
async-http (0.75.0)
|
||||
async (>= 2.10.2)
|
||||
async-pool (~> 0.7)
|
||||
io-endpoint (~> 0.11)
|
||||
io-stream (~> 0.4)
|
||||
protocol-http (~> 0.30)
|
||||
protocol-http1 (~> 0.20)
|
||||
protocol-http2 (~> 0.18)
|
||||
traces (>= 0.10)
|
||||
async-container (0.16.13)
|
||||
async
|
||||
async-io
|
||||
async-http (0.61.0)
|
||||
async (>= 1.25)
|
||||
async-io (>= 1.28)
|
||||
async-pool (>= 0.2)
|
||||
protocol-http (~> 0.25.0)
|
||||
protocol-http1 (~> 0.16.0)
|
||||
protocol-http2 (~> 0.15.0)
|
||||
traces (>= 0.10.0)
|
||||
async-http-cache (0.4.4)
|
||||
async-http (~> 0.56)
|
||||
async-io (1.43.2)
|
||||
async
|
||||
async-pool (0.8.1)
|
||||
async (>= 1.25)
|
||||
metrics
|
||||
traces
|
||||
async-service (0.12.0)
|
||||
async
|
||||
async-container (~> 0.16)
|
||||
attr_required (1.0.2)
|
||||
babel-source (5.8.35)
|
||||
babel-transpiler (0.7.0)
|
||||
|
@ -119,6 +118,7 @@ GEM
|
|||
bindex (0.8.1)
|
||||
bootsnap (1.18.4)
|
||||
msgpack (~> 1.2)
|
||||
build-environment (1.13.0)
|
||||
builder (3.3.0)
|
||||
childprocess (5.1.0)
|
||||
logger (~> 1.5)
|
||||
|
@ -150,19 +150,19 @@ GEM
|
|||
activemodel
|
||||
erubi (1.13.0)
|
||||
execjs (2.9.1)
|
||||
falcon (0.48.0)
|
||||
falcon (0.43.0)
|
||||
async
|
||||
async-container (~> 0.18)
|
||||
async-http (~> 0.75)
|
||||
async-http-cache (~> 0.4)
|
||||
async-service (~> 0.10)
|
||||
async-container (~> 0.16.0)
|
||||
async-http (~> 0.57)
|
||||
async-http-cache (~> 0.4.0)
|
||||
async-io (~> 1.22)
|
||||
build-environment (~> 1.13)
|
||||
bundler
|
||||
localhost (~> 1.1)
|
||||
openssl (~> 3.0)
|
||||
process-metrics (~> 0.2)
|
||||
protocol-http (~> 0.31)
|
||||
protocol-rack (~> 0.7)
|
||||
samovar (~> 2.3)
|
||||
process-metrics (~> 0.2.0)
|
||||
protocol-rack (~> 0.1)
|
||||
samovar (~> 2.1)
|
||||
faraday (2.11.0)
|
||||
faraday-net_http (>= 2.0, < 3.4)
|
||||
logger
|
||||
|
@ -190,9 +190,7 @@ GEM
|
|||
i18n (1.14.5)
|
||||
concurrent-ruby (~> 1.0)
|
||||
io-console (0.7.2)
|
||||
io-endpoint (0.13.1)
|
||||
io-event (1.6.5)
|
||||
io-stream (0.4.0)
|
||||
irb (1.14.0)
|
||||
rdoc (>= 4.0.0)
|
||||
reline (>= 0.4.2)
|
||||
|
@ -282,19 +280,18 @@ GEM
|
|||
parser (3.3.4.2)
|
||||
ast (~> 2.4.1)
|
||||
racc
|
||||
process-metrics (0.3.0)
|
||||
process-metrics (0.2.1)
|
||||
console (~> 1.8)
|
||||
json (~> 2)
|
||||
samovar (~> 2.1)
|
||||
protocol-hpack (1.5.0)
|
||||
protocol-http (0.33.0)
|
||||
protocol-http1 (0.22.0)
|
||||
protocol-http (0.25.0)
|
||||
protocol-http1 (0.16.1)
|
||||
protocol-http (~> 0.22)
|
||||
protocol-http2 (0.18.0)
|
||||
protocol-http2 (0.15.1)
|
||||
protocol-hpack (~> 1.4)
|
||||
protocol-http (~> 0.18)
|
||||
protocol-rack (0.7.0)
|
||||
protocol-http (~> 0.27)
|
||||
protocol-rack (0.6.0)
|
||||
protocol-http (~> 0.23)
|
||||
rack (>= 1.0)
|
||||
psych (5.1.2)
|
||||
stringio
|
||||
|
@ -498,13 +495,13 @@ PLATFORMS
|
|||
DEPENDENCIES
|
||||
RocketAMF!
|
||||
addressable (~> 2.8)
|
||||
async (~> 2.17)
|
||||
async-http (~> 0.75.0)
|
||||
async (~> 2.6)
|
||||
async-http (~> 0.61.0)
|
||||
bootsnap (~> 1.16)
|
||||
devise (~> 4.9, >= 4.9.2)
|
||||
devise-encryptable (~> 0.2.0)
|
||||
dotenv-rails (~> 2.8, >= 2.8.1)
|
||||
falcon (~> 0.48.0)
|
||||
falcon (~> 0.43.0)
|
||||
haml (~> 6.1, >= 6.1.1)
|
||||
http_accept_language (~> 2.1, >= 2.1.1)
|
||||
httparty (~> 0.22.0)
|
||||
|
|
|
@ -112,7 +112,9 @@ class OutfitLayer extends HTMLElement {
|
|||
this.#setStatus("error"),
|
||||
);
|
||||
} else {
|
||||
console.warn(`<outfit-layer> contained no image or iframe: `, this);
|
||||
throw new Error(
|
||||
`<outfit-layer> must contain an <img> or <iframe> tag`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -25,40 +25,39 @@ module NCMall
|
|||
ROOT_DOCUMENT_URL = "https://ncmall.neopets.com/mall/shop.phtml"
|
||||
PAGE_LINK_PATTERN = /load_items_pane\(['"](.+?)['"], ([0-9]+)\).+?>(.+?)</
|
||||
def self.load_page_links
|
||||
html = Sync do
|
||||
INTERNET.get(ROOT_DOCUMENT_URL, [
|
||||
Sync do
|
||||
response = INTERNET.get(ROOT_DOCUMENT_URL, [
|
||||
["User-Agent", Rails.configuration.user_agent_for_neopets],
|
||||
]) do |response|
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{url})"
|
||||
end
|
||||
])
|
||||
|
||||
response.read
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{url})"
|
||||
end
|
||||
end
|
||||
|
||||
# Extract `load_items_pane` calls from the root document's HTML. (We use
|
||||
# a very simplified regex, rather than actually parsing the full HTML!)
|
||||
html.scan(PAGE_LINK_PATTERN).
|
||||
map { |type, cat, label| {type:, cat:, label:} }.
|
||||
uniq
|
||||
# Extract `load_items_pane` calls from the root document's HTML. (We use
|
||||
# a very simplified regex, rather than actually parsing the full HTML!)
|
||||
html = response.read
|
||||
html.scan(PAGE_LINK_PATTERN).
|
||||
map { |type, cat, label| {type:, cat:, label:} }.
|
||||
uniq
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def self.load_page_by_url(url)
|
||||
Sync do
|
||||
INTERNET.get(url, [
|
||||
response = INTERNET.get(url, [
|
||||
["User-Agent", Rails.configuration.user_agent_for_neopets],
|
||||
]) do |response|
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{url})"
|
||||
end
|
||||
])
|
||||
|
||||
parse_nc_page response.read
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{url})"
|
||||
end
|
||||
|
||||
parse_nc_page response.read
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -31,20 +31,20 @@ module NeoPass
|
|||
|
||||
LINKAGE_URL = "https://oidc.neopets.com/linkage/all"
|
||||
def self.load_linkages(access_token)
|
||||
linkages_str = Sync do
|
||||
INTERNET.get(LINKAGE_URL, [
|
||||
response = Sync do
|
||||
response = INTERNET.get(LINKAGE_URL, [
|
||||
["User-Agent", Rails.configuration.user_agent_for_neopets],
|
||||
["Authorization", "Bearer #{access_token}"],
|
||||
]) do |response|
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{LINKAGE_URL})"
|
||||
end
|
||||
|
||||
response.read
|
||||
end
|
||||
])
|
||||
end
|
||||
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{LINKAGE_URL})"
|
||||
end
|
||||
|
||||
linkages_str = response.body.read
|
||||
|
||||
begin
|
||||
linkages = JSON.parse(linkages_str)
|
||||
rescue JSON::ParserError
|
||||
|
|
|
@ -72,15 +72,14 @@ module NeopetsMediaArchive
|
|||
# We use this in the `swf_assets:manifests:load` task to perform many
|
||||
# requests in parallel!
|
||||
Sync do
|
||||
INTERNET.get(uri, [
|
||||
response = INTERNET.get(uri, [
|
||||
["User-Agent", Rails.configuration.user_agent_for_neopets],
|
||||
]) do |response|
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{uri})"
|
||||
end
|
||||
response.read
|
||||
])
|
||||
if response.status != 200
|
||||
raise ResponseNotOK.new(response.status),
|
||||
"expected status 200 but got #{response.status} (#{uri})"
|
||||
end
|
||||
response.body.read
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -20,7 +20,5 @@
|
|||
}
|
||||
- if swf_asset.canvas_movie?
|
||||
%iframe{src: swf_asset_path(swf_asset, playing: outfit_viewer_is_playing ? true : nil)}
|
||||
- elsif swf_asset.image_url.present?
|
||||
= image_tag swf_asset.image_url, alt: ""
|
||||
- else
|
||||
/ No movie or image available for SWF asset: #{swf_asset.url}
|
||||
= image_tag swf_asset.image_url, alt: ""
|
|
@ -1,102 +0,0 @@
|
|||
require "addressable/template"
|
||||
require "async/http/internet/instance"
|
||||
|
||||
namespace :rainbow_pool do
|
||||
desc "Import all basic image hashes from the Rainbow Pool, onto PetTypes"
|
||||
task :import => :environment do
|
||||
neologin = STDIN.getpass("Neologin cookie: ")
|
||||
|
||||
all_pet_types = PetType.all.to_a
|
||||
all_pet_types_by_species_id_and_color_id = all_pet_types.
|
||||
to_h { |pt| [[pt.species_id, pt.color_id], pt] }
|
||||
all_colors_by_name = Color.all.to_h { |c| [c.human_name.downcase, c] }
|
||||
|
||||
# TODO: Do these in parallel? I set up the HTTP requests to be able to
|
||||
# handle it, and just didn't set up the rest of the code for it, lol
|
||||
Species.order(:name).each do |species|
|
||||
begin
|
||||
hashes_by_color_name = RainbowPool.load_hashes_for_species(
|
||||
species.id, neologin)
|
||||
rescue => error
|
||||
puts "Failed to load #{species.name} page, skipping: #{error.message}"
|
||||
next
|
||||
end
|
||||
|
||||
changed_pet_types = []
|
||||
|
||||
hashes_by_color_name.each do |color_name, image_hash|
|
||||
color = all_colors_by_name[color_name.downcase]
|
||||
if color.nil?
|
||||
puts "Skipping unrecognized color name: #{color_name}"
|
||||
next
|
||||
end
|
||||
|
||||
pet_type = all_pet_types_by_species_id_and_color_id[
|
||||
[species.id, color.id]]
|
||||
if pet_type.nil?
|
||||
puts "Skipping unrecognized pet type: " +
|
||||
"#{color_name} #{species.human_name}"
|
||||
next
|
||||
end
|
||||
|
||||
if pet_type.basic_image_hash.nil?
|
||||
puts "Found new image hash: #{image_hash} (#{pet_type.human_name})"
|
||||
pet_type.basic_image_hash = image_hash
|
||||
changed_pet_types << pet_type
|
||||
elsif pet_type.basic_image_hash != image_hash
|
||||
puts "Updating image hash: #{image_hash} ({#{pet_type.human_name})"
|
||||
pet_type.basic_image_hash = image_hash
|
||||
changed_pet_types << pet_type
|
||||
else
|
||||
# No need to do anything with image hashes that match!
|
||||
end
|
||||
end
|
||||
|
||||
PetType.transaction { changed_pet_types.each(&:save!) }
|
||||
puts "Saved #{changed_pet_types.size} image hashes for " +
|
||||
"#{species.human_name}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module RainbowPool
|
||||
# Share a pool of persistent connections, rather than reconnecting on
|
||||
# each request. (This library does that automatically!)
|
||||
INTERNET = Async::HTTP::Internet.instance
|
||||
|
||||
class << self
|
||||
SPECIES_PAGE_URL_TEMPLATE = Addressable::Template.new(
|
||||
"https://www.neopets.com/pool/all_pb.phtml{?f_species_id}"
|
||||
)
|
||||
def load_hashes_for_species(species_id, neologin)
|
||||
Sync do
|
||||
url = SPECIES_PAGE_URL_TEMPLATE.expand(f_species_id: species_id)
|
||||
INTERNET.get(url, [
|
||||
["User-Agent", Rails.configuration.user_agent_for_neopets],
|
||||
["Cookie", "neologin=#{neologin}"],
|
||||
]) do |response|
|
||||
if response.status != 200
|
||||
raise "expected status 200 but got #{response.status} (#{url})"
|
||||
end
|
||||
|
||||
parse_hashes_from_page response.read
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
IMAGE_HASH_PATTERN = %r{
|
||||
set_pet_img\(
|
||||
'https?://pets\.neopets\.com/cp/(?<hash>[0-9a-z]+)/[0-9]+/[0-9]+\.png',
|
||||
\s*
|
||||
'(?<color_name>.+?)'
|
||||
\)
|
||||
}x
|
||||
def parse_hashes_from_page(html)
|
||||
html.scan(IMAGE_HASH_PATTERN).to_h do |(image_hash, color_name)|
|
||||
[color_name, image_hash]
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
BIN
vendor/cache/async-2.16.1.gem
vendored
Normal file
BIN
vendor/cache/async-2.16.1.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/async-2.17.0.gem
vendored
BIN
vendor/cache/async-2.17.0.gem
vendored
Binary file not shown.
BIN
vendor/cache/async-container-0.16.13.gem
vendored
Normal file
BIN
vendor/cache/async-container-0.16.13.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/async-container-0.18.3.gem
vendored
BIN
vendor/cache/async-container-0.18.3.gem
vendored
Binary file not shown.
BIN
vendor/cache/async-http-0.61.0.gem
vendored
Normal file
BIN
vendor/cache/async-http-0.61.0.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/async-http-0.75.0.gem
vendored
BIN
vendor/cache/async-http-0.75.0.gem
vendored
Binary file not shown.
BIN
vendor/cache/async-io-1.43.2.gem
vendored
Normal file
BIN
vendor/cache/async-io-1.43.2.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/async-service-0.12.0.gem
vendored
BIN
vendor/cache/async-service-0.12.0.gem
vendored
Binary file not shown.
BIN
vendor/cache/build-environment-1.13.0.gem
vendored
Normal file
BIN
vendor/cache/build-environment-1.13.0.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/falcon-0.43.0.gem
vendored
Normal file
BIN
vendor/cache/falcon-0.43.0.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/falcon-0.48.0.gem
vendored
BIN
vendor/cache/falcon-0.48.0.gem
vendored
Binary file not shown.
BIN
vendor/cache/io-endpoint-0.13.1.gem
vendored
BIN
vendor/cache/io-endpoint-0.13.1.gem
vendored
Binary file not shown.
BIN
vendor/cache/io-stream-0.4.0.gem
vendored
BIN
vendor/cache/io-stream-0.4.0.gem
vendored
Binary file not shown.
BIN
vendor/cache/process-metrics-0.2.1.gem
vendored
Normal file
BIN
vendor/cache/process-metrics-0.2.1.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/process-metrics-0.3.0.gem
vendored
BIN
vendor/cache/process-metrics-0.3.0.gem
vendored
Binary file not shown.
BIN
vendor/cache/protocol-http-0.25.0.gem
vendored
Normal file
BIN
vendor/cache/protocol-http-0.25.0.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/protocol-http-0.33.0.gem
vendored
BIN
vendor/cache/protocol-http-0.33.0.gem
vendored
Binary file not shown.
BIN
vendor/cache/protocol-http1-0.16.1.gem
vendored
Normal file
BIN
vendor/cache/protocol-http1-0.16.1.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/protocol-http1-0.22.0.gem
vendored
BIN
vendor/cache/protocol-http1-0.22.0.gem
vendored
Binary file not shown.
BIN
vendor/cache/protocol-http2-0.15.1.gem
vendored
Normal file
BIN
vendor/cache/protocol-http2-0.15.1.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/protocol-http2-0.18.0.gem
vendored
BIN
vendor/cache/protocol-http2-0.18.0.gem
vendored
Binary file not shown.
BIN
vendor/cache/protocol-rack-0.6.0.gem
vendored
Normal file
BIN
vendor/cache/protocol-rack-0.6.0.gem
vendored
Normal file
Binary file not shown.
BIN
vendor/cache/protocol-rack-0.7.0.gem
vendored
BIN
vendor/cache/protocol-rack-0.7.0.gem
vendored
Binary file not shown.
Loading…
Reference in a new issue