Use new DTIRequests.load_many
helper in SwfAsset.preload_manifests
This commit is contained in:
parent
2be21a9238
commit
5546d6df5d
1 changed files with 3 additions and 21 deletions
|
@ -1,7 +1,4 @@
|
||||||
require 'addressable/template'
|
require 'addressable/template'
|
||||||
require 'async'
|
|
||||||
require 'async/barrier'
|
|
||||||
require 'async/semaphore'
|
|
||||||
|
|
||||||
class SwfAsset < ApplicationRecord
|
class SwfAsset < ApplicationRecord
|
||||||
# We use the `type` column to mean something other than what Rails means!
|
# We use the `type` column to mean something other than what Rails means!
|
||||||
|
@ -335,19 +332,9 @@ class SwfAsset < ApplicationRecord
|
||||||
# Given a list of SWF assets, ensure all of their manifests are loaded, with
|
# Given a list of SWF assets, ensure all of their manifests are loaded, with
|
||||||
# fast concurrent execution!
|
# fast concurrent execution!
|
||||||
def self.preload_manifests(swf_assets)
|
def self.preload_manifests(swf_assets)
|
||||||
# Blocks all tasks beneath it.
|
DTIRequests.load_many(max_at_once: 10) do |task|
|
||||||
barrier = Async::Barrier.new
|
swf_assets.each do |swf_asset|
|
||||||
|
task.async do
|
||||||
Sync do
|
|
||||||
# Only allow 10 manifests to be loaded at a time.
|
|
||||||
semaphore = Async::Semaphore.new(10, parent: barrier)
|
|
||||||
|
|
||||||
# Load all the manifests in async tasks. This will load them 10 at a time
|
|
||||||
# rather than all at once (because of the semaphore), and the
|
|
||||||
# NeopetsMediaArchive will share a pool of persistent connections for
|
|
||||||
# them.
|
|
||||||
swf_assets.map do |swf_asset|
|
|
||||||
semaphore.async do
|
|
||||||
begin
|
begin
|
||||||
# Don't save changes in this big async situation; we'll do it all
|
# Don't save changes in this big async situation; we'll do it all
|
||||||
# in one batch after, to avoid too much database concurrency!
|
# in one batch after, to avoid too much database concurrency!
|
||||||
|
@ -358,11 +345,6 @@ class SwfAsset < ApplicationRecord
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# Wait until all tasks are done.
|
|
||||||
barrier.wait
|
|
||||||
ensure
|
|
||||||
barrier.stop # If something goes wrong, clean up all tasks.
|
|
||||||
end
|
end
|
||||||
|
|
||||||
SwfAsset.transaction do
|
SwfAsset.transaction do
|
||||||
|
|
Loading…
Reference in a new issue