moving toward s3 image storage

This commit is contained in:
Emi Matchu 2011-05-20 19:19:14 -04:00
parent 883ecde836
commit b13fd7ae99
30 changed files with 261 additions and 70 deletions

View file

@ -23,7 +23,11 @@ gem 'addressable', :require => ['addressable/template', 'addressable/uri']
gem 'whenever', '~> 0.6.2', :require => false gem 'whenever', '~> 0.6.2', :require => false
gem 'swf_converter', '~> 0.0.1' gem 'swf_converter', '~> 0.0.3'
gem 'resque', '~> 1.15.0'
gem 'right_aws', '~> 2.1.0'
group :development_async do group :development_async do
# async wrappers # async wrappers

View file

@ -74,7 +74,6 @@ GEM
arel (2.0.8) arel (2.0.8)
bcrypt-ruby (2.1.4) bcrypt-ruby (2.1.4)
builder (2.1.2) builder (2.1.2)
chunky_png (1.2.0)
closure-compiler (1.0.0) closure-compiler (1.0.0)
compass (0.10.6) compass (0.10.6)
haml (>= 3.0.4) haml (>= 3.0.4)
@ -98,6 +97,7 @@ GEM
jammit (0.5.4) jammit (0.5.4)
closure-compiler (>= 0.1.0) closure-compiler (>= 0.1.0)
yui-compressor (>= 0.9.1) yui-compressor (>= 0.9.1)
json (1.4.6)
mail (2.2.15) mail (2.2.15)
activesupport (>= 2.3.6) activesupport (>= 2.3.6)
i18n (>= 0.4.0) i18n (>= 0.4.0)
@ -107,8 +107,6 @@ GEM
mime-types (1.16) mime-types (1.16)
msgpack (0.4.4) msgpack (0.4.4)
mysql2 (0.2.6) mysql2 (0.2.6)
oily_png (1.0.1)
chunky_png (~> 1)
openneo-auth-signatory (0.1.0) openneo-auth-signatory (0.1.0)
ruby-hmac ruby-hmac
polyglot (0.3.1) polyglot (0.3.1)
@ -133,6 +131,17 @@ GEM
thor (~> 0.14.4) thor (~> 0.14.4)
rake (0.8.7) rake (0.8.7)
rdiscount (1.6.8) rdiscount (1.6.8)
redis (2.2.0)
redis-namespace (0.10.0)
redis (< 3.0.0)
resque (1.15.0)
json (~> 1.4.6)
redis-namespace (>= 0.10.0)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
right_aws (2.1.0)
right_http_connection (>= 1.2.5)
right_http_connection (1.3.0)
rspec (2.0.1) rspec (2.0.1)
rspec-core (~> 2.0.1) rspec-core (~> 2.0.1)
rspec-expectations (~> 2.0.1) rspec-expectations (~> 2.0.1)
@ -146,17 +155,21 @@ GEM
rspec-rails (2.0.1) rspec-rails (2.0.1)
rspec (~> 2.0.0) rspec (~> 2.0.0)
ruby-hmac (0.4.0) ruby-hmac (0.4.0)
swf_converter (0.0.1) sinatra (1.2.6)
chunky_png (~> 1.2.0) rack (~> 1.1)
oily_png (~> 1.0.1) tilt (< 2.0, >= 1.2.2)
swf_converter (0.0.3)
thin (1.2.7) thin (1.2.7)
daemons (>= 1.0.9) daemons (>= 1.0.9)
eventmachine (>= 0.12.6) eventmachine (>= 0.12.6)
rack (>= 1.0.0) rack (>= 1.0.0)
thor (0.14.6) thor (0.14.6)
tilt (1.3)
treetop (1.4.9) treetop (1.4.9)
polyglot (>= 0.3.1) polyglot (>= 0.3.1)
tzinfo (0.3.24) tzinfo (0.3.24)
vegas (0.1.8)
rack (>= 1.0.0)
warden (1.0.3) warden (1.0.3)
rack (>= 1.0.0) rack (>= 1.0.0)
whenever (0.6.2) whenever (0.6.2)
@ -189,8 +202,10 @@ DEPENDENCIES
rack-fiber_pool rack-fiber_pool
rails (= 3.0.4) rails (= 3.0.4)
rdiscount (~> 1.6.5) rdiscount (~> 1.6.5)
resque (~> 1.15.0)
right_aws (~> 2.1.0)
rspec-rails (~> 2.0.0.beta.22) rspec-rails (~> 2.0.0.beta.22)
swf_converter (~> 0.0.1) swf_converter (~> 0.0.3)
thin (~> 1.2.7) thin (~> 1.2.7)
whenever (~> 0.6.2) whenever (~> 0.6.2)
will_paginate (~> 3.0.pre2) will_paginate (~> 3.0.pre2)

View file

@ -8,3 +8,4 @@ require 'rake/testtask'
require 'rake/rdoctask' require 'rake/rdoctask'
OpenneoImpressItems::Application.load_tasks OpenneoImpressItems::Application.load_tasks

View file

@ -0,0 +1,8 @@
class AssetImageConversionRequestsController < ApplicationController
def create
@swf_asset = SwfAsset.find params[:swf_asset_id]
@swf_asset.request_image_conversion!
render :nothing => true
end
end

View file

@ -30,5 +30,10 @@ class SwfAssetsController < ApplicationController
json ||= @swf_assets ? @swf_assets.all : nil json ||= @swf_assets ? @swf_assets.all : nil
render :json => json render :json => json
end end
def show
@swf_asset = SwfAsset.find params[:id]
render :json => @swf_asset
end
end end

View file

@ -0,0 +1,13 @@
class AssetImageConversionRequest
@queue = :requested_asset_images
def self.perform(asset_id)
asset = SwfAsset.find asset_id
asset.convert_swf_if_not_converted!
end
class OnCreation < AssetImageConversionRequest
@queue = :requested_asset_images_on_creation
end
end

View file

@ -1,8 +1,15 @@
require 'fileutils'
require 'uri'
class SwfAsset < ActiveRecord::Base class SwfAsset < ActiveRecord::Base
PUBLIC_ASSET_DIR = File.join('swfs', 'outfit') PUBLIC_ASSET_DIR = File.join('swfs', 'outfit')
LOCAL_ASSET_DIR = Rails.root.join('public', PUBLIC_ASSET_DIR) LOCAL_ASSET_DIR = Rails.root.join('public', PUBLIC_ASSET_DIR)
PUBLIC_IMAGE_DIR = File.join('images', 'outfit') IMAGE_BUCKET = IMPRESS_S3.bucket('impress-asset-images')
LOCAL_IMAGE_DIR = Rails.root.join('public', PUBLIC_IMAGE_DIR) IMAGE_PERMISSION = 'public-read'
IMAGE_HEADERS = {
'Cache-Control' => 'max-age=315360000',
'Content-Type' => 'image/png'
}
NEOPETS_ASSET_SERVER = 'http://images.neopets.com' NEOPETS_ASSET_SERVER = 'http://images.neopets.com'
set_inheritance_column 'inheritance_type' set_inheritance_column 'inheritance_type'
@ -15,9 +22,47 @@ class SwfAsset < ActiveRecord::Base
end end
def swf_image_path(size) def swf_image_path(size)
base_dir = File.dirname(local_path_within_outfit_swfs) Rails.root.join('tmp', 'asset_images_before_upload', self.id.to_s, "#{size.join 'x'}.png")
image_dir = LOCAL_IMAGE_DIR.join(base_dir).join(id.to_s) end
image_dir.join("#{id}_#{size[0]}x#{size[1]}.png")
def after_swf_conversion(images)
images.each do |size, path|
s3_key = URI.encode("#{self.id}/#{size.join 'x'}.png")
print "Uploading #{s3_key}..."
IMAGE_BUCKET.put(
s3_key,
File.open(path),
{}, # meta headers
IMAGE_PERMISSION, # permission
IMAGE_HEADERS
)
puts "done."
FileUtils.rm path
end
end
def convert_swf_if_not_converted!
if has_image?
false
else
convert_swf!
self.has_image = true
save!
true
end
end
def request_image_conversion!(klass=AssetImageConversionRequest)
if image_requested?
false
else
Resque.enqueue(klass, self.id)
self.image_requested = true
save!
true
end
end end
attr_accessor :item attr_accessor :item
@ -60,7 +105,8 @@ class SwfAsset < ActiveRecord::Base
:body_id => body_id, :body_id => body_id,
:zone_id => zone_id, :zone_id => zone_id,
:zones_restrict => zones_restrict, :zones_restrict => zones_restrict,
:is_body_specific => body_specific? :is_body_specific => body_specific?,
:has_image => has_image?
} }
if options[:for] == 'wardrobe' if options[:for] == 'wardrobe'
json[:local_path] = local_url json[:local_path] = local_url
@ -133,6 +179,10 @@ class SwfAsset < ActiveRecord::Base
self.body_id = 0 if !self.body_specific? || (!self.new_record? && self.body_id_changed?) self.body_id = 0 if !self.body_specific? || (!self.new_record? && self.body_id_changed?)
end end
after_create do
request_image_conversion!(AssetImageConversionRequest::OnCreation)
end
class DownloadError < Exception;end class DownloadError < Exception;end
private private

2
config/.gitignore vendored
View file

@ -1,3 +1,5 @@
aws_s3.yml
database.yml database.yml
deploy.rb deploy.rb
openneo_auth.yml openneo_auth.yml

3
config/aws_s3.sample.yml Normal file
View file

@ -0,0 +1,3 @@
access_key_id: ACCESS_KEY_ID
secret_access_key: SECRET_ACCESS_KEY

View file

@ -0,0 +1,8 @@
require 'yaml'
config = YAML.load_file Rails.root.join('config', 'aws_s3.yml')
access_key_id = config.delete 'access_key_id'
secret_access_key = config.delete 'secret_access_key'
IMPRESS_S3 = RightAws::S3.new access_key_id, secret_access_key, config

View file

@ -22,6 +22,9 @@ OpenneoImpressItems::Application.routes.draw do |map|
end end
resources :outfits, :only => [:show, :create, :update, :destroy] resources :outfits, :only => [:show, :create, :update, :destroy]
resources :pet_attributes, :only => [:index] resources :pet_attributes, :only => [:index]
resources :swf_assets, :only => [:show] do
resources :asset_image_conversion_requests, :path => 'conversions', :only => [:create]
end
match '/users/current-user/outfits' => 'outfits#index', :as => :current_user_outfits match '/users/current-user/outfits' => 'outfits#index', :as => :current_user_outfits

View file

@ -0,0 +1,10 @@
class AddHasImageToSwfAssets < ActiveRecord::Migration
def self.up
add_column :swf_assets, :has_image, :boolean, :null => false, :default => false
end
def self.down
remove_column :swf_assets, :has_image
end
end

View file

@ -0,0 +1,10 @@
class AddImageRequestedToSwfAssets < ActiveRecord::Migration
def self.up
add_column :swf_assets, :image_requested, :boolean, :null => false, :default => false
end
def self.down
remove_column :swf_assets, :image_requested
end
end

View file

@ -10,14 +10,14 @@
# #
# It's strongly recommended to check this file into your version control system. # It's strongly recommended to check this file into your version control system.
ActiveRecord::Schema.define(:version => 20110210222230) do ActiveRecord::Schema.define(:version => 20110515134542) do
create_table "auth_servers", :force => true do |t| create_table "auth_servers", :force => true do |t|
t.string "short_name", :limit => 10, :null => false t.string "short_name", :limit => 10, :null => false
t.string "name", :limit => 40, :null => false t.string "name", :limit => 40, :null => false
t.text "icon", :null => false t.text "icon", :limit => 16777215, :null => false
t.text "gateway", :null => false t.text "gateway", :limit => 16777215, :null => false
t.string "secret", :limit => 64, :null => false t.string "secret", :limit => 64, :null => false
end end
create_table "contributions", :force => true do |t| create_table "contributions", :force => true do |t|
@ -27,6 +27,14 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
t.datetime "created_at", :null => false t.datetime "created_at", :null => false
end end
create_table "forums", :force => true do |t|
t.string "name"
t.text "description"
t.datetime "created_at"
t.datetime "updated_at"
t.integer "sort_index"
end
create_table "item_outfit_relationships", :force => true do |t| create_table "item_outfit_relationships", :force => true do |t|
t.integer "item_id" t.integer "item_id"
t.integer "outfit_id" t.integer "outfit_id"
@ -45,18 +53,18 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
add_index "login_cookies", ["user_id"], :name => "login_cookies_user_id" add_index "login_cookies", ["user_id"], :name => "login_cookies_user_id"
create_table "objects", :force => true do |t| create_table "objects", :force => true do |t|
t.text "zones_restrict", :limit => 255, :null => false t.text "zones_restrict", :null => false
t.text "thumbnail_url", :null => false t.text "thumbnail_url", :limit => 16777215, :null => false
t.string "name", :limit => 100, :null => false t.string "name", :limit => 100, :null => false
t.text "description", :null => false t.text "description", :limit => 16777215, :null => false
t.string "category", :limit => 50 t.string "category", :limit => 50
t.string "type", :limit => 50 t.string "type", :limit => 50
t.string "rarity", :limit => 25 t.string "rarity", :limit => 25
t.integer "rarity_index", :limit => 2 t.integer "rarity_index", :limit => 2
t.integer "price", :limit => 3, :null => false t.integer "price", :limit => 3, :null => false
t.integer "weight_lbs", :limit => 2 t.integer "weight_lbs", :limit => 2
t.text "species_support_ids" t.text "species_support_ids", :limit => 16777215
t.boolean "sold_in_mall", :null => false t.boolean "sold_in_mall", :null => false
t.datetime "last_spidered" t.datetime "last_spidered"
end end
@ -83,14 +91,14 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
add_index "parents_swf_assets", ["swf_asset_id"], :name => "parents_swf_assets_swf_asset_id" add_index "parents_swf_assets", ["swf_asset_id"], :name => "parents_swf_assets_swf_asset_id"
create_table "pet_loads", :force => true do |t| create_table "pet_loads", :force => true do |t|
t.string "pet_name", :limit => 20, :null => false t.string "pet_name", :limit => 20, :null => false
t.text "amf", :null => false t.text "amf", :limit => 16777215, :null => false
t.datetime "created_at", :null => false t.datetime "created_at", :null => false
end end
create_table "pet_states", :force => true do |t| create_table "pet_states", :force => true do |t|
t.integer "pet_type_id", :limit => 3, :null => false t.integer "pet_type_id", :limit => 3, :null => false
t.text "swf_asset_ids", :limit => 255, :null => false t.text "swf_asset_ids", :null => false
end end
add_index "pet_states", ["pet_type_id"], :name => "pet_states_pet_type_id" add_index "pet_states", ["pet_type_id"], :name => "pet_states_pet_type_id"
@ -113,24 +121,43 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
add_index "pets", ["name"], :name => "pets_name", :unique => true add_index "pets", ["name"], :name => "pets_name", :unique => true
add_index "pets", ["pet_type_id"], :name => "pets_pet_type_id" add_index "pets", ["pet_type_id"], :name => "pets_pet_type_id"
create_table "posts", :force => true do |t|
t.integer "topic_id"
t.integer "user_id"
t.text "body"
t.datetime "created_at"
t.datetime "updated_at"
end
create_table "schema_info", :id => false, :force => true do |t| create_table "schema_info", :id => false, :force => true do |t|
t.integer "version", :default => 0, :null => false t.integer "version", :default => 0, :null => false
end end
create_table "swf_assets", :id => false, :force => true do |t| create_table "swf_assets", :id => false, :force => true do |t|
t.string "type", :limit => 7, :null => false t.string "type", :limit => 7, :null => false
t.integer "id", :limit => 3, :null => false t.integer "id", :limit => 3, :null => false
t.text "url", :null => false t.text "url", :limit => 16777215, :null => false
t.integer "zone_id", :limit => 1, :null => false t.integer "zone_id", :limit => 1, :null => false
t.text "zones_restrict", :limit => 255, :null => false t.text "zones_restrict", :null => false
t.datetime "created_at", :null => false t.datetime "created_at", :null => false
t.integer "body_id", :limit => 2, :null => false t.integer "body_id", :limit => 2, :null => false
t.boolean "has_image", :default => false, :null => false
t.boolean "image_requested", :default => false, :null => false
end end
add_index "swf_assets", ["body_id"], :name => "swf_assets_body_id_and_object_id" add_index "swf_assets", ["body_id"], :name => "swf_assets_body_id_and_object_id"
add_index "swf_assets", ["type", "id"], :name => "swf_assets_type_and_id" add_index "swf_assets", ["type", "id"], :name => "swf_assets_type_and_id"
add_index "swf_assets", ["zone_id"], :name => "idx_swf_assets_zone_id" add_index "swf_assets", ["zone_id"], :name => "idx_swf_assets_zone_id"
create_table "topics", :force => true do |t|
t.string "title"
t.integer "user_id"
t.datetime "created_at"
t.datetime "updated_at"
t.integer "forum_id"
t.integer "original_post_id"
end
create_table "users", :force => true do |t| create_table "users", :force => true do |t|
t.string "name", :limit => 20, :null => false t.string "name", :limit => 20, :null => false
t.integer "auth_server_id", :limit => 1, :null => false t.integer "auth_server_id", :limit => 1, :null => false
@ -139,6 +166,8 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
t.boolean "beta", :default => false, :null => false t.boolean "beta", :default => false, :null => false
t.string "remember_token" t.string "remember_token"
t.datetime "remember_created_at" t.datetime "remember_created_at"
t.boolean "forum_admin", :default => false, :null => false
t.boolean "forum_moderator"
end end
create_table "zones", :force => true do |t| create_table "zones", :force => true do |t|

4
lib/tasks/resque.rake Normal file
View file

@ -0,0 +1,4 @@
require 'resque/tasks'
task "resque:setup" => :environment

View file

@ -78,11 +78,16 @@ function Wardrobe() {
function Asset(data) { function Asset(data) {
var asset = this; var asset = this;
for(var key in data) { for(var key in data) {
if(data.hasOwnProperty(key)) { if(data.hasOwnProperty(key)) {
asset[key] = data[key]; asset[key] = data[key];
} }
} }
this.requestImageConversion = function () {
$.post('/swf_assets/' + this.id + '/conversions');
}
} }
function BiologyAsset(data) { function BiologyAsset(data) {
@ -703,8 +708,6 @@ function Wardrobe() {
return pet_type; return pet_type;
} }
function SwfAsset() {}
/* /*
* *
* Controllers * Controllers
@ -1161,43 +1164,66 @@ Wardrobe.getStandardView = function (options) {
StandardView.Preview = function (wardrobe) { StandardView.Preview = function (wardrobe) {
var preview_el = $(options.Preview.wrapper), var preview_el = $(options.Preview.wrapper),
preview_swf_placeholder = $(options.Preview.placeholder), preview_swf_placeholder = $(options.Preview.placeholder);
preview_swf_id = preview_swf_placeholder.attr('id'),
preview_swf,
update_pending_flash = false;
swfobject.embedSWF( function SWFAdapter() {
options.Preview.swf_url, var preview_swf_id = preview_swf_placeholder.attr('id'),
preview_swf_id, preview_swf,
'100%',
'100%',
'9',
'/assets/js/swfobject/expressInstall.swf',
{'id': preview_swf_id},
{'wmode': 'transparent'}
);
Wardrobe.StandardPreview.views_by_swf_id[preview_swf_id] = this;
this.previewSWFIsReady = function () {
preview_swf = document.getElementById(preview_swf_id);
if(update_pending_flash) {
update_pending_flash = false; update_pending_flash = false;
updateAssets();
swfobject.embedSWF(
options.Preview.swf_url,
preview_swf_id,
'100%',
'100%',
'9',
'/assets/js/swfobject/expressInstall.swf',
{'id': preview_swf_id},
{'wmode': 'transparent'}
);
Wardrobe.StandardPreview.views_by_swf_id[preview_swf_id] = this;
this.previewSWFIsReady = function () {
preview_swf = document.getElementById(preview_swf_id);
if(update_pending_flash) {
update_pending_flash = false;
this.updateAssets();
}
}
this.updateAssets = function () {
var assets, assets_for_swf;
if(update_pending_flash) return false;
if(preview_swf && preview_swf.setAssets) {
assets = wardrobe.outfit.getVisibleAssets();
preview_swf.setAssets(assets);
} else {
update_pending_flash = true;
}
} }
} }
function updateAssets() { function ImageAdapter() {
var assets, assets_for_swf; this.updateAssets = function () {
if(update_pending_flash) return false; var assets = wardrobe.outfit.getVisibleAssets(), asset;
if(preview_swf && preview_swf.setAssets) { var imagesPending = 0;
assets = wardrobe.outfit.getVisibleAssets(); for(var i in assets) {
preview_swf.setAssets(assets); if(!assets.hasOwnProperty(i)) continue;
} else { asset = assets[i];
update_pending_flash = true; if(!asset.has_image) {
assets[i].requestImageConversion();
imagesPending++;
}
}
preview_swf_placeholder.text("Waiting on " + imagesPending + " images.");
} }
} }
//this.adapter = new SWFAdapter();
this.adapter = new ImageAdapter();
var updateAssets = $.proxy(this.adapter, 'updateAssets');
wardrobe.outfit.bind('updateWornItems', updateAssets); wardrobe.outfit.bind('updateWornItems', updateAssets);
wardrobe.outfit.bind('updateItemAssets', updateAssets); wardrobe.outfit.bind('updateItemAssets', updateAssets);
wardrobe.outfit.bind('updatePetState', updateAssets); wardrobe.outfit.bind('updatePetState', updateAssets);

View file

Binary file not shown.

BIN
vendor/cache/json-1.4.6.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/redis-2.2.0.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/redis-namespace-0.10.0.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/resque-1.15.0.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/right_aws-2.1.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/sinatra-1.2.6.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/swf_converter-0.0.3.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/tilt-1.3.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/vegas-0.1.8.gem vendored Normal file

Binary file not shown.