1
0
Fork 0
forked from OpenNeo/impress

moving toward s3 image storage

This commit is contained in:
Emi Matchu 2011-05-20 19:19:14 -04:00
parent 883ecde836
commit b13fd7ae99
30 changed files with 261 additions and 70 deletions

View file

@ -23,7 +23,11 @@ gem 'addressable', :require => ['addressable/template', 'addressable/uri']
gem 'whenever', '~> 0.6.2', :require => false
gem 'swf_converter', '~> 0.0.1'
gem 'swf_converter', '~> 0.0.3'
gem 'resque', '~> 1.15.0'
gem 'right_aws', '~> 2.1.0'
group :development_async do
# async wrappers

View file

@ -74,7 +74,6 @@ GEM
arel (2.0.8)
bcrypt-ruby (2.1.4)
builder (2.1.2)
chunky_png (1.2.0)
closure-compiler (1.0.0)
compass (0.10.6)
haml (>= 3.0.4)
@ -98,6 +97,7 @@ GEM
jammit (0.5.4)
closure-compiler (>= 0.1.0)
yui-compressor (>= 0.9.1)
json (1.4.6)
mail (2.2.15)
activesupport (>= 2.3.6)
i18n (>= 0.4.0)
@ -107,8 +107,6 @@ GEM
mime-types (1.16)
msgpack (0.4.4)
mysql2 (0.2.6)
oily_png (1.0.1)
chunky_png (~> 1)
openneo-auth-signatory (0.1.0)
ruby-hmac
polyglot (0.3.1)
@ -133,6 +131,17 @@ GEM
thor (~> 0.14.4)
rake (0.8.7)
rdiscount (1.6.8)
redis (2.2.0)
redis-namespace (0.10.0)
redis (< 3.0.0)
resque (1.15.0)
json (~> 1.4.6)
redis-namespace (>= 0.10.0)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
right_aws (2.1.0)
right_http_connection (>= 1.2.5)
right_http_connection (1.3.0)
rspec (2.0.1)
rspec-core (~> 2.0.1)
rspec-expectations (~> 2.0.1)
@ -146,17 +155,21 @@ GEM
rspec-rails (2.0.1)
rspec (~> 2.0.0)
ruby-hmac (0.4.0)
swf_converter (0.0.1)
chunky_png (~> 1.2.0)
oily_png (~> 1.0.1)
sinatra (1.2.6)
rack (~> 1.1)
tilt (< 2.0, >= 1.2.2)
swf_converter (0.0.3)
thin (1.2.7)
daemons (>= 1.0.9)
eventmachine (>= 0.12.6)
rack (>= 1.0.0)
thor (0.14.6)
tilt (1.3)
treetop (1.4.9)
polyglot (>= 0.3.1)
tzinfo (0.3.24)
vegas (0.1.8)
rack (>= 1.0.0)
warden (1.0.3)
rack (>= 1.0.0)
whenever (0.6.2)
@ -189,8 +202,10 @@ DEPENDENCIES
rack-fiber_pool
rails (= 3.0.4)
rdiscount (~> 1.6.5)
resque (~> 1.15.0)
right_aws (~> 2.1.0)
rspec-rails (~> 2.0.0.beta.22)
swf_converter (~> 0.0.1)
swf_converter (~> 0.0.3)
thin (~> 1.2.7)
whenever (~> 0.6.2)
will_paginate (~> 3.0.pre2)

View file

@ -8,3 +8,4 @@ require 'rake/testtask'
require 'rake/rdoctask'
OpenneoImpressItems::Application.load_tasks

View file

@ -0,0 +1,8 @@
class AssetImageConversionRequestsController < ApplicationController
def create
@swf_asset = SwfAsset.find params[:swf_asset_id]
@swf_asset.request_image_conversion!
render :nothing => true
end
end

View file

@ -30,5 +30,10 @@ class SwfAssetsController < ApplicationController
json ||= @swf_assets ? @swf_assets.all : nil
render :json => json
end
def show
@swf_asset = SwfAsset.find params[:id]
render :json => @swf_asset
end
end

View file

@ -0,0 +1,13 @@
class AssetImageConversionRequest
@queue = :requested_asset_images
def self.perform(asset_id)
asset = SwfAsset.find asset_id
asset.convert_swf_if_not_converted!
end
class OnCreation < AssetImageConversionRequest
@queue = :requested_asset_images_on_creation
end
end

View file

@ -1,8 +1,15 @@
require 'fileutils'
require 'uri'
class SwfAsset < ActiveRecord::Base
PUBLIC_ASSET_DIR = File.join('swfs', 'outfit')
LOCAL_ASSET_DIR = Rails.root.join('public', PUBLIC_ASSET_DIR)
PUBLIC_IMAGE_DIR = File.join('images', 'outfit')
LOCAL_IMAGE_DIR = Rails.root.join('public', PUBLIC_IMAGE_DIR)
IMAGE_BUCKET = IMPRESS_S3.bucket('impress-asset-images')
IMAGE_PERMISSION = 'public-read'
IMAGE_HEADERS = {
'Cache-Control' => 'max-age=315360000',
'Content-Type' => 'image/png'
}
NEOPETS_ASSET_SERVER = 'http://images.neopets.com'
set_inheritance_column 'inheritance_type'
@ -15,9 +22,47 @@ class SwfAsset < ActiveRecord::Base
end
def swf_image_path(size)
base_dir = File.dirname(local_path_within_outfit_swfs)
image_dir = LOCAL_IMAGE_DIR.join(base_dir).join(id.to_s)
image_dir.join("#{id}_#{size[0]}x#{size[1]}.png")
Rails.root.join('tmp', 'asset_images_before_upload', self.id.to_s, "#{size.join 'x'}.png")
end
def after_swf_conversion(images)
images.each do |size, path|
s3_key = URI.encode("#{self.id}/#{size.join 'x'}.png")
print "Uploading #{s3_key}..."
IMAGE_BUCKET.put(
s3_key,
File.open(path),
{}, # meta headers
IMAGE_PERMISSION, # permission
IMAGE_HEADERS
)
puts "done."
FileUtils.rm path
end
end
def convert_swf_if_not_converted!
if has_image?
false
else
convert_swf!
self.has_image = true
save!
true
end
end
def request_image_conversion!(klass=AssetImageConversionRequest)
if image_requested?
false
else
Resque.enqueue(klass, self.id)
self.image_requested = true
save!
true
end
end
attr_accessor :item
@ -60,7 +105,8 @@ class SwfAsset < ActiveRecord::Base
:body_id => body_id,
:zone_id => zone_id,
:zones_restrict => zones_restrict,
:is_body_specific => body_specific?
:is_body_specific => body_specific?,
:has_image => has_image?
}
if options[:for] == 'wardrobe'
json[:local_path] = local_url
@ -133,6 +179,10 @@ class SwfAsset < ActiveRecord::Base
self.body_id = 0 if !self.body_specific? || (!self.new_record? && self.body_id_changed?)
end
after_create do
request_image_conversion!(AssetImageConversionRequest::OnCreation)
end
class DownloadError < Exception;end
private

2
config/.gitignore vendored
View file

@ -1,3 +1,5 @@
aws_s3.yml
database.yml
deploy.rb
openneo_auth.yml

3
config/aws_s3.sample.yml Normal file
View file

@ -0,0 +1,3 @@
access_key_id: ACCESS_KEY_ID
secret_access_key: SECRET_ACCESS_KEY

View file

@ -0,0 +1,8 @@
require 'yaml'
config = YAML.load_file Rails.root.join('config', 'aws_s3.yml')
access_key_id = config.delete 'access_key_id'
secret_access_key = config.delete 'secret_access_key'
IMPRESS_S3 = RightAws::S3.new access_key_id, secret_access_key, config

View file

@ -22,6 +22,9 @@ OpenneoImpressItems::Application.routes.draw do |map|
end
resources :outfits, :only => [:show, :create, :update, :destroy]
resources :pet_attributes, :only => [:index]
resources :swf_assets, :only => [:show] do
resources :asset_image_conversion_requests, :path => 'conversions', :only => [:create]
end
match '/users/current-user/outfits' => 'outfits#index', :as => :current_user_outfits

View file

@ -0,0 +1,10 @@
class AddHasImageToSwfAssets < ActiveRecord::Migration
def self.up
add_column :swf_assets, :has_image, :boolean, :null => false, :default => false
end
def self.down
remove_column :swf_assets, :has_image
end
end

View file

@ -0,0 +1,10 @@
class AddImageRequestedToSwfAssets < ActiveRecord::Migration
def self.up
add_column :swf_assets, :image_requested, :boolean, :null => false, :default => false
end
def self.down
remove_column :swf_assets, :image_requested
end
end

View file

@ -10,14 +10,14 @@
#
# It's strongly recommended to check this file into your version control system.
ActiveRecord::Schema.define(:version => 20110210222230) do
ActiveRecord::Schema.define(:version => 20110515134542) do
create_table "auth_servers", :force => true do |t|
t.string "short_name", :limit => 10, :null => false
t.string "name", :limit => 40, :null => false
t.text "icon", :null => false
t.text "gateway", :null => false
t.string "secret", :limit => 64, :null => false
t.string "short_name", :limit => 10, :null => false
t.string "name", :limit => 40, :null => false
t.text "icon", :limit => 16777215, :null => false
t.text "gateway", :limit => 16777215, :null => false
t.string "secret", :limit => 64, :null => false
end
create_table "contributions", :force => true do |t|
@ -27,6 +27,14 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
t.datetime "created_at", :null => false
end
create_table "forums", :force => true do |t|
t.string "name"
t.text "description"
t.datetime "created_at"
t.datetime "updated_at"
t.integer "sort_index"
end
create_table "item_outfit_relationships", :force => true do |t|
t.integer "item_id"
t.integer "outfit_id"
@ -45,18 +53,18 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
add_index "login_cookies", ["user_id"], :name => "login_cookies_user_id"
create_table "objects", :force => true do |t|
t.text "zones_restrict", :limit => 255, :null => false
t.text "thumbnail_url", :null => false
t.string "name", :limit => 100, :null => false
t.text "description", :null => false
t.text "zones_restrict", :null => false
t.text "thumbnail_url", :limit => 16777215, :null => false
t.string "name", :limit => 100, :null => false
t.text "description", :limit => 16777215, :null => false
t.string "category", :limit => 50
t.string "type", :limit => 50
t.string "rarity", :limit => 25
t.integer "rarity_index", :limit => 2
t.integer "price", :limit => 3, :null => false
t.integer "price", :limit => 3, :null => false
t.integer "weight_lbs", :limit => 2
t.text "species_support_ids"
t.boolean "sold_in_mall", :null => false
t.text "species_support_ids", :limit => 16777215
t.boolean "sold_in_mall", :null => false
t.datetime "last_spidered"
end
@ -83,14 +91,14 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
add_index "parents_swf_assets", ["swf_asset_id"], :name => "parents_swf_assets_swf_asset_id"
create_table "pet_loads", :force => true do |t|
t.string "pet_name", :limit => 20, :null => false
t.text "amf", :null => false
t.datetime "created_at", :null => false
t.string "pet_name", :limit => 20, :null => false
t.text "amf", :limit => 16777215, :null => false
t.datetime "created_at", :null => false
end
create_table "pet_states", :force => true do |t|
t.integer "pet_type_id", :limit => 3, :null => false
t.text "swf_asset_ids", :limit => 255, :null => false
t.integer "pet_type_id", :limit => 3, :null => false
t.text "swf_asset_ids", :null => false
end
add_index "pet_states", ["pet_type_id"], :name => "pet_states_pet_type_id"
@ -113,24 +121,43 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
add_index "pets", ["name"], :name => "pets_name", :unique => true
add_index "pets", ["pet_type_id"], :name => "pets_pet_type_id"
create_table "posts", :force => true do |t|
t.integer "topic_id"
t.integer "user_id"
t.text "body"
t.datetime "created_at"
t.datetime "updated_at"
end
create_table "schema_info", :id => false, :force => true do |t|
t.integer "version", :default => 0, :null => false
end
create_table "swf_assets", :id => false, :force => true do |t|
t.string "type", :limit => 7, :null => false
t.integer "id", :limit => 3, :null => false
t.text "url", :null => false
t.integer "zone_id", :limit => 1, :null => false
t.text "zones_restrict", :limit => 255, :null => false
t.datetime "created_at", :null => false
t.integer "body_id", :limit => 2, :null => false
t.string "type", :limit => 7, :null => false
t.integer "id", :limit => 3, :null => false
t.text "url", :limit => 16777215, :null => false
t.integer "zone_id", :limit => 1, :null => false
t.text "zones_restrict", :null => false
t.datetime "created_at", :null => false
t.integer "body_id", :limit => 2, :null => false
t.boolean "has_image", :default => false, :null => false
t.boolean "image_requested", :default => false, :null => false
end
add_index "swf_assets", ["body_id"], :name => "swf_assets_body_id_and_object_id"
add_index "swf_assets", ["type", "id"], :name => "swf_assets_type_and_id"
add_index "swf_assets", ["zone_id"], :name => "idx_swf_assets_zone_id"
create_table "topics", :force => true do |t|
t.string "title"
t.integer "user_id"
t.datetime "created_at"
t.datetime "updated_at"
t.integer "forum_id"
t.integer "original_post_id"
end
create_table "users", :force => true do |t|
t.string "name", :limit => 20, :null => false
t.integer "auth_server_id", :limit => 1, :null => false
@ -139,6 +166,8 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
t.boolean "beta", :default => false, :null => false
t.string "remember_token"
t.datetime "remember_created_at"
t.boolean "forum_admin", :default => false, :null => false
t.boolean "forum_moderator"
end
create_table "zones", :force => true do |t|

4
lib/tasks/resque.rake Normal file
View file

@ -0,0 +1,4 @@
require 'resque/tasks'
task "resque:setup" => :environment

View file

@ -78,11 +78,16 @@ function Wardrobe() {
function Asset(data) {
var asset = this;
for(var key in data) {
if(data.hasOwnProperty(key)) {
asset[key] = data[key];
}
}
this.requestImageConversion = function () {
$.post('/swf_assets/' + this.id + '/conversions');
}
}
function BiologyAsset(data) {
@ -703,8 +708,6 @@ function Wardrobe() {
return pet_type;
}
function SwfAsset() {}
/*
*
* Controllers
@ -1161,43 +1164,66 @@ Wardrobe.getStandardView = function (options) {
StandardView.Preview = function (wardrobe) {
var preview_el = $(options.Preview.wrapper),
preview_swf_placeholder = $(options.Preview.placeholder),
preview_swf_id = preview_swf_placeholder.attr('id'),
preview_swf,
update_pending_flash = false;
preview_swf_placeholder = $(options.Preview.placeholder);
swfobject.embedSWF(
options.Preview.swf_url,
preview_swf_id,
'100%',
'100%',
'9',
'/assets/js/swfobject/expressInstall.swf',
{'id': preview_swf_id},
{'wmode': 'transparent'}
);
Wardrobe.StandardPreview.views_by_swf_id[preview_swf_id] = this;
this.previewSWFIsReady = function () {
preview_swf = document.getElementById(preview_swf_id);
if(update_pending_flash) {
function SWFAdapter() {
var preview_swf_id = preview_swf_placeholder.attr('id'),
preview_swf,
update_pending_flash = false;
updateAssets();
swfobject.embedSWF(
options.Preview.swf_url,
preview_swf_id,
'100%',
'100%',
'9',
'/assets/js/swfobject/expressInstall.swf',
{'id': preview_swf_id},
{'wmode': 'transparent'}
);
Wardrobe.StandardPreview.views_by_swf_id[preview_swf_id] = this;
this.previewSWFIsReady = function () {
preview_swf = document.getElementById(preview_swf_id);
if(update_pending_flash) {
update_pending_flash = false;
this.updateAssets();
}
}
this.updateAssets = function () {
var assets, assets_for_swf;
if(update_pending_flash) return false;
if(preview_swf && preview_swf.setAssets) {
assets = wardrobe.outfit.getVisibleAssets();
preview_swf.setAssets(assets);
} else {
update_pending_flash = true;
}
}
}
function updateAssets() {
var assets, assets_for_swf;
if(update_pending_flash) return false;
if(preview_swf && preview_swf.setAssets) {
assets = wardrobe.outfit.getVisibleAssets();
preview_swf.setAssets(assets);
} else {
update_pending_flash = true;
function ImageAdapter() {
this.updateAssets = function () {
var assets = wardrobe.outfit.getVisibleAssets(), asset;
var imagesPending = 0;
for(var i in assets) {
if(!assets.hasOwnProperty(i)) continue;
asset = assets[i];
if(!asset.has_image) {
assets[i].requestImageConversion();
imagesPending++;
}
}
preview_swf_placeholder.text("Waiting on " + imagesPending + " images.");
}
}
//this.adapter = new SWFAdapter();
this.adapter = new ImageAdapter();
var updateAssets = $.proxy(this.adapter, 'updateAssets');
wardrobe.outfit.bind('updateWornItems', updateAssets);
wardrobe.outfit.bind('updateItemAssets', updateAssets);
wardrobe.outfit.bind('updatePetState', updateAssets);

View file

Binary file not shown.

BIN
vendor/cache/json-1.4.6.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/redis-2.2.0.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/redis-namespace-0.10.0.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/resque-1.15.0.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/right_aws-2.1.0.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/sinatra-1.2.6.gem vendored Normal file

Binary file not shown.

Binary file not shown.

BIN
vendor/cache/swf_converter-0.0.3.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/tilt-1.3.gem vendored Normal file

Binary file not shown.

BIN
vendor/cache/vegas-0.1.8.gem vendored Normal file

Binary file not shown.