moving toward s3 image storage

Gemfile

@@ -23,7 +23,11 @@ gem 'addressable', :require => ['addressable/template', 'addressable/uri']
 
 gem 'whenever', '~> 0.6.2', :require => false
 
-gem 'swf_converter', '~> 0.0.1'
+gem 'swf_converter', '~> 0.0.3'
+
+gem 'resque', '~> 1.15.0'
+
+gem 'right_aws', '~> 2.1.0'
 
 group :development_async do
   # async wrappers

Gemfile.lock

@@ -74,7 +74,6 @@ GEM
     arel (2.0.8)
     bcrypt-ruby (2.1.4)
     builder (2.1.2)
-    chunky_png (1.2.0)
     closure-compiler (1.0.0)
     compass (0.10.6)
       haml (>= 3.0.4)
@@ -98,6 +97,7 @@ GEM
     jammit (0.5.4)
       closure-compiler (>= 0.1.0)
       yui-compressor (>= 0.9.1)
+    json (1.4.6)
     mail (2.2.15)
       activesupport (>= 2.3.6)
       i18n (>= 0.4.0)
@@ -107,8 +107,6 @@ GEM
     mime-types (1.16)
     msgpack (0.4.4)
     mysql2 (0.2.6)
-    oily_png (1.0.1)
-      chunky_png (~> 1)
     openneo-auth-signatory (0.1.0)
       ruby-hmac
     polyglot (0.3.1)
@@ -133,6 +131,17 @@ GEM
       thor (~> 0.14.4)
     rake (0.8.7)
     rdiscount (1.6.8)
+    redis (2.2.0)
+    redis-namespace (0.10.0)
+      redis (< 3.0.0)
+    resque (1.15.0)
+      json (~> 1.4.6)
+      redis-namespace (>= 0.10.0)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    right_aws (2.1.0)
+      right_http_connection (>= 1.2.5)
+    right_http_connection (1.3.0)
     rspec (2.0.1)
       rspec-core (~> 2.0.1)
       rspec-expectations (~> 2.0.1)
@@ -146,17 +155,21 @@ GEM
     rspec-rails (2.0.1)
       rspec (~> 2.0.0)
     ruby-hmac (0.4.0)
-    swf_converter (0.0.1)
-      chunky_png (~> 1.2.0)
-      oily_png (~> 1.0.1)
+    sinatra (1.2.6)
+      rack (~> 1.1)
+      tilt (< 2.0, >= 1.2.2)
+    swf_converter (0.0.3)
     thin (1.2.7)
       daemons (>= 1.0.9)
       eventmachine (>= 0.12.6)
       rack (>= 1.0.0)
     thor (0.14.6)
+    tilt (1.3)
     treetop (1.4.9)
       polyglot (>= 0.3.1)
     tzinfo (0.3.24)
+    vegas (0.1.8)
+      rack (>= 1.0.0)
     warden (1.0.3)
       rack (>= 1.0.0)
     whenever (0.6.2)
@@ -189,8 +202,10 @@ DEPENDENCIES
   rack-fiber_pool
   rails (= 3.0.4)
   rdiscount (~> 1.6.5)
+  resque (~> 1.15.0)
+  right_aws (~> 2.1.0)
   rspec-rails (~> 2.0.0.beta.22)
-  swf_converter (~> 0.0.1)
+  swf_converter (~> 0.0.3)
   thin (~> 1.2.7)
   whenever (~> 0.6.2)
   will_paginate (~> 3.0.pre2)

Rakefile

@@ -8,3 +8,4 @@ require 'rake/testtask'
 require 'rake/rdoctask'
 
 OpenneoImpressItems::Application.load_tasks
+

app/controllers/asset_image_conversion_requests_controller.rb

@@ -0,0 +1,8 @@
+class AssetImageConversionRequestsController < ApplicationController
+  def create
+    @swf_asset = SwfAsset.find params[:swf_asset_id]
+    @swf_asset.request_image_conversion!
+    render :nothing => true
+  end
+end
+

app/controllers/swf_assets_controller.rb

@@ -30,5 +30,10 @@ class SwfAssetsController < ApplicationController
     json ||= @swf_assets ? @swf_assets.all : nil
     render :json => json
   end
+
+  def show
+    @swf_asset = SwfAsset.find params[:id]
+    render :json => @swf_asset
+  end
 end
 

app/models/asset_image_conversion_request.rb

@@ -0,0 +1,13 @@
+class AssetImageConversionRequest
+  @queue = :requested_asset_images
+
+  def self.perform(asset_id)
+    asset = SwfAsset.find asset_id
+    asset.convert_swf_if_not_converted!
+  end
+
+  class OnCreation < AssetImageConversionRequest
+    @queue = :requested_asset_images_on_creation
+  end
+end
+

app/models/swf_asset.rb

@@ -1,8 +1,15 @@
+require 'fileutils'
+require 'uri'
+
 class SwfAsset < ActiveRecord::Base
   PUBLIC_ASSET_DIR = File.join('swfs', 'outfit')
   LOCAL_ASSET_DIR = Rails.root.join('public', PUBLIC_ASSET_DIR)
-  PUBLIC_IMAGE_DIR = File.join('images', 'outfit')
-  LOCAL_IMAGE_DIR = Rails.root.join('public', PUBLIC_IMAGE_DIR)
+  IMAGE_BUCKET = IMPRESS_S3.bucket('impress-asset-images')
+  IMAGE_PERMISSION = 'public-read'
+  IMAGE_HEADERS = {
+    'Cache-Control' => 'max-age=315360000',
+    'Content-Type' => 'image/png'
+  }
   NEOPETS_ASSET_SERVER = 'http://images.neopets.com'
 
   set_inheritance_column 'inheritance_type'
@@ -15,9 +22,47 @@ class SwfAsset < ActiveRecord::Base
   end
 
   def swf_image_path(size)
-    base_dir = File.dirname(local_path_within_outfit_swfs)
-    image_dir = LOCAL_IMAGE_DIR.join(base_dir).join(id.to_s)
-    image_dir.join("#{id}_#{size[0]}x#{size[1]}.png")
+    Rails.root.join('tmp', 'asset_images_before_upload', self.id.to_s, "#{size.join 'x'}.png")
+  end
+
+  def after_swf_conversion(images)
+    images.each do |size, path|
+      s3_key = URI.encode("#{self.id}/#{size.join 'x'}.png")
+
+      print "Uploading #{s3_key}..."
+      IMAGE_BUCKET.put(
+        s3_key,
+        File.open(path),
+        {}, # meta headers
+        IMAGE_PERMISSION, # permission
+        IMAGE_HEADERS
+      )
+      puts "done."
+
+      FileUtils.rm path
+    end
+  end
+
+  def convert_swf_if_not_converted!
+    if has_image?
+      false
+    else
+      convert_swf!
+      self.has_image = true
+      save!
+      true
+    end
+  end
+
+  def request_image_conversion!(klass=AssetImageConversionRequest)
+    if image_requested?
+      false
+    else
+      Resque.enqueue(klass, self.id)
+      self.image_requested = true
+      save!
+      true
+    end
   end
 
   attr_accessor :item
@@ -60,7 +105,8 @@ class SwfAsset < ActiveRecord::Base
       :body_id => body_id,
       :zone_id => zone_id,
       :zones_restrict => zones_restrict,
-      :is_body_specific => body_specific?
+      :is_body_specific => body_specific?,
+      :has_image => has_image?
     }
     if options[:for] == 'wardrobe'
       json[:local_path] = local_url
@@ -133,6 +179,10 @@ class SwfAsset < ActiveRecord::Base
     self.body_id = 0 if !self.body_specific? || (!self.new_record? && self.body_id_changed?)
   end
 
+  after_create do
+    request_image_conversion!(AssetImageConversionRequest::OnCreation)
+  end
+
   class DownloadError < Exception;end
 
   private

config/.gitignore

@@ -1,3 +1,5 @@
+aws_s3.yml
 database.yml
 deploy.rb
 openneo_auth.yml
+

config/aws_s3.sample.yml

@@ -0,0 +1,3 @@
+access_key_id: ACCESS_KEY_ID
+secret_access_key: SECRET_ACCESS_KEY
+

config/initializers/aws_s3.rb

@@ -0,0 +1,8 @@
+require 'yaml'
+
+config = YAML.load_file Rails.root.join('config', 'aws_s3.yml')
+access_key_id = config.delete 'access_key_id'
+secret_access_key = config.delete 'secret_access_key'
+
+IMPRESS_S3 = RightAws::S3.new access_key_id, secret_access_key, config
+

config/routes.rb

@@ -22,6 +22,9 @@ OpenneoImpressItems::Application.routes.draw do |map|
   end
   resources :outfits, :only => [:show, :create, :update, :destroy]
   resources :pet_attributes, :only => [:index]
+  resources :swf_assets, :only => [:show] do
+    resources :asset_image_conversion_requests, :path => 'conversions', :only => [:create]
+  end
 
   match '/users/current-user/outfits' => 'outfits#index', :as => :current_user_outfits
 

db/migrate/20110514145602_add_has_image_to_swf_assets.rb

@@ -0,0 +1,10 @@
+class AddHasImageToSwfAssets < ActiveRecord::Migration
+  def self.up
+    add_column :swf_assets, :has_image, :boolean, :null => false, :default => false
+  end
+
+  def self.down
+    remove_column :swf_assets, :has_image
+  end
+end
+

db/migrate/20110515134542_add_image_requested_to_swf_assets.rb

@@ -0,0 +1,10 @@
+class AddImageRequestedToSwfAssets < ActiveRecord::Migration
+  def self.up
+    add_column :swf_assets, :image_requested, :boolean, :null => false, :default => false
+  end
+
+  def self.down
+    remove_column :swf_assets, :image_requested
+  end
+end
+

db/schema.rb

@@ -10,14 +10,14 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20110210222230) do
+ActiveRecord::Schema.define(:version => 20110515134542) do
 
   create_table "auth_servers", :force => true do |t|
-    t.string "short_name", :limit => 10, :null => false
-    t.string "name",       :limit => 40, :null => false
-    t.text   "icon",                     :null => false
-    t.text   "gateway",                  :null => false
-    t.string "secret",     :limit => 64, :null => false
+    t.string "short_name", :limit => 10,       :null => false
+    t.string "name",       :limit => 40,       :null => false
+    t.text   "icon",       :limit => 16777215, :null => false
+    t.text   "gateway",    :limit => 16777215, :null => false
+    t.string "secret",     :limit => 64,       :null => false
   end
 
   create_table "contributions", :force => true do |t|
@@ -27,6 +27,14 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
     t.datetime "created_at",                    :null => false
   end
 
+  create_table "forums", :force => true do |t|
+    t.string   "name"
+    t.text     "description"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.integer  "sort_index"
+  end
+
   create_table "item_outfit_relationships", :force => true do |t|
     t.integer  "item_id"
     t.integer  "outfit_id"
@@ -45,18 +53,18 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
   add_index "login_cookies", ["user_id"], :name => "login_cookies_user_id"
 
   create_table "objects", :force => true do |t|
-    t.text     "zones_restrict",      :limit => 255, :null => false
-    t.text     "thumbnail_url",                      :null => false
-    t.string   "name",                :limit => 100, :null => false
-    t.text     "description",                        :null => false
+    t.text     "zones_restrict",                          :null => false
+    t.text     "thumbnail_url",       :limit => 16777215, :null => false
+    t.string   "name",                :limit => 100,      :null => false
+    t.text     "description",         :limit => 16777215, :null => false
     t.string   "category",            :limit => 50
     t.string   "type",                :limit => 50
     t.string   "rarity",              :limit => 25
     t.integer  "rarity_index",        :limit => 2
-    t.integer  "price",               :limit => 3,   :null => false
+    t.integer  "price",               :limit => 3,        :null => false
     t.integer  "weight_lbs",          :limit => 2
-    t.text     "species_support_ids"
-    t.boolean  "sold_in_mall",                       :null => false
+    t.text     "species_support_ids", :limit => 16777215
+    t.boolean  "sold_in_mall",                            :null => false
     t.datetime "last_spidered"
   end
 
@@ -83,14 +91,14 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
   add_index "parents_swf_assets", ["swf_asset_id"], :name => "parents_swf_assets_swf_asset_id"
 
   create_table "pet_loads", :force => true do |t|
-    t.string   "pet_name",   :limit => 20, :null => false
-    t.text     "amf",                      :null => false
-    t.datetime "created_at",               :null => false
+    t.string   "pet_name",   :limit => 20,       :null => false
+    t.text     "amf",        :limit => 16777215, :null => false
+    t.datetime "created_at",                     :null => false
   end
 
   create_table "pet_states", :force => true do |t|
-    t.integer "pet_type_id",   :limit => 3,   :null => false
-    t.text    "swf_asset_ids", :limit => 255, :null => false
+    t.integer "pet_type_id",   :limit => 3, :null => false
+    t.text    "swf_asset_ids",              :null => false
   end
 
   add_index "pet_states", ["pet_type_id"], :name => "pet_states_pet_type_id"
@@ -113,24 +121,43 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
   add_index "pets", ["name"], :name => "pets_name", :unique => true
   add_index "pets", ["pet_type_id"], :name => "pets_pet_type_id"
 
+  create_table "posts", :force => true do |t|
+    t.integer  "topic_id"
+    t.integer  "user_id"
+    t.text     "body"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
   create_table "schema_info", :id => false, :force => true do |t|
     t.integer "version", :default => 0, :null => false
   end
 
   create_table "swf_assets", :id => false, :force => true do |t|
-    t.string   "type",           :limit => 7,   :null => false
-    t.integer  "id",             :limit => 3,   :null => false
-    t.text     "url",                           :null => false
-    t.integer  "zone_id",        :limit => 1,   :null => false
-    t.text     "zones_restrict", :limit => 255, :null => false
-    t.datetime "created_at",                    :null => false
-    t.integer  "body_id",        :limit => 2,   :null => false
+    t.string   "type",            :limit => 7,                           :null => false
+    t.integer  "id",              :limit => 3,                           :null => false
+    t.text     "url",             :limit => 16777215,                    :null => false
+    t.integer  "zone_id",         :limit => 1,                           :null => false
+    t.text     "zones_restrict",                                         :null => false
+    t.datetime "created_at",                                             :null => false
+    t.integer  "body_id",         :limit => 2,                           :null => false
+    t.boolean  "has_image",                           :default => false, :null => false
+    t.boolean  "image_requested",                     :default => false, :null => false
   end
 
   add_index "swf_assets", ["body_id"], :name => "swf_assets_body_id_and_object_id"
   add_index "swf_assets", ["type", "id"], :name => "swf_assets_type_and_id"
   add_index "swf_assets", ["zone_id"], :name => "idx_swf_assets_zone_id"
 
+  create_table "topics", :force => true do |t|
+    t.string   "title"
+    t.integer  "user_id"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.integer  "forum_id"
+    t.integer  "original_post_id"
+  end
+
   create_table "users", :force => true do |t|
     t.string   "name",                :limit => 20,                    :null => false
     t.integer  "auth_server_id",      :limit => 1,                     :null => false
@@ -139,6 +166,8 @@ ActiveRecord::Schema.define(:version => 20110210222230) do
     t.boolean  "beta",                              :default => false, :null => false
     t.string   "remember_token"
     t.datetime "remember_created_at"
+    t.boolean  "forum_admin",                       :default => false, :null => false
+    t.boolean  "forum_moderator"
   end
 
   create_table "zones", :force => true do |t|

lib/tasks/resque.rake

@@ -0,0 +1,4 @@
+require 'resque/tasks'
+
+task "resque:setup" => :environment
+

public/javascripts/wardrobe.js

@@ -78,11 +78,16 @@ function Wardrobe() {
 
   function Asset(data) {
     var asset = this;
+
     for(var key in data) {
       if(data.hasOwnProperty(key)) {
         asset[key] = data[key];
       }
     }
+
+    this.requestImageConversion = function () {
+      $.post('/swf_assets/' + this.id + '/conversions');
+    }
   }
 
   function BiologyAsset(data) {
@@ -703,8 +708,6 @@ function Wardrobe() {
     return pet_type;
   }
 
-  function SwfAsset() {}
-
   /*
   *
   * Controllers
@@ -1161,43 +1164,66 @@ Wardrobe.getStandardView = function (options) {
 
   StandardView.Preview = function (wardrobe) {
     var preview_el = $(options.Preview.wrapper),
-      preview_swf_placeholder = $(options.Preview.placeholder),
-      preview_swf_id = preview_swf_placeholder.attr('id'),
-      preview_swf,
-      update_pending_flash = false;
+      preview_swf_placeholder = $(options.Preview.placeholder);
 
-    swfobject.embedSWF(
-      options.Preview.swf_url,
-      preview_swf_id,
-      '100%',
-      '100%',
-      '9',
-      '/assets/js/swfobject/expressInstall.swf',
-      {'id': preview_swf_id},
-      {'wmode': 'transparent'}
-    );
-
-    Wardrobe.StandardPreview.views_by_swf_id[preview_swf_id] = this;
-
-    this.previewSWFIsReady = function () {
-      preview_swf = document.getElementById(preview_swf_id);
-      if(update_pending_flash) {
+    function SWFAdapter() {
+      var preview_swf_id = preview_swf_placeholder.attr('id'),
+        preview_swf,
         update_pending_flash = false;
-        updateAssets();
+
+      swfobject.embedSWF(
+        options.Preview.swf_url,
+        preview_swf_id,
+        '100%',
+        '100%',
+        '9',
+        '/assets/js/swfobject/expressInstall.swf',
+        {'id': preview_swf_id},
+        {'wmode': 'transparent'}
+      );
+
+      Wardrobe.StandardPreview.views_by_swf_id[preview_swf_id] = this;
+
+      this.previewSWFIsReady = function () {
+        preview_swf = document.getElementById(preview_swf_id);
+        if(update_pending_flash) {
+          update_pending_flash = false;
+          this.updateAssets();
+        }
+      }
+
+      this.updateAssets = function () {
+        var assets, assets_for_swf;
+        if(update_pending_flash) return false;
+        if(preview_swf && preview_swf.setAssets) {
+          assets = wardrobe.outfit.getVisibleAssets();
+          preview_swf.setAssets(assets);
+        } else {
+          update_pending_flash = true;
+        }
       }
     }
 
-    function updateAssets() {
-      var assets, assets_for_swf;
-      if(update_pending_flash) return false;
-      if(preview_swf && preview_swf.setAssets) {
-        assets = wardrobe.outfit.getVisibleAssets();
-        preview_swf.setAssets(assets);
-      } else {
-        update_pending_flash = true;
+    function ImageAdapter() {
+      this.updateAssets = function () {
+        var assets = wardrobe.outfit.getVisibleAssets(), asset;
+        var imagesPending = 0;
+        for(var i in assets) {
+          if(!assets.hasOwnProperty(i)) continue;
+          asset = assets[i];
+          if(!asset.has_image) {
+            assets[i].requestImageConversion();
+            imagesPending++;
+          }
+        }
+        preview_swf_placeholder.text("Waiting on " + imagesPending + " images.");
       }
     }
 
+    //this.adapter = new SWFAdapter();
+    this.adapter = new ImageAdapter();
+
+    var updateAssets = $.proxy(this.adapter, 'updateAssets');
     wardrobe.outfit.bind('updateWornItems', updateAssets);
     wardrobe.outfit.bind('updateItemAssets', updateAssets);
     wardrobe.outfit.bind('updatePetState', updateAssets);