Add sidekiq

Gemfile

@@ -18,13 +18,14 @@ gem 'redis'
 gem 'redis-namespace'
 gem 'savon', "~> 2.1.0"
 gem 'coffee-script'
+# Error reporting
+gem 'error-reporter', :git => 'https://github.com/SupportBee/ErrorReporter.git'
 # unicorn 4.1.0 fixes a bug that broke unicorn restarts
 # @see http://mongrel-unicorn.rubyforge.narkive.com/QM9xHegx/ruby-2-0-bad-file-descriptor-errno-ebadf
 gem 'unicorn', '>= 4.1.1'
+gem "sidekiq-pro", "3.4.5", :path => "vendor/gems/sidekiq-pro-3.4.5"
 # Monitoring
 gem 'newrelic_rpm', :require => false
-# Error reporting
-gem 'error-reporter', :git => 'https://github.com/SupportBee/ErrorReporter.git'
 
 # App gems
 gem 'tinder'
@@ -51,7 +52,7 @@ group :development do
   # Deploy gems
   gem 'capistrano', "= 2.15.5"
   gem 'capistrano-ext'
-  gem 'rvm-capistrano'
+  gem 'rvm-capistrano', :require => false
 end
 
 group :test do

Gemfile.lock

@@ -5,6 +5,12 @@ GIT
     error-reporter (0.1.0)
       honeybadger (= 2.2.0)
 
+PATH
+  remote: vendor/gems/sidekiq-pro-3.4.5
+  specs:
+    sidekiq-pro (3.4.5)
+      sidekiq (>= 4.1.5)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -40,6 +46,8 @@ GEM
       coffee-script-source
       execjs
     coffee-script-source (1.3.3)
+    concurrent-ruby (1.0.5)
+    connection_pool (2.2.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.2.5)
@@ -108,16 +116,16 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    rack (1.4.1)
-    rack-protection (1.2.0)
+    rack (1.6.5)
+    rack-protection (1.5.3)
       rack
     rack-test (0.6.1)
       rack (>= 1.0)
     raindrops (0.17.0)
     rake (0.9.2.2)
-    redis (3.0.2)
-    redis-namespace (1.2.1)
-      redis (~> 3.0.0)
+    redis (3.3.3)
+    redis-namespace (1.5.3)
+      redis (~> 3.0, >= 3.0.4)
     rest-client (1.6.7)
       mime-types (>= 1.16)
     restforce (1.4.2)
@@ -161,6 +169,11 @@ GEM
       nokogiri (>= 1.4.0)
       nori (~> 2.0.3)
       wasabi (~> 3.0.0)
+    sidekiq (4.2.10)
+      concurrent-ruby (~> 1.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      rack-protection (>= 1.5.0)
+      redis (~> 3.2, >= 3.2.1)
     simple_oauth (0.1.9)
     sinatra (1.3.2)
       rack (~> 1.3, >= 1.3.6)
@@ -241,6 +254,7 @@ DEPENDENCIES
   rubyzoho (= 0.1.7)
   rvm-capistrano
   savon (~> 2.1.0)
+  sidekiq-pro (= 3.4.5)!
   sinatra
   sinatra-initializers
   thor

config.ru

@@ -3,4 +3,6 @@ require './config/load'
 puts "Preparing Assets..."
 SupportBeeApp::Build.build if PLATFORM_ENV == 'development'
 
-run RunApp
+# run RunApp
+require "sidekiq/web"
+run Rack::URLMap.new('/' => RunApp, '/sidekiq' => Sidekiq::Web)

config/eye/staging.eye

@@ -0,0 +1,23 @@
+RAILS_ENV = ENV['RAILS_ENV'] || 'staging'
+WORKING_DIR = ENV['RAILS_ROOT'] || '/home/rails/apps/supportbee_app_platform/current'
+PID_DIR = "#{WORKING_DIR}/tmp/pids"
+
+Eye.config do
+  logger File.join WORKING_DIR, 'log', 'eye.log'
+end
+
+Eye.application 'AppPlatform' do
+  working_dir WORKING_DIR
+
+  process 'sidekiq' do
+    start_command "rvm-exec ruby-2.2.3 bundle exec sidekiq -C #{WORKING_DIR}/config/sidekiq/staging.yml -r #{WORKING_DIR}/config/load.rb"
+    stop_signals [:USR1, 25.seconds, :TERM, 15.seconds] # See https://github.com/mperham/sidekiq/wiki/Signals
+
+    env 'RAILS_ENV' => RAILS_ENV
+    daemonize true
+    pid_file "#{PID_DIR}/sidekiq.pid"
+
+    stdall "#{WORKING_DIR}/log/sidekiq.log"
+  end
+end
+

config/load.rb

@@ -19,6 +19,7 @@
 Dir["#{PLATFORM_ROOT}/lib/helpers/**/*.rb"].each { |f| require f }
 Dir["#{PLATFORM_ROOT}/lib/*.rb"].each { |f| require f }
 Dir["#{PLATFORM_ROOT}/apps/*/*.rb"].each { |f| require f }
+Dir["#{PLATFORM_ROOT}/workers/*.rb"].each { |f| require f }
 
 app_config = YAML.load_file("#{PLATFORM_ROOT}/config/sba_config.yml")[PLATFORM_ENV]['app_platform']
 SECRET_CONFIG = YAML.load_file("#{PLATFORM_ROOT}/config/secret_config.yml")[PLATFORM_ENV]
@@ -39,13 +40,13 @@
 FileUtils.mkdir(log_dir) unless File.exists?(log_dir)
 LOGGER = Logger.new(log_url)
 
-redis_options = APP_CONFIG['redis']
+REDIS_CONFIG = APP_CONFIG['redis']
 redis = nil
 if PLATFORM_ENV == 'test'
   require 'mock_redis'
-  redis = MockRedis.new(db: redis_options['db'])
+  redis = MockRedis.new(db: REDIS_CONFIG['db'])
 else
-  redis = Redis.new(host: redis_options['host'], db: redis_options['db'])
+  redis = Redis.new(host: REDIS_CONFIG['host'], db: REDIS_CONFIG['db'])
   redis = Redis::Namespace.new(:ap, redis: redis)
 end
 REDIS = redis
@@ -56,3 +57,18 @@
 
 require 'error-reporter'
 require "#{PLATFORM_ROOT}/run_app"
+
+#
+# Sidekiq
+#
+
+SIDEKIQ_REDIS_CONFIG = APP_CONFIG['sidekiq_redis']
+default_redis_port = 6379
+port = SIDEKIQ_REDIS_CONFIG['port'] || default_redis_port
+redis_url = URI::Generic.build(scheme: "redis", host: REDIS_CONFIG['host'], port: port, path: "/#{REDIS_CONFIG['db']}").to_s # redis://127.0.0.1:6379/2
+Sidekiq.configure_server do |config|
+  config.redis = { url: redis_url }
+end
+Sidekiq.configure_client do |config|
+  config.redis = { url: redis_url }
+end

config/sba_config.yml.example

@@ -6,7 +6,11 @@ development:
     cloudfront_base_url: http://localhost:9292
     redis:
       host: '127.0.0.1'
-      db: 1
+      db: 6
+    sidekiq_redis:
+      host: '127.0.0.1'
+      db: 6
+      # port: 6479
 
 test:
   app_platform:
@@ -16,4 +20,8 @@ test:
     cloudfront_base_url: http://localhost:9292
     redis:
       host: '127.0.0.1'
-      db: 2
+      db: 9
+    sidekiq_redis:
+      host: '127.0.0.1'
+      db: 9
+      # port: 6479

config/sidekiq/staging.yml

@@ -0,0 +1,2 @@
+:queues:
+  - app_events

lib/base.rb

@@ -234,6 +234,10 @@ def inherited(app)
       def setup_for(sinatra_app)
         sinatra_app.setup(self)
       end
+
+      def find_from_slug(app_slug)
+        SupportBeeApp::Base.apps.detect { |app_class| app_class.slug == app_slug }
+      end
     end
 
     self.env ||= PLATFORM_ENV

run_app.rb

@@ -58,13 +58,11 @@ def self.setup(app_class)
     end
 
     post "/#{app_class.slug}/event/:event" do
-      data, payload = parse_request
       event = params[:event]
-      if app_class.trigger_event(event, data, payload)
-        status 204
-      else
-        status 500
-      end
+      data, payload = parse_request
+      Sidekiq::Client.enqueue(TriggerAppEvent, app_class.slug, event, data, payload)
+
+      status 204
     end
 
     post "/#{app_class.slug}/action/:action" do

vendor/gems/sidekiq-pro-3.4.5/README.md

@@ -0,0 +1,54 @@
+# Sidekiq Pro
+
+This gem adds advanced functionality and a commercial license for the Sidekiq
+background job framework.
+
+## What's Provided
+
+* **Batch** - adds the notion of a set of jobs, so you can track progress
+  of the batch and receive notification when the batch is complete.  The
+  Sidekiq Web UI provides a convenient overview of all Batches being processed
+  and their status.
+
+* **Reliability** - adds reliability upgrades to the client push to Redis
+  and the server fetch from Redis, to better withstand network outages
+  and process crashes.
+
+* **Much, much more** - Statsd, pause queues, API extensions, expire
+  jobs, etc.
+
+
+## Download
+
+When you purchase Sidekiq Pro, you will receive an email within 24 hours
+with your own personalized download URL.  This URL can be used with a
+Gemfile:
+
+    source 'https://rubygems.org'
+    source 'https://YOUR:[email protected]/'
+
+    gem 'sidekiq-pro'
+
+Please keep this URL private; I do reserve the right to revoke access if
+the URL is made public and/or is being used to illegally download Sidekiq Pro.
+
+
+## Usage
+
+Please see the Sidekiq wiki for in-depth documentation on each Sidekiq
+Pro feature and how to use it.
+
+
+## Licensing
+
+This library is sold commercially to provide support for the development of Sidekiq.
+**The gem file or any of its contents may not be publicly distributed.**
+
+See COMM-LICENSE for the license terms.
+
+
+## Support
+
+Please open an issue in the Sidekiq issue tracker or send an email to
+the sidekiq mailing list.  If you note that you are a Pro user, I will
+make an effort to reply within 24 hours.

vendor/gems/sidekiq-pro-3.4.5/lib/sidekiq-pro.rb

@@ -0,0 +1,82 @@
+require 'sidekiq'
+require 'sidekiq/pro/version'
+require 'sidekiq/pro/worker'
+require 'sidekiq/pro/api'
+require 'sidekiq/pro/push'
+require 'sidekiq/pro/util'
+require 'sidekiq/batch'
+
+Sidekiq.send(:remove_const, :LICENSE)
+Sidekiq.send(:remove_const, :NAME)
+Sidekiq::NAME = "Sidekiq Pro"
+Sidekiq::LICENSE = "Sidekiq Pro #{Sidekiq::Pro::VERSION}, commercially licensed.  Thanks for your support!"
+
+Sidekiq.configure_server do
+  class Sidekiq::CLI
+    def self.banner
+      File.read(File.expand_path(File.join(__FILE__, '../sidekiq/intro.ans')))
+    end
+  end
+  require 'sidekiq/pro/basic_fetch'
+  Sidekiq.options[:fetch] = Sidekiq::Pro::BasicFetch
+end
+
+
+# Enable various reliability add-ons:
+#
+#   Sidekiq.configure_server do |config|
+#     config.reliable_fetch!
+#     config.reliable_scheduler!
+#     config.timed_fetch!
+#     # enable both
+#     config.reliable!
+#   end
+#
+module Sidekiq
+  def self.reliable_fetch!
+    require 'sidekiq/pro/fetch'
+    Sidekiq.options[:fetch] = Sidekiq::Pro::ReliableFetch
+    Sidekiq.options[:ephemeral_hostname] ||= !!ENV['DYNO']
+    env = Sidekiq.options[:environment]
+    Sidekiq.options[:index] ||= 0 if !env || env == 'development'
+    Array(Sidekiq.options[:labels]) << 'reliable'
+    nil
+  end
+
+  def self.super_fetch!
+    require 'sidekiq/pro/super_fetch'
+    Sidekiq.options[:fetch] = Sidekiq::Pro::SuperFetch
+    Array(Sidekiq.options[:labels]) << 'reliable'
+    nil
+  end
+
+  def self.timed_fetch!(timeout = 3600)
+    require 'sidekiq/pro/timed_fetch'
+    Sidekiq.options[:fetch] = Sidekiq::Pro::TimedFetch
+    Array(Sidekiq.options[:labels]) << 'reliable'
+
+    Sidekiq.configure_server do |config|
+      config.on(:startup) do
+        klass = Sidekiq::Pro::TimedFetch::Manager
+        klass.instance = klass.new(Sidekiq.options)
+        klass.instance.timeout = timeout
+      end
+    end
+    nil
+  end
+
+  def self.reliable_scheduler!
+    require 'sidekiq/pro/scheduler'
+    Sidekiq.options[:scheduled_enq] = Sidekiq::Scheduled::FastEnq
+  end
+
+  def self.reliable!
+    reliable_fetch!
+    reliable_scheduler!
+  end
+
+  def self.redis_pool
+    # Slight tweak to allow sharding support
+    Thread.current[:sidekiq_redis_pool] || (@redis ||= Sidekiq::RedisConnection.create)
+  end
+end