Merge branch 'feature/52448_kill_sidekiq' into release/undev_v5.4.0

* feature/52448_kill_sidekiq:
  Fix weekly and monthly workers
  fix mistake
  Make another logic for send digests
  Fix resque-scheduler
  fix workers
  save

Conflicts:
	Gemfile.lock

Gemfile

@@ -129,8 +129,15 @@ gem "acts-as-taggable-on"
 # Background jobs
 gem 'slim'
 gem 'sinatra', require: nil
-gem 'sidekiq', '~> 3.0.1'
-gem 'sidetiq', github: 'tobiassvn/sidetiq'
+
+# Resque 2.x
+# gem "resque", "~> 2.0.0.pre.1", github: "resque/resque"
+# gem 'resque-web', github: 'resque/resque-web', branch: "resque-2", require: 'resque_web'
+# Resque 1.25
+gem "resque"
+gem 'resque-scheduler'
+gem 'resque-web', require: 'resque_web'
+gem 'resque-multi-job-forks'
 
 # HTTP requests
 gem "httparty"

Gemfile.lock

@@ -14,15 +14,6 @@ GIT
   specs:
     elasticsearch-rails (0.1.0.rc1)
 
-GIT
-  remote: git://github.com/tobiassvn/sidetiq.git
-  revision: 4f7d7daea3873443b17bf2f3da61619532bbeba2
-  specs:
-    sidetiq (0.5.0)
-      celluloid (>= 0.14.1)
-      ice_cube (~> 0.11.0)
-      sidekiq (>= 2.16.0)
-
 GIT
   remote: git://github.com/zzet/elasticsearch-git.git
   revision: a3fb57b303ad8b8c629b301ca009cd7d486089fb
@@ -136,7 +127,6 @@ GEM
     coffee-script-source (1.6.3)
     colored (1.2)
     colorize (0.5.8)
-    connection_pool (2.0.0)
     cookiejar (0.3.2)
     coveralls (0.7.0)
       multi_json (~> 1.3)
@@ -310,7 +300,6 @@ GEM
       multi_xml (>= 0.5.2)
     httpauth (0.2.0)
     i18n (0.6.11)
-    ice_cube (0.11.3)
     ice_nine (0.10.0)
     jasmine (2.0.2)
       jasmine-core (~> 2.0.0)
@@ -359,6 +348,7 @@ GEM
     mime-types (1.25.1)
     mini_portile (0.6.0)
     minitest (5.3.5)
+    mono_logger (1.1.0)
     multi_json (1.10.1)
     multi_xml (0.5.5)
     multipart-post (1.2.0)
@@ -487,8 +477,8 @@ GEM
     redis-activesupport (4.0.0)
       activesupport (~> 4)
       redis-store (~> 1.1.0)
-    redis-namespace (1.4.1)
-      redis (~> 3.0.4)
+    redis-namespace (1.5.1)
+      redis (~> 3.0, >= 3.0.4)
     redis-rack (1.5.0)
       rack (~> 1.5)
       redis-store (~> 1.1.0)
@@ -501,6 +491,26 @@ GEM
     ref (1.0.5)
     request_store (1.0.5)
     require_all (1.3.2)
+    resque (1.25.2)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.3)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    resque-multi-job-forks (0.4.2)
+      json
+      resque (~> 1.22)
+    resque-scheduler (3.0.0)
+      mono_logger (~> 1.0)
+      redis (~> 3.0)
+      resque (~> 1.25)
+      rufus-scheduler (~> 2.0)
+    resque-web (0.0.6)
+      coffee-rails
+      jquery-rails
+      resque
+      sass-rails
+      twitter-bootstrap-rails
     rest-client (1.6.7)
       mime-types (>= 1.16)
     rouge (1.3.3)
@@ -522,6 +532,8 @@ GEM
     ruby-progressbar (1.2.0)
     rubyntlm (0.1.1)
     rubypants (0.2.0)
+    rufus-scheduler (2.0.24)
+      tzinfo (>= 0.3.22)
     rugged (0.21.0)
     safe_yaml (0.9.7)
     sanitize (2.1.0)
@@ -546,12 +558,6 @@ GEM
     sexp_processor (4.4.0)
     shoulda-matchers (2.1.0)
       activesupport (>= 3.0.0)
-    sidekiq (3.0.1)
-      celluloid (>= 0.15.2)
-      connection_pool (>= 2.0.0)
-      json
-      redis (>= 3.0.6)
-      redis-namespace (>= 1.3.1)
     simple_oauth (0.1.9)
     simplecov (0.8.2)
       docile (~> 1.1.0)
@@ -628,6 +634,11 @@ GEM
       polyglot (>= 0.3.1)
     turbolinks (2.0.0)
       coffee-rails
+    twitter-bootstrap-rails (2.2.8)
+      actionpack (>= 3.1)
+      execjs
+      rails (>= 3.1)
+      railties (>= 3.1)
     twitter-stream (0.1.16)
       eventmachine (>= 0.12.8)
       http_parser.rb (~> 0.5.1)
@@ -647,6 +658,8 @@ GEM
       raindrops (~> 0.7)
     unicorn-worker-killer (0.4.2)
       unicorn (~> 4)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
     version_sorter (1.1.0)
     virtus (1.0.1)
       axiom-types (~> 0.0.5)
@@ -765,6 +778,10 @@ DEPENDENCIES
   redcarpet (~> 3.1.2)
   redis-rails
   request_store
+  resque
+  resque-multi-job-forks
+  resque-scheduler
+  resque-web
   rspec-rails
   rugged
   sanitize (~> 2.0)
@@ -775,8 +792,6 @@ DEPENDENCIES
   semantic-ui-sass (~> 0.16.1.0)
   settingslogic
   shoulda-matchers (~> 2.1.0)
-  sidekiq (~> 3.0.1)
-  sidetiq!
   simplecov
   simplecov-vim
   sinatra

Procfile

@@ -1,3 +1,3 @@
 web: bundle exec unicorn_rails -p ${PORT:="3000"} -E ${RAILS_ENV:="development"} -c ${UNICORN_CONFIG:="config/unicorn.rb"}
-worker: bundle exec sidekiq -q post_receive -q mailer -q system_hook -q project_web_hook -q common -q default -q gitlab_shell -q subscribe
-faye: bundle exec rackup private_pub.ru -s thin -E production
+worker: bundle exec rake resque:work QUEUE=mail_notifications,post_receive,mailer,system_hook,project_web_hook,common,default,gitlab_shell,subscribe
+#faye: bundle exec rackup private_pub.ru -s thin -E production

Rakefile

@@ -5,3 +5,5 @@
 require File.expand_path('../config/application', __FILE__)
 
 Gitlab::Application.load_tasks
+require 'resque/tasks'
+require 'resque/scheduler/tasks'

app/controllers/admin/background_jobs_controller.rb

@@ -1,6 +1,6 @@
 class Admin::BackgroundJobsController < Admin::ApplicationController
   def show
     ps_output, _ = Gitlab::Popen.popen(%W(ps -U #{Settings.gitlab.user} -o pid,pcpu,pmem,stat,start,command))
-    @sidekiq_processes = ps_output.split("\n").grep(/sidekiq/)
+    @resque = ps_output.split("\n").grep(/resque/)
   end
 end

app/controllers/admin/users_controller.rb

@@ -22,7 +22,7 @@ def edit
   end
 
   def block
-    Sidekiq::Client.enqueue_to(:common, BlockUserWorker, user.id, @current_user.id)
+    Resque.enqueue(BlockUserWorker, user.id, @current_user.id)
 
     redirect_to :back, alert: "Task on block #{user.name} user was successfully added in queue"
   end

app/elastic/application_search.rb

@@ -47,10 +47,10 @@ module ApplicationSearch
       }
     }
 
-    after_commit lambda { Elastic::BaseIndexer.perform_async(:index,  self.class.to_s, self.id) }, on: :create
-    after_commit lambda { Elastic::BaseIndexer.perform_async(:update, self.class.to_s, self.id) }, on: :update
-    after_commit lambda { Elastic::BaseIndexer.perform_async(:delete, self.class.to_s, self.id) }, on: :destroy
-    after_touch  lambda { Elastic::BaseIndexer.perform_async(:update, self.class.to_s, self.id) }
+    after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :index,  self.class.to_s, self.id) }, on: :create
+    after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }, on: :update
+    after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :delete, self.class.to_s, self.id) }, on: :destroy
+    after_touch  lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }
   end
 
    module ClassMethods

app/models/event/subscription/notification.rb

@@ -55,7 +55,7 @@ class Event::Subscription::Notification < ActiveRecord::Base
   def async_send_notification
     begin
       unless notification_state == :delayed
-        Sidekiq::Client.enqueue_to(:mail_notifications, MailNotificationWorker, self.id)
+        Resque.enqueue(MailNotificationWorker, self.id)
       end
     rescue
     end

app/models/key.rb

@@ -50,19 +50,11 @@ def shell_id
   end
 
   def add_to_shell
-    GitlabShellWorker.perform_async(
-      :add_key,
-      shell_id,
-      key
-    )
+    Resque.enqueue(GitlabShellWorker, :add_key, shell_id, key)
   end
 
   def remove_from_shell
-    GitlabShellWorker.perform_async(
-      :remove_key,
-      shell_id,
-      key,
-    )
+    Resque.enqueue(GitlabShellWorker, :remove_key, shell_id, key)
   end
 
   private

app/models/project.rb

@@ -383,7 +383,7 @@ def saved?
   end
 
   def add_import_job
-    RepositoryImportWorker.perform_in(2.seconds, id)
+    Resque.enqueue(RepositoryImportWorker, id)
   end
 
   def import?

app/models/web_hook.rb

@@ -58,6 +58,6 @@ def execute(data)
   end
 
   def async_execute(data)
-    Sidekiq::Client.enqueue(ProjectWebHookWorker, id, data)
+    Resque.enqueue(ProjectWebHookWorker, id, data)
   end
 end

app/observers/project_observer.rb

@@ -12,15 +12,8 @@ def before_destroy(project)
   end
 
   def after_destroy(project)
-    GitlabShellWorker.perform_async(
-      :remove_repository,
-      project.path_with_namespace
-    )
-
-    GitlabShellWorker.perform_async(
-      :remove_repository,
-      project.path_with_namespace + ".wiki"
-    )
+    Resque.enqueue(GitlabShellWorker, :remove_repository, project.path_with_namespace)
+    Resque.enqueue(GitlabShellWorker, :remove_repository, project.path_with_namespace + ".wiki")
 
     project.satellite.destroy
 

app/services/base_service.rb

@@ -20,13 +20,13 @@ def can?(object, action, subject)
   def receive_delayed_notifications
     notifications = Event::Subscription::Notification.delayed
     notifications.each do |notification|
-      Sidekiq::Client.enqueue_to(:mail_notifications, MailNotificationWorker, notification.id)
+      Resque.enqueue(MailNotificationWorker, notification.id)
     end
   end
 
   def reindex_with_elastic(klass, id, action = :update)
     begin
-      Elastic::BaseIndexer.perform_async(action, klass.name, id)
+      Resque.enqueue(Elastic::BaseIndexer, action, klass.name, id)
     rescue
     end
   end

app/services/git_push_service.rb

@@ -56,7 +56,7 @@ def execute
     if push.tag?
       project.execute_hooks(@push_data.dup, :tag_push_hooks)
     else
-      Elastic::RepositoryIndexer.perform_async(push.id)
+      Resque.enqueue(Elastic::RepositoryIndexer, push.id)
       project.execute_hooks(@push_data.dup, :push_hooks)
     end
 

app/services/projects/base_actions.rb

@@ -62,10 +62,7 @@ def create_action
       if @project.import?
         @project.import_start
       else
-        GitlabShellWorker.perform_async(
-          :add_repository,
-          @project.path_with_namespace
-        )
+        Resque.enqueue(GitlabShellWorker, :add_repository, @project.path_with_namespace)
       end
 
       if @project.wiki_enabled?
@@ -266,18 +263,12 @@ def transfer_to(new_namespace)
 
   def enable_git_protocol(project)
     Gitlab::AppLogger.info("#{project.owner.name} granted public access via git protocol for project \"#{project.name_with_namespace}\"")
-    GitlabShellWorker.perform_async(
-      :enable_git_protocol,
-      project.path_with_namespace
-    )
+    Resque.enqueue(GitlabShellWorker, :enable_git_protocol, project.path_with_namespace)
   end
 
   def disable_git_protocol(project)
     Gitlab::AppLogger.info("#{project.owner.name} removed public access via git protocol for project \"#{project.name_with_namespace}\"")
-    GitlabShellWorker.perform_async(
-      :disable_git_protocol,
-      project.path_with_namespace
-    )
+    Resque.enqueue(GitlabShellWorker, :disable_git_protocol, project.path_with_namespace)
   end
 
   def git_checkpoint_service

app/services/system_hooks_service.rb

@@ -12,7 +12,7 @@ def execute_hooks(data)
   end
 
   def async_execute_hook(hook, data)
-    Sidekiq::Client.enqueue(SystemHookWorker, hook.id, data)
+    Resque.enqueue(SystemHookWorker, hook.id, data)
   end
 
   def build_event_data(model, event)

app/views/admin/background_jobs/show.html.haml

@@ -1,15 +1,15 @@
 %h3.page-title Background Jobs
-%p.light GitLab uses #{link_to "sidekiq", "http://sidekiq.org/"} library for async job processing
+%p.light GitLab uses #{link_to "resque", "http://resquework.org/"} library for async job processing
 
 %hr
 
 .panel.panel-default
-  .panel-heading Sidekiq running processes
+  .panel-heading Resque running processes
   .panel-body
-    - if @sidekiq_processes.empty?
+    - if @resque.empty?
       %h4.cred
         %i.icon-warning-sign
-        There are no running sidekiq processes. Please restart GitLab
+        There are no running resque processes. Please restart GitLab
     - else
       %table.table
         %thead
@@ -21,24 +21,15 @@
           %th START
           %th COMMAND
         %tbody
-          - @sidekiq_processes.each do |process|
-            - next unless process.match(/(sidekiq \d+\.\d+\.\d+.+$)/)
+          - @resque.each do |process|
+            - next unless process.match(/(resque \d+\.\d+\.\d+.+$)/)
             - data = process.strip.split(' ')
             %tr
               %td= Settings.gitlab.user
               - 5.times do
                 %td= data.shift
               %td= data.join(' ')
 
-      .clearfix
-        %p
-          %i.icon-exclamation-sign
-          If '[25 of 25 busy]' is shown, restart GitLab with 'sudo service gitlab reload'.
-        %p
-          %i.icon-exclamation-sign
-          If more than one sidekiq process is listed, stop GitLab, kill the remaining sidekiq processes (sudo pkill -u #{Settings.gitlab.user} -f sidekiq) and restart GitLab.
-
-
 
 .panel.panel-default
-  %iframe{src: sidekiq_path, width: '100%', height: 900, style: "border: none"}
+  %iframe{src: resque_web_path, width: '100%', height: 900, style: "border: none"}