Gitlab::Popenを追加

Gemfile

@@ -3,6 +3,7 @@ ruby '2.2.1'
 
 # Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
 gem 'rails', '4.2.1'
+gem 'sprockets', '~> 2.0'
 # Use postgresql as the database for Active Record
 gem 'pg'
 # Use SCSS for stylesheets

config/environments/development.rb

@@ -42,4 +42,5 @@
   config.middleware.delete Rack::Lock
 end
 
+ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] = '131072'
 BetterErrors::Middleware.allow_ip! '0.0.0.0'

lib/gitlab/popen.rb

@@ -0,0 +1,35 @@
+require 'fileutils'
+require 'open3'
+
+module Gitlab
+  module Popen
+    extend self
+
+    def popen(cmd, path=nil)
+      unless cmd.is_a?(Array)
+        raise "System commands must be given as an array of strings"
+      end
+
+      path ||= Dir.pwd
+      vars = { "PWD" => path }
+      options = { chdir: path }
+
+      unless File.directory?(path)
+        FileUtils.mkdir_p(path)
+      end
+
+      @cmd_output = ""
+      @cmd_status = 0
+      Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
+        # We are not using stdin so we should close it, in case the command we
+        # are running waits for input.
+        stdin.close
+        @cmd_output << stdout.read
+        @cmd_output << stderr.read
+        @cmd_status = wait_thr.value.exitstatus
+      end
+
+      [@cmd_output, @cmd_status]
+    end
+  end
+end

lib/gitlab/sidekiq_memory_killer.rb

@@ -0,0 +1,51 @@
+module Gitlab
+  class SidekiqMemoryKiller
+    # Default the RSS limit to 0, meaning the MemoryKiller is disabled
+    MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
+    # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
+    GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
+    # Wait 30 seconds for running jobs to finish during graceful shutdown
+    SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
+
+    # Create a mutex used to ensure there will be only one thread waiting to
+    # shut Sidekiq down
+    MUTEX = Mutex.new
+
+    def call(worker, job, queue)
+      yield
+      current_rss = get_rss
+
+      return unless MAX_RSS > 0 && current_rss > MAX_RSS
+
+      Thread.new do
+        # Return if another thread is already waiting to shut Sidekiq down
+        return unless MUTEX.try_lock
+
+        Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
+          "#{MAX_RSS}"
+        Sidekiq.logger.warn "spawned thread that will shut down PID "\
+          "#{Process.pid} in #{GRACE_TIME} seconds"
+        sleep(GRACE_TIME)
+
+        Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}"
+        Process.kill('SIGUSR1', Process.pid)
+
+        Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\
+          "SIGTERM to PID #{Process.pid}"
+        sleep(SHUTDOWN_WAIT)
+
+        Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}"
+        Process.kill('SIGTERM', Process.pid)
+      end
+    end
+
+    private
+
+    def get_rss
+      output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{Process.pid}))
+      return 0 unless status.zero?
+
+      output.to_i
+    end
+  end
+end