Add server_max_queued_requests and server_max_retry_delay parameters

This commit adds support for queueing and retrying requests if the Puppet
Server is busy.

https://puppet.com/docs/puppet/5.3/release_notes.html#new-feature-puppet-agents-can-retry-requests-on-a-configurable-delay-if-puppet-server-is-busy

Fixes theforeman#569

manifests/init.pp

@@ -446,6 +446,14 @@
 #
 # $server_max_requests_per_instance::       Max number of requests a jruby instances will handle. Defaults to 0 (disabled)
 #
+# $server_max_queued_requests::             The maximum number of requests that may be queued waiting to borrow a
+#                                           JRuby from the pool. (Puppetserver 5.x only)
+#                                           Defaults to 0 (disabled) for Puppetserver >= 5.0
+#
+# $server_max_retry_delay::                 Sets the upper limit for the random sleep set as a Retry-After header on 
+#                                           503 responses returned when max-queued-requests is enabled. (Puppetserver 5.x only)
+#                                           Defaults to 1800 for Puppetserver >= 5.0
+#
 # $server_idle_timeout::                    How long the server will wait for a response on an existing connection
 #
 # $server_connect_timeout::                 How long the server will wait for a response to a connection attempt
@@ -699,6 +707,8 @@
   Optional[Stdlib::Absolutepath] $server_jruby_gem_home = $puppet::params::server_jruby_gem_home,
   Integer[1] $server_max_active_instances = $puppet::params::server_max_active_instances,
   Integer[0] $server_max_requests_per_instance = $puppet::params::server_max_requests_per_instance,
+  Integer[0] $server_max_queued_requests = $puppet::params::server_max_queued_requests,
+  Integer[0] $server_max_retry_delay = $puppet::params::server_max_retry_delay,
   Boolean $server_use_legacy_auth_conf = $puppet::params::server_use_legacy_auth_conf,
   Boolean $server_check_for_updates = $puppet::params::server_check_for_updates,
   Boolean $server_environment_class_cache_enabled = $puppet::params::server_environment_class_cache_enabled,

manifests/params.pp

@@ -429,6 +429,8 @@
   $server_default_manifest_path           = '/etc/puppet/manifests/default_manifest.pp'
   $server_default_manifest_content        = '' # lint:ignore:empty_string_assignment
   $server_max_requests_per_instance       = 0
+  $server_max_queued_requests             = 0
+  $server_max_retry_delay                 = 1800
   $server_idle_timeout                    = 1200000
   $server_web_idle_timeout                = 30000
   $server_connect_timeout                 = 120000

manifests/server.pp

@@ -432,6 +432,8 @@
   Optional[Stdlib::Absolutepath] $jruby_gem_home = $::puppet::server_jruby_gem_home,
   Integer[1] $max_active_instances = $::puppet::server_max_active_instances,
   Integer[0] $max_requests_per_instance = $::puppet::server_max_requests_per_instance,
+  Integer[0] $max_queued_requests = $puppet::server_max_queued_requests,
+  Integer[0] $max_retry_delay = $puppet::server_max_retry_delay,
   Boolean $use_legacy_auth_conf = $::puppet::server_use_legacy_auth_conf,
   Boolean $check_for_updates = $::puppet::server_check_for_updates,
   Boolean $environment_class_cache_enabled = $::puppet::server_environment_class_cache_enabled,

manifests/server/puppetserver.pp

@@ -48,6 +48,14 @@
 # * `server_max_requests_per_instance`
 # Puppetserver number of max requests per jruby instance
 #
+# * `server_max_queued_requests`
+# The maximum number of requests that may be queued waiting
+# to borrow a JRuby from the pool.
+#
+# * `server_max_retry_delay`
+# Sets the upper limit for the random sleep set as a Retry-After
+# header on 503 responses returned when max-queued-requests is enabled.
+#
 # === Example
 #
 # @example
@@ -75,6 +83,8 @@
   $server_cipher_suites                   = $::puppet::server::cipher_suites,
   $server_max_active_instances            = $::puppet::server::max_active_instances,
   $server_max_requests_per_instance       = $::puppet::server::max_requests_per_instance,
+  $server_max_queued_requests             = $::puppet::server::max_queued_requests,
+  $server_max_retry_delay                 = $::puppet::server::max_retry_delay,
   $server_ssl_protocols                   = $::puppet::server::ssl_protocols,
   $server_ssl_ca_crl                      = $::puppet::server::ssl_ca_crl,
   $server_ssl_ca_cert                     = $::puppet::server::ssl_ca_cert,

spec/classes/puppet_server_puppetserver_spec.rb

@@ -31,6 +31,8 @@
                                                      'TLS_RSA_WITH_AES_128_CBC_SHA', ],
         :server_max_active_instances            => 2,
         :server_max_requests_per_instance       => 0,
+        :server_max_queued_requests             => 0,
+        :server_max_retry_delay                 => 1800,
         :server_http                            => false,
         :server_http_allow                      => [],
         :server_ca                              => true,
@@ -270,6 +272,88 @@
         end
       end
 
+      describe 'server_max_queued_requests' do
+        context 'when server_puppetserver_version >= 5.0 with default parameters' do
+          let(:params) do
+            default_params.merge(
+              :server_puppetserver_dir     => '/etc/custom/puppetserver',
+              :server_puppetserver_version => '5.0.0',
+            )
+            end
+          it 'should have max-queued-requests: 0' do
+            content = catalogue.resource('file', '/etc/custom/puppetserver/conf.d/puppetserver.conf').send(:parameters)[:content]
+            expect(content).to include(%Q[    max-queued-requests: 0\n])
+          end
+        end
+        context 'when server_puppetserver_version >= 5.0 with custom server_max_queued_requests' do
+          let(:params) do
+            default_params.merge(
+              :server_puppetserver_dir     => '/etc/custom/puppetserver',
+              :server_puppetserver_version => '5.0.0',
+              :server_max_queued_requests  => 100,
+            )
+          end
+
+          it 'should have custom max-queued-requests: 100' do
+            content = catalogue.resource('file', '/etc/custom/puppetserver/conf.d/puppetserver.conf').send(:parameters)[:content]
+            expect(content).to include(%Q[    max-queued-requests: 100\n])
+          end
+        end
+        context 'when server_puppetserver_version < 5.0 with default parameters' do
+          let(:params) do
+            default_params.merge(
+              :server_puppetserver_dir     => '/etc/custom/puppetserver',
+              :server_puppetserver_version => '2.7.0',
+            )
+            end
+          it 'should not have max-queued-requests' do
+            content = catalogue.resource('file', '/etc/custom/puppetserver/conf.d/puppetserver.conf').send(:parameters)[:content]
+            expect(content).not_to include('max-queued-requests')
+          end
+        end
+      end
+
+      describe 'server_max_retry_delay' do
+        context 'when server_puppetserver_version >= 5.0 with default parameters' do
+          let(:params) do
+            default_params.merge(
+              :server_puppetserver_dir     => '/etc/custom/puppetserver',
+              :server_puppetserver_version => '5.0.0',
+            )
+            end
+          it 'should have max-retry-delay: 1800' do
+            content = catalogue.resource('file', '/etc/custom/puppetserver/conf.d/puppetserver.conf').send(:parameters)[:content]
+            expect(content).to include(%Q[    max-retry-delay: 1800\n])
+          end
+        end
+        context 'when server_puppetserver_version >= 5.0 custom server_max_retry_delay' do
+          let(:params) do
+            default_params.merge(
+              :server_puppetserver_dir     => '/etc/custom/puppetserver',
+              :server_puppetserver_version => '5.0.0',
+              :server_max_retry_delay      => 100
+            )
+          end
+
+          it 'should have custom max-retry-delay: 100' do
+            content = catalogue.resource('file', '/etc/custom/puppetserver/conf.d/puppetserver.conf').send(:parameters)[:content]
+            expect(content).to include(%Q[    max-retry-delay: 100\n])
+          end
+        end
+        context 'when server_puppetserver_version < 5.0 with default parameters' do
+          let(:params) do
+            default_params.merge(
+              :server_puppetserver_dir     => '/etc/custom/puppetserver',
+              :server_puppetserver_version => '2.7.0',
+            )
+            end
+          it 'should not have max-retry-delay' do
+            content = catalogue.resource('file', '/etc/custom/puppetserver/conf.d/puppetserver.conf').send(:parameters)[:content]
+            expect(content).not_to include('max-retry-delay')
+          end
+        end
+      end
+
       describe 'versioned-code-service' do
         context 'when server_puppetserver_version >= 2.5' do
           let(:params) do

templates/server/puppetserver/conf.d/puppetserver.conf.erb

@@ -56,6 +56,13 @@ jruby-puppet: {
     # (optional) the number of HTTP requests a given JRuby instance will handle in its lifetime.
     max-requests-per-instance: <%= @server_max_requests_per_instance %>
 
+<%- if scope.function_versioncmp([@server_puppetserver_version, '5.0']) >= 0 -%>
+    # (optional) The maximum number of requests that may be queued waiting to borrow a JRuby from the pool.
+    max-queued-requests: <%= @server_max_queued_requests %>
+
+    # (optional) Sets the upper limit for the random sleep set as a Retry-After header on 503 responses returned when max-queued-requests is enabled.
+    max-retry-delay: <%= @server_max_retry_delay %>
+<%- end -%>
     # (optional) Authorize access to Puppet master endpoints via rules
     # specified in the legacy Puppet auth.conf file (if true) or via rules
     # specified in the Puppet Server HOCON-formatted auth.conf (if false or not