Merge branch 'searchhead_clustering' of https://github.com/vidkun/che…

…f-splunk into vidkun-searchhead_clustering

.kitchen.dokken.yml

@@ -199,7 +199,7 @@ suites:
       dev_mode: true
       splunk:
         server:
-          runasroot: true
+          runasroot: false
         is_server: true
         accept_license: true
         clustering:
@@ -210,6 +210,22 @@ suites:
         ssl_options:
           enable_ssl: true
 
+  - name: server-shcluster-member
+    run_list:
+      - recipe[chef-splunk::default]
+    attributes:
+      dev_mode: true
+      splunk:
+        server:
+          runasroot: false
+        is_server: true
+        accept_license: true
+        shclustering:
+          enabled: true
+        ssl_options:
+          enable_ssl: true
+        web_port: 8443
+
   - name: disabled
     run_list:
       - recipe[chef-splunk::default]

.kitchen.yml

@@ -83,7 +83,7 @@ suites:
       dev_mode: true
       splunk:
         server:
-          runasroot: true
+          runasroot: false
         is_server: true
         accept_license: true
         clustering:
@@ -94,6 +94,22 @@ suites:
         ssl_options:
           enable_ssl: true
 
+  - name: server-shcluster-member
+    run_list:
+      - recipe[chef-splunk::default]
+    attributes:
+      dev_mode: true
+      splunk:
+        server:
+          runasroot: false
+        is_server: true
+        accept_license: true
+        shclustering:
+          enabled: true
+        ssl_options:
+          enable_ssl: true
+        web_port: 8443
+
   - name: disabled
     run_list:
       - recipe[chef-splunk::default]

README.md

@@ -157,6 +157,36 @@ clustering in the `setup_clustering` recipe:
     `node['splunk']['clustering']['mode']='master'` and multisite is true. Defaults to 'origin:1,total:2'.
     Refer to [Splunk Admin docs](http://docs.splunk.com/Documentation/Splunk/latest/Admin/serverconf) for exact syntax and more details.
 
+The following attributes are related to setting up a Splunk server with search head
+clustering in the `setup_shclustering` recipe:
+
+* `node['splunk']['shclustering']`: A hash of search head clustering configurations
+  used in the `setup_shclustering` recipe
+* `node['splunk']['shclustering']['enabled']`: Whether to enable search head clustering,
+  must be set to `true` to use the `setup_shclustering` recipe. Defaults to `false`,
+  must be a boolean literal `true` or `false`.
+* `node['splunk']['shclustering']['mode']`: The search head clustering mode of the node within
+  the cluster. This is used to determine if the node needs to bootstrap the shcluster and initialize
+  the node as the captain. Must be set using string literal 'member' or 'captain'.
+* `node['splunk']['shclustering']['label']`: The label for the shcluster. Used to differentiate 
+  from other shclusters in the environment. Must be a string. Defaults to `shcluster1`.
+  captain election. Must be set using string literal 'member' or 'captain'.
+* `node['splunk']['shclustering']['replication_factor']`: The replication factor
+  of the shcluster. Defaults to 3, must be a positive integer.
+* `node['splunk']['shclustering']['replication_port']`: The replication port
+  of the shcluster members. Defaults to 9900.
+* `node['splunk']['shclustering']['deployer_url']`: The management url for the
+  shcluster deployer server, must be set to a string such as: `https://deployer.domain.tld:8089`.
+  This attribute is optional. Defaults to empty.
+* `node['splunk']['shclustering']['mgmt_uri']`: The management url for the
+  shcluster member node, must be set to a string such as: `https://shx.domain.tld:8089`. You can
+  use the node's IP address instead of the FQDN if desired. Defaults to `https://#{node['fqdn']}:8089`.
+* `node['splunk']['shclustering']['shcluster_members']`: An array of all search head 
+  cluster members referenced by their `mgmt_uri`. Currently this will do a Chef search for nodes that
+  are in the same environment, with search head clustering enabled, and with the same
+  cluster label. Alternatively, this can be hard-coded with a list of all shcluster 
+  members including the current node. Must be an array of strings. Defaults to an empty array.
+
 The following attributes are related to setting up a splunk forwarder
 with the `client` recipe
 
@@ -429,6 +459,44 @@ of clustering stanza in `etc/system/local/server.conf`).
 Indexer clustering is used to achieve some data availability & recovery. To learn
 more about Splunk indexer clustering, refer to [Splunk Docs](http://docs.splunk.com/Documentation/Splunk/latest/Indexer/Aboutclusters).
 
+## setup_shclustering
+
+This recipe sets up Splunk search head clustering. The attribute
+`node['splunk']['shclustering']['enabled']` must be set to true in order to
+run this recipe. Similar to `setup_auth`, this recipes loads
+the same encrypted data bag with the Splunk `secret` key (to be shared among
+cluster members), using the [chef-vault cookbook](http://ckbk.it/chef-vault)
+helper method, `chef_vault_item`. See __Usage__ for how to set this up. The
+recipe will edit the cluster configuration, and then write a state file to
+`etc/.setup_shcluster` to indicate in future Chef runs that it has set the node's 
+search head clustering configuration. If cluster configuration should be changed, 
+then that file should be removed.
+
+It will also search a Chef Server for a Splunk Enterprise (server)
+node of type cluster master, that is with `splunk_shclustering_enable:true` and
+the same `splunk_shclustering_label` in the same `chef_environment` and
+use that server's IP when building the list of `shcluster_members`.
+
+The search head cluster configuration is deployed as a custom Splunk app that
+is written to `etc/apps/0_autogen_shcluster_config` to take advantage of Splunk's
+built in config layering. All nodes with `splunk_shclustering_enable:true` will
+receive this app.
+
+On the first Chef run on a node with `splunk_shclustering_mode:captain`, this recipe
+will build and execute the Splunk command to bootstrap the search head cluster and
+initiate the captain election process.
+
+In addition to using this recipe for configuring the search head cluster members, you
+will also have to manually configure a search head instance to serve as the 
+search head cluster's deployer. This is done by adding a `[shclustering]` stanza to
+that instance's `etc/system/local/server.conf` with the same `pass4SymmKey = <secret>`
+and the same `shcluster_label = <splunk_shclustering_label>`. This deployer is optional, but should be configured prior to 
+running the bootstrap on the captain and then the search head cluster member nodes 
+configured with this deployer node's mgmt_uri set in the member node's `splunk_shclustering_deployer_url`
+
+Search head clustering is used to achieve high availability & scaling. To learn
+more about Splunk search head clustering, refer to [Splunk Docs](http://docs.splunk.com/Documentation/Splunk/latest/DistSearch/AboutSHC).
+
 ## upgrade
 
 **Important** Read the upgrade documentation and release notes for any

attributes/default.rb

@@ -55,6 +55,17 @@
   'site_search_factor' => 'origin:1,total:2',
 }
 
+default['splunk']['shclustering'] = {
+  'enabled' => false,
+  'mode' => 'member', # member|captain
+  'label' => 'shcluster1',
+  'replication_factor' => 3,
+  'replication_port' => 9900,
+  'deployer_url' => '',
+  'mgmt_uri' => "https://#{node['fqdn']}:8089",
+  'shcluster_members' => [],
+}
+
 # Add key value pairs to this to add configuration pairs to the output.conf file
 # 'sslCertPath' => '$SPLUNK_HOME/etc/certs/cert.pem'
 default['splunk']['outputs_conf'] = {

recipes/server.rb

@@ -62,3 +62,7 @@
 if node['splunk']['clustering']['enabled']
   include_recipe 'chef-splunk::setup_clustering'
 end
+
+if node['splunk']['shclustering']['enabled']
+  include_recipe 'chef-splunk::setup_shclustering'
+end

recipes/setup_shclustering.rb

@@ -0,0 +1,101 @@
+#
+# Cookbook Name:: splunk
+# Recipe:: setup_shclustering
+#
+# Author: Ryan LeViseur <[email protected]>
+# Copyright (c) 2014, Chef Software, Inc <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+unless node['splunk']['shclustering']['enabled']
+  Chef::Log.debug('The chef-splunk::setup_shclustering recipe was added to the node,')
+  Chef::Log.debug('but the attribute to enable search head clustering was not set.')
+  return
+end
+
+# ensure that the splunk service resource is available without cloning
+# the resource (CHEF-3694). this is so the later notification works,
+# especially when using chefspec to run this cookbook's specs.
+begin
+  resources('service[splunk]')
+rescue Chef::Exceptions::ResourceNotFound
+  service 'splunk'
+end
+
+include_recipe 'chef-vault'
+
+passwords = chef_vault_item('vault', "splunk_#{node.chef_environment}")
+splunk_auth_info = passwords['auth']
+shcluster_secret = passwords['secret']
+shcluster_params = node['splunk']['shclustering']
+
+# create app directories to house our server.conf with our shcluster configuration
+shcluster_app_dir = "#{splunk_dir}/etc/apps/0_autogen_shcluster_config"
+
+directory shcluster_app_dir do
+  owner node['splunk']['user']['username']
+  group node['splunk']['user']['username']
+  mode 0755
+end
+
+directory "#{shcluster_app_dir}/local" do
+  owner node['splunk']['user']['username']
+  group node['splunk']['user']['username']
+  mode 0755
+end
+
+template "#{shcluster_app_dir}/local/server.conf" do
+  source 'shclustering/server.conf.erb'
+  mode 0600
+  owner node['splunk']['user']['username']
+  group node['splunk']['user']['username']
+  variables(
+    shcluster_params: node['splunk']['shclustering'],
+    shcluster_secret: shcluster_secret
+  )
+  sensitive true
+  notifies :restart, 'service[splunk]', :immediately
+end
+
+# bootstrap the shcluster and elect a captain if initial_captain set to true and this is the initial shcluster build
+if node['splunk']['shclustering']['mode'] == 'captain'
+  # unless shcluster members are staticly assigned via the node attribute,
+  # try to find the other shcluster members via Chef search
+  if shcluster_params['shcluster_members'].empty?
+    shcluster_servers_list = []
+    search( # ~FC003
+      :node,
+      "\
+      splunk_shclustering_enabled:true AND \
+      splunk_shclustering_label:#{node['splunk']['shclustering']['label']} AND \
+      chef_environment:#{node.chef_environment}"
+    ).each do |result|
+      shcluster_servers_list << result['splunk']['shclustering']['mgmt_uri']
+    end
+  else
+    shcluster_servers_list = shcluster_params['shcluster_members']
+  end
+
+  execute 'bootstrap-shcluster' do
+    command "#{splunk_cmd} bootstrap shcluster-captain -servers_list '#{shcluster_servers_list.join(',')}' -auth '#{splunk_auth_info}'"
+    not_if { ::File.exist?("#{splunk_dir}/etc/.setup_shcluster") }
+    notifies :restart, 'service[splunk]'
+  end
+end
+
+file "#{splunk_dir}/etc/.setup_shcluster" do
+  content 'true\n'
+  owner node['splunk']['user']['username']
+  group node['splunk']['user']['username']
+  mode 00600
+end

spec/recipes/setup_shclustering_spec.rb

@@ -0,0 +1,124 @@
+require_relative '../spec_helper'
+
+describe 'chef-splunk::setup_shclustering' do
+  let(:secrets) do
+    {
+      'splunk__default' => {
+        'id' => 'splunk__default',
+        'auth' => 'admin:notarealpassword',
+        'secret' => 'notarealsecret',
+      },
+    }
+  end
+
+  let(:deployer_node) do
+    stub_node(platform: 'ubuntu', version: '12.04') do |node|
+      node.automatic['fqdn'] = 'deploy.cluster.example.com'
+      node.automatic['ipaddress'] = '192.168.0.10'
+      node.normal['dev_mode'] = true
+      node.normal['splunk']['is_server'] = true
+      node.normal['splunk']['shclustering']['enabled'] = true
+    end
+  end
+
+  context 'default server settings' do
+    let(:chef_run) do
+      ChefSpec::ServerRunner.new do |node, server|
+        node.normal['splunk']['is_server'] = true
+        # Populate mock vault data bag to the server
+        server.create_data_bag('vault', secrets)
+      end.converge(described_recipe)
+    end
+
+    it 'does nothing' do
+      expect(chef_run.resource_collection).to be_empty
+    end
+  end
+
+  context 'search head cluster member settings' do
+    let(:chef_run) do
+      ChefSpec::ServerRunner.new do |node, server|
+        node.normal['dev_mode'] = true
+        node.normal['splunk']['is_server'] = true
+        node.normal['splunk']['shclustering']['enabled'] = true
+        node.normal['splunk']['shclustering']['deployer_url'] = "https://#{deployer_node.fqdn}:8089"
+        node.normal['splunk']['shclustering']['mgmt_uri'] = "https://#{node['fqdn']}:8089"
+        node.normal['splunk']['shclustering']['shcluster_members'] = [
+          'https://shcluster-member01:8089',
+          'https://shcluster-member02:8089',
+          'https://shcluster-member03:8089',
+        ]
+        # Populate mock vault data bag to the server
+        server.create_data_bag('vault', secrets)
+      end.converge(described_recipe)
+    end
+
+    let(:shcluster_servers_list) do
+      chef_run.node['splunk']['shclustering']['shcluster_members'].join(',')
+    end
+
+    it 'includes chef-vault' do
+      expect(chef_run).to include_recipe('chef-vault::default')
+    end
+
+    it 'writes a file marker to ensure convergence' do
+      expect(chef_run).to render_file('/opt/splunk/etc/.setup_shcluster').with_content('true\n')
+    end
+
+    it 'writes server.conf with replication port' do
+      expect(chef_run).to render_file('/opt/splunk/etc/apps/0_autogen_shcluster_config/local/server.conf')
+        .with_content("[replication_port://#{chef_run.node['splunk']['shclustering']['replication_port']}]")
+    end
+
+    it 'writes server.conf with a shclustering stanza' do
+      expect(chef_run).to render_file('/opt/splunk/etc/apps/0_autogen_shcluster_config/local/server.conf')
+        .with_content('[shclustering]')
+    end
+
+    it 'writes server.conf with the deployer url' do
+      expect(chef_run).to render_file('/opt/splunk/etc/apps/0_autogen_shcluster_config/local/server.conf')
+        .with_content("conf_deploy_fetch_url = #{chef_run.node['splunk']['shclustering']['deployer_url']}")
+    end
+
+    it 'writes server.conf with the node mgmt uri' do
+      expect(chef_run).to render_file('/opt/splunk/etc/apps/0_autogen_shcluster_config/local/server.conf')
+        .with_content("mgmt_uri = #{chef_run.node['splunk']['shclustering']['mgmt_uri']}")
+    end
+
+    it 'writes server.conf with the shcluster replication factor' do
+      expect(chef_run).to render_file('/opt/splunk/etc/apps/0_autogen_shcluster_config/local/server.conf')
+        .with_content("replication_factor = #{chef_run.node['splunk']['shclustering']['replication_factor']}")
+    end
+
+    it 'writes server.conf with the shcluster label' do
+      expect(chef_run).to render_file('/opt/splunk/etc/apps/0_autogen_shcluster_config/local/server.conf')
+        .with_content("shcluster_label = #{chef_run.node['splunk']['shclustering']['label']}")
+    end
+
+    it 'writes server.conf with the shcluster secret' do
+      expect(chef_run).to render_file('/opt/splunk/etc/apps/0_autogen_shcluster_config/local/server.conf')
+        .with_content("pass4SymmKey = #{secrets['splunk__default']['secret']}")
+    end
+
+    it 'does not run command to bootstrap captain' do
+      expect(chef_run).to_not run_execute('bootstrap-shcluster')
+    end
+
+    context 'while set to captain mode' do
+      before(:each) do
+        chef_run.node.normal['splunk']['shclustering']['mode'] = 'captain'
+        chef_run.converge(described_recipe)
+      end
+
+      context 'during initial chef run' do
+        it 'runs command to bootstrap captain with correct parameters' do
+          expect(chef_run).to run_execute('bootstrap-shcluster').with(
+            'command' => "/opt/splunk/bin/splunk bootstrap shcluster-captain -servers_list '#{shcluster_servers_list}'\
+ -auth '#{secrets['splunk__default']['auth']}'"
+          )
+          expect(chef_run.execute('bootstrap-shcluster')).to notify('service[splunk]').to(:restart)
+        end
+      end
+    end
+  end
+end