From da58c5f0dd54c49beb5a191179028ecb5b23f8f7 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 19 Jun 2018 10:39:16 +0200 Subject: [PATCH 1/8] Move to new testing suite names This commit is just moving the tests to their new names. The config, packge and issue test suites have been removed and the tests from these will be incorporated into the oss and xpack tests. oss: Standard elasticsearch-oss role with idempotency test oss-upgrade: Upgrade from previous minor version oss to current minor version oss oss-to-xpack-upgrade: Upgrade from previous minor version oss to current minor version xpack xpack: Standard elasticsearch (with xpack) role with idempotency test xpack-upgrade: Upgrade from previous minor version xpack to current minor version xpack multi: Tests multiple instances of elasticsearch on a single machine --- .kitchen.yml | 51 ++-- .../helpers/serverspec/config_spec.rb | 163 ----------- .../{standard_spec.rb => oss_spec.rb} | 2 +- .../serverspec/oss_to_xpack_upgrade_spec.rb | 12 + .../helpers/serverspec/package_spec.rb | 112 -------- .../helpers/serverspec/xpack_spec.rb | 180 ++---------- .../helpers/serverspec/xpack_standard_spec.rb | 138 ---------- .../helpers/serverspec/xpack_upgrade_spec.rb | 260 ++++++++++++++++++ test/integration/oss-to-xpack-upgrade.yml | 25 ++ .../oss-to-xpack-upgrade.yml} | 0 .../serverspec/default_spec.rb | 7 + test/integration/{standard.yml => oss.yml} | 0 .../xpack-standard.yml => oss/oss.yml} | 0 .../oss/serverspec/default_spec.rb | 10 + .../standard/serverspec/default_spec.rb | 10 - test/integration/xpack-standard.yml | 16 -- .../xpack-standard/serverspec/default_spec.rb | 7 - test/integration/xpack-upgrade.yml | 158 +++++++++++ .../xpack-upgrade/serverspec/default_spec.rb | 7 + .../xpack-upgrade/xpack-upgrade.yml | 2 + test/integration/xpack.yml | 158 +---------- .../xpack/serverspec/default_spec.rb | 2 +- test/matrix.yml | 12 +- 23 files changed, 542 insertions(+), 790 deletions(-) delete mode 100644 test/integration/helpers/serverspec/config_spec.rb rename test/integration/helpers/serverspec/{standard_spec.rb => oss_spec.rb} (98%) create mode 100644 test/integration/helpers/serverspec/oss_to_xpack_upgrade_spec.rb delete mode 100644 test/integration/helpers/serverspec/package_spec.rb delete mode 100644 test/integration/helpers/serverspec/xpack_standard_spec.rb create mode 100644 test/integration/helpers/serverspec/xpack_upgrade_spec.rb create mode 100644 test/integration/oss-to-xpack-upgrade.yml rename test/integration/{standard/standard.yml => oss-to-xpack-upgrade/oss-to-xpack-upgrade.yml} (100%) create mode 100644 test/integration/oss-to-xpack-upgrade/serverspec/default_spec.rb rename test/integration/{standard.yml => oss.yml} (100%) rename test/integration/{xpack-standard/xpack-standard.yml => oss/oss.yml} (100%) create mode 100644 test/integration/oss/serverspec/default_spec.rb delete mode 100644 test/integration/standard/serverspec/default_spec.rb delete mode 100644 test/integration/xpack-standard.yml delete mode 100644 test/integration/xpack-standard/serverspec/default_spec.rb create mode 100644 test/integration/xpack-upgrade.yml create mode 100644 test/integration/xpack-upgrade/serverspec/default_spec.rb create mode 100644 test/integration/xpack-upgrade/xpack-upgrade.yml diff --git a/.kitchen.yml b/.kitchen.yml index 5087feff..834c150b 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -41,6 +41,7 @@ platforms: use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers - name: ubuntu-16.04 driver_config: image: dliappis/ubuntu-devopsci:16.04 @@ -54,6 +55,7 @@ platforms: use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers run_command: "/sbin/init" - name: debian-8 driver_config: @@ -69,6 +71,7 @@ platforms: - pip uninstall -y ansible volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers use_sudo: false run_command: "/sbin/init" - name: centos-7 @@ -85,53 +88,33 @@ platforms: - pip install jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers run_command: "/usr/sbin/init" privileged: true use_sudo: false suites: - - name: standard + - name: oss provisioner: idempotency_test: true - playbook: test/integration/standard.yml - run_list: - attributes: - - name: package - run_list: - attributes: - provisioner: - playbook: test/integration/package.yml - - name: config - run_list: - attributes: + playbook: test/integration/oss.yml + - name: oss-upgrade provisioner: - playbook: test/integration/config.yml - - name: multi - run_list: - attributes: + playbook: test/integration/oss-upgrade.yml + idempotency_test: false + - name: oss-to-xpack-upgrade provisioner: - playbook: test/integration/multi.yml - idempotency_test: true + playbook: test/integration/oss-to-xpack-upgrade.yml + idempotency_test: false - name: xpack - run_list: - attributes: provisioner: playbook: test/integration/xpack.yml - - name: xpack-standard - run_list: - attributes: - provisioner: - playbook: test/integration/xpack-standard.yml idempotency_test: true - - name: issue-test - run_list: - attributes: + - name: xpack-upgrade provisioner: - playbook: test/integration/issue-test.yml + playbook: test/integration/xpack-upgrade.yml idempotency_test: false - - name: oss-upgrade - run_list: - attributes: + - name: multi provisioner: - playbook: test/integration/oss-upgrade.yml - idempotency_test: false + playbook: test/integration/multi.yml + idempotency_test: true diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb deleted file mode 100644 index d84a280e..00000000 --- a/test/integration/helpers/serverspec/config_spec.rb +++ /dev/null @@ -1,163 +0,0 @@ -require 'spec_helper' - -shared_examples 'config::init' do |vars| - - describe user('elasticsearch') do - it { should exist } - end - - describe group('elasticsearch') do - it { should have_gid 333 } - end - - describe user('elasticsearch') do - it { should have_uid 333 } - end - - describe service('node1_elasticsearch') do - it { should be_running } - end - - describe package(vars['es_package_name']) do - it { should be_installed } - end - - describe file('/etc/elasticsearch/node1/elasticsearch.yml') do - it { should be_file } - end - - #test configuration parameters have been set - test all appropriately set in config file - describe file('/etc/elasticsearch/node1/elasticsearch.yml') do - it { should contain 'http.port: 9401' } - it { should contain 'transport.tcp.port: 9501' } - it { should contain 'node.data: true' } - it { should contain 'node.master: true' } - it { should contain 'cluster.name: custom-cluster' } - it { should contain 'node.name: node1' } - it { should contain 'bootstrap.memory_lock: true' } - it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9501' } - if vars['es_major_version'] == '6.x' - it { should_not contain 'path.conf: /etc/elasticsearch/node1' } - else - it { should contain 'path.conf: /etc/elasticsearch/node1' } - end - it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' } - it { should contain 'path.logs: /opt/elasticsearch/logs/localhost-node1' } - end - - #test directories exist - describe file('/etc/elasticsearch/node1') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/opt/elasticsearch/data-1/localhost-node1') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/opt/elasticsearch/data-2/localhost-node1') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/opt/elasticsearch/logs/localhost-node1') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - #test we started on the correct port was used - describe command('curl -s "localhost:9401"') do - #TODO: This is returning an empty string - #its(:stdout) { should match /\"status\" : 200/ } - its(:exit_status) { should eq 0 } - end - - #test to make sure mlock was applied - describe command('curl -s "localhost:9401/_nodes/process?pretty" | grep mlockall') do - its(:stdout) { should match /true/ } - its(:exit_status) { should eq 0 } - end - - - describe 'version check' do - it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9401 | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) - end - end - - for plugin in vars['es_plugins'] - plugin = plugin['plugin'] - describe file('/usr/share/elasticsearch/plugins/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - #confirm plugins are installed and the correct version - describe command('curl -s localhost:9401/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do - its(:exit_status) { should eq 0 } - end - end - - #explit test to make sure ingest-geoip is not installed - describe file('/usr/share/elasticsearch/plugins/ingest-geoip') do - it { should_not exist } - end - #confirm plugins are installed and the correct version - describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"ingest-geoip","version":"'+vars['es_version']+'"\'') do - its(:exit_status) { should eq 1 } - end - - describe file('/etc/init.d/elasticsearch') do - it { should_not exist } - end - - if ['debian', 'ubuntu'].include?(os[:family]) - describe file('/etc/default/elasticsearch') do - its(:content) { should match '' } - end - end - - if ['centos', 'redhat'].include?(os[:family]) - describe file('/etc/sysconfig/elasticsearch') do - its(:content) { should match '' } - end - end - - describe file('/etc/elasticsearch/elasticsearch.yml') do - it { should_not exist } - end - - describe file('/etc/elasticsearch/logging.yml') do - it { should_not exist } - end - - #Init vs Systemd tests - #Ubuntu 15 and up - #Debian 8 and up - #Centos 7 and up - - if (((os[:family] == 'redhat' || os[:family] == 'centos') && os[:release].to_f >= 7.0) || - (os[:family] == 'ubuntu' && os[:release].to_f >= 15.0) || - (os[:family] == 'debian' && os[:release].to_f >= 8.0)) - describe file('/usr/lib/systemd/system/node1_elasticsearch.service') do - it { should be_file } - it { should contain 'LimitMEMLOCK=infinity' } - it { should contain 'LimitNPROC=3000' } - end - else - describe file('/etc/init.d/node1_elasticsearch') do - it { should be_file } - end - end - - describe file('/etc/elasticsearch/node1/log4j2.properties') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - it { should contain 'CUSTOM LOG4J FILE' } - end - - -end - diff --git a/test/integration/helpers/serverspec/standard_spec.rb b/test/integration/helpers/serverspec/oss_spec.rb similarity index 98% rename from test/integration/helpers/serverspec/standard_spec.rb rename to test/integration/helpers/serverspec/oss_spec.rb index 1fd6c11d..be2cb7b5 100644 --- a/test/integration/helpers/serverspec/standard_spec.rb +++ b/test/integration/helpers/serverspec/oss_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -shared_examples 'standard::init' do |vars| +shared_examples 'oss::init' do |vars| describe user('elasticsearch') do it { should exist } diff --git a/test/integration/helpers/serverspec/oss_to_xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/oss_to_xpack_upgrade_spec.rb new file mode 100644 index 00000000..8cb5c05a --- /dev/null +++ b/test/integration/helpers/serverspec/oss_to_xpack_upgrade_spec.rb @@ -0,0 +1,12 @@ +require 'spec_helper' + +shared_examples 'oss_to_xpack_upgrade::init' do |vars| + describe 'version check' do + it 'should be reported as version '+vars['es_version'] do + expect(curl_json('http://localhost:9200', username='elastic', password='changeme')['version']['number']).to eq(vars['es_version']) + end + it 'should be be running the standard (xpack) version' do + expect(curl_json('http://localhost:9200/_xpack', username='elastic', password='changeme')['tagline']).to eq('You know, for X') + end + end +end diff --git a/test/integration/helpers/serverspec/package_spec.rb b/test/integration/helpers/serverspec/package_spec.rb deleted file mode 100644 index 788cd93a..00000000 --- a/test/integration/helpers/serverspec/package_spec.rb +++ /dev/null @@ -1,112 +0,0 @@ -require 'spec_helper' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -shared_examples 'package::init' do |vars| - - describe user('elasticsearch') do - it { should exist } - end - - describe service('node1_elasticsearch') do - it { should be_running } - end - - describe package(vars['es_package_name']) do - it { should be_installed } - end - - describe file('/etc/elasticsearch/node1/elasticsearch.yml') do - it { should be_file } - it { should contain 'http.port: 9200' } - it { should contain 'transport.tcp.port: 9300' } - it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9300' } - end - - describe file('/etc/elasticsearch/node1/scripts') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/node1/scripts/calculate-score.groovy') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe 'Node listening' do - it 'listening in port 9200' do - expect(port 9200).to be_listening - end - end - - describe file('/etc/elasticsearch/templates') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/templates/basic.json') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe 'Template Installed' do - it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - command = command('curl -s "localhost:9200/_template/basic"') - expect(command.stdout).to match(/basic/) - expect(command.exit_status).to eq(0) - end - end - - describe 'version check' do - it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200 | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) - end - end - - describe file('/usr/share/elasticsearch/plugins') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - - for plugin in vars['es_plugins'] - plugin = plugin['plugin'] - describe file('/usr/share/elasticsearch/plugins/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - #confirm plugins are installed and the correct version - describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do - its(:exit_status) { should eq 0 } - end - end - - - describe file('/etc/init.d/elasticsearch') do - it { should_not exist } - end - - if ['debian', 'ubuntu'].include?(os[:family]) - describe file('/etc/default/elasticsearch') do - its(:content) { should match '' } - end - end - - if ['centos', 'redhat'].include?(os[:family]) - describe file('/etc/sysconfig/elasticsearch') do - its(:content) { should match '' } - end - end - - describe file('/etc/elasticsearch/elasticsearch.yml') do - it { should_not exist } - end - - describe file('/etc/elasticsearch/logging.yml') do - it { should_not exist } - end - -end - diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index d3548acb..f50694ee 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -1,6 +1,4 @@ require 'spec_helper' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) shared_examples 'xpack::init' do |vars| @@ -36,6 +34,9 @@ end it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' } + it { should contain 'xpack.security.enabled: false' } + it { should contain 'xpack.watcher.enabled: false' } + end describe 'Node listening' do @@ -46,7 +47,7 @@ describe 'version check' do it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number') + command = command('curl -s localhost:9200 | grep number') expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end @@ -76,30 +77,31 @@ it { should_not exist } end - # X-Pack is no longer installed as a plugin in elasticsearch - if vars['es_major_version'] == '5.x' - describe file('/usr/share/elasticsearch/plugins') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/usr/share/elasticsearch/plugins/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end + #Xpack specific tests + describe file('/usr/share/elasticsearch/plugins') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do - its(:exit_status) { should eq 0 } + #Test if x-pack is activated + describe 'x-pack activation' do + it 'should be activated and valid' do + command = command('curl -s localhost:9200/_license?pretty=true') + expect(command.stdout).to match('"status" : "active"') + expect(command.exit_status).to eq(0) end + end + # X-Pack is no longer installed as a plugin in elasticsearch + if vars['es_major_version'] == '5.x' describe file('/usr/share/elasticsearch/plugins/x-pack') do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - describe 'xpack plugin' do + describe 'x-pack-core plugin' do it 'should be installed with the correct version' do - plugins = curl_json('http://localhost:9200/_nodes/plugins', username='es_admin', password='changeMeAgain') + plugins = curl_json('http://localhost:9200/_nodes/plugins') node, data = plugins['nodes'].first version = 'plugin not found' name = 'x-pack' @@ -112,149 +114,25 @@ expect(version).to eql(vars['es_version']) end end - end - #Test if x-pack is activated - describe 'x-pack activation' do - it 'should be activated and valid' do - command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain') - expect(command.stdout).to match('"status" : "active"') - expect(command.exit_status).to eq(0) - end - end - - describe file('/etc/elasticsearch/security_node/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - for plugin in vars['es_plugins'] - plugin = plugin['plugin'] - - describe file('/usr/share/elasticsearch/plugins/'+plugin) do + describe file('/etc/elasticsearch/security_node/x-pack') do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do - its(:exit_status) { should eq 0 } - end - end - - #Test users file, users_roles and roles.yml - describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/users_roles') do - it { should be_owned_by 'elasticsearch' } - it { should contain 'admin:es_admin' } - it { should contain 'power_user:testUser' } end - describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/users') do - it { should be_owned_by 'elasticsearch' } - it { should contain 'testUser:' } - it { should contain 'es_admin:' } + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do + its(:exit_status) { should eq 0 } end - describe 'security roles' do - it 'should list the security roles' do - roles = curl_json('http://localhost:9200/_xpack/security/role', username='es_admin', password='changeMeAgain') - expect(roles.key?('superuser')) - end + describe command('curl -s localhost:9200/_xpack') do + its(:stdout_as_json) { should include('features' => include('security' => include('enabled' => false))) } + its(:stdout_as_json) { should include('features' => include('watcher' => include('enabled' => false))) } + its(:stdout_as_json) { should include('features' => include('graph' => include('enabled' => true))) } + its(:stdout_as_json) { should include('features' => include('monitoring' => include('enabled' => true))) } + its(:stdout_as_json) { should include('features' => include('ml' => include('enabled' => true))) } end - describe file('/etc/elasticsearch/templates') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/templates/basic.json') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe 'Template Installed' do - it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain') - expect(command.stdout).to match(/basic/) - expect(command.exit_status).to eq(0) - end - end - - #This is possibly subject to format changes in the response across versions so may fail in the future - describe 'Template Contents Correct' do - it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - template = curl_json('http://localhost:9200/_template/basic', username='es_admin', password='changeMeAgain') - expect(template.key?('basic')) - expect(template['basic']['settings']['index']['number_of_shards']).to eq("1") - expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false) - end - end - - #Test contents of Elasticsearch.yml file - describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do - it { should contain 'security.authc.realms.file1.order: 0' } - it { should contain 'security.authc.realms.file1.type: file' } - it { should contain 'security.authc.realms.native1.order: 1' } - it { should contain 'security.authc.realms.native1.type: native' } - end - - #Test contents of role_mapping.yml - describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/role_mapping.yml') do - it { should be_owned_by 'elasticsearch' } - it { should contain 'power_user:' } - it { should contain '- cn=admins,dc=example,dc=com' } - it { should contain 'user:' } - it { should contain '- cn=admins,dc=example,dc=com' } - end - - #check accounts are correct i.e. we can auth and they have the correct roles - - describe 'kibana4_server access check' do - it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200/ -u kibana4_server:changeMe | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) - end - end - - describe 'security users' do - result = curl_json('http://localhost:9200/_xpack/security/user', username='elastic', password='elasticChanged') - it 'should have the elastic user' do - expect(result['elastic']['username']).to eq('elastic') - expect(result['elastic']['roles']).to eq(['superuser']) - expect(result['elastic']['enabled']).to eq(true) - end - it 'should have the kibana user' do - expect(result['kibana']['username']).to eq('kibana') - expect(result['kibana']['roles']).to eq(['kibana_system']) - expect(result['kibana']['enabled']).to eq(true) - end - it 'should have the kibana_server user' do - expect(result['kibana4_server']['username']).to eq('kibana4_server') - expect(result['kibana4_server']['roles']).to eq(['kibana4_server']) - expect(result['kibana4_server']['enabled']).to eq(true) - end - it 'should have the logstash user' do - expect(result['logstash_system']['username']).to eq('logstash_system') - expect(result['logstash_system']['roles']).to eq(['logstash_system']) - expect(result['logstash_system']['enabled']).to eq(true) - end - end - - describe 'logstash_system access check' do - it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200/ -u logstash_system:aNewLogstashPassword | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) - end - end - - if vars['es_major_version'] == '5.x' # kibana default password has been removed in 6.x - describe 'kibana access check' do - it 'should be reported as version '+vars['es_version'] do - result = curl_json('http://localhost:9200/', username='kibana', password='changeme') - expect(result['version']['number']).to eq(vars['es_version']) - end - end - end end diff --git a/test/integration/helpers/serverspec/xpack_standard_spec.rb b/test/integration/helpers/serverspec/xpack_standard_spec.rb deleted file mode 100644 index 9c35ae5e..00000000 --- a/test/integration/helpers/serverspec/xpack_standard_spec.rb +++ /dev/null @@ -1,138 +0,0 @@ -require 'spec_helper' - -shared_examples 'xpack_standard::init' do |vars| - - describe user('elasticsearch') do - it { should exist } - end - - describe service('security_node_elasticsearch') do - it { should be_running } - end - - describe package(vars['es_package_name']) do - it { should be_installed } - end - - describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/security_node/log4j2.properties') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do - it { should contain 'node.name: localhost-security_node' } - it { should contain 'cluster.name: elasticsearch' } - if vars['es_major_version'] == '6.x' - it { should_not contain 'path.conf: /etc/elasticsearch/security_node' } - else - it { should contain 'path.conf: /etc/elasticsearch/security_node' } - end - it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' } - it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' } - it { should contain 'xpack.security.enabled: false' } - it { should contain 'xpack.watcher.enabled: false' } - - end - - describe 'Node listening' do - it 'listening in port 9200' do - expect(port 9200).to be_listening - end - end - - describe 'version check' do - it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200 | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) - end - end - - describe file('/etc/init.d/elasticsearch') do - it { should_not exist } - end - - if ['debian', 'ubuntu'].include?(os[:family]) - describe file('/etc/default/elasticsearch') do - its(:content) { should match '' } - end - end - - if ['centos', 'redhat'].include?(os[:family]) - describe file('/etc/sysconfig/elasticsearch') do - its(:content) { should match '' } - end - end - - describe file('/etc/elasticsearch/elasticsearch.yml') do - it { should_not exist } - end - - describe file('/etc/elasticsearch/logging.yml') do - it { should_not exist } - end - - #Xpack specific tests - describe file('/usr/share/elasticsearch/plugins') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - #Test if x-pack is activated - describe 'x-pack activation' do - it 'should be activated and valid' do - command = command('curl -s localhost:9200/_license?pretty=true') - expect(command.stdout).to match('"status" : "active"') - expect(command.exit_status).to eq(0) - end - end - - # X-Pack is no longer installed as a plugin in elasticsearch - if vars['es_major_version'] == '5.x' - describe file('/usr/share/elasticsearch/plugins/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe 'x-pack-core plugin' do - it 'should be installed with the correct version' do - plugins = curl_json('http://localhost:9200/_nodes/plugins') - node, data = plugins['nodes'].first - version = 'plugin not found' - name = 'x-pack' - - data['plugins'].each do |plugin| - if plugin['name'] == name - version = plugin['version'] - end - end - expect(version).to eql(vars['es_version']) - end - end - - describe file('/etc/elasticsearch/security_node/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - end - - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do - its(:exit_status) { should eq 0 } - end - - describe command('curl -s localhost:9200/_xpack') do - its(:stdout_as_json) { should include('features' => include('security' => include('enabled' => false))) } - its(:stdout_as_json) { should include('features' => include('watcher' => include('enabled' => false))) } - its(:stdout_as_json) { should include('features' => include('graph' => include('enabled' => true))) } - its(:stdout_as_json) { should include('features' => include('monitoring' => include('enabled' => true))) } - its(:stdout_as_json) { should include('features' => include('ml' => include('enabled' => true))) } - end - -end - diff --git a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb new file mode 100644 index 00000000..a3658858 --- /dev/null +++ b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb @@ -0,0 +1,260 @@ +require 'spec_helper' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) + +shared_examples 'xpack_upgrade::init' do |vars| + + describe user('elasticsearch') do + it { should exist } + end + + describe service('security_node_elasticsearch') do + it { should be_running } + end + + describe package(vars['es_package_name']) do + it { should be_installed } + end + + describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/security_node/log4j2.properties') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do + it { should contain 'node.name: localhost-security_node' } + it { should contain 'cluster.name: elasticsearch' } + if vars['es_major_version'] == '6.x' + it { should_not contain 'path.conf: /etc/elasticsearch/security_node' } + else + it { should contain 'path.conf: /etc/elasticsearch/security_node' } + end + it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' } + it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' } + end + + describe 'Node listening' do + it 'listening in port 9200' do + expect(port 9200).to be_listening + end + end + + describe 'version check' do + it 'should be reported as version '+vars['es_version'] do + command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number') + expect(command.stdout).to match(vars['es_version']) + expect(command.exit_status).to eq(0) + end + end + + describe file('/etc/init.d/elasticsearch') do + it { should_not exist } + end + + if ['debian', 'ubuntu'].include?(os[:family]) + describe file('/etc/default/elasticsearch') do + its(:content) { should match '' } + end + end + + if ['centos', 'redhat'].include?(os[:family]) + describe file('/etc/sysconfig/elasticsearch') do + its(:content) { should match '' } + end + end + + describe file('/etc/elasticsearch/elasticsearch.yml') do + it { should_not exist } + end + + describe file('/etc/elasticsearch/logging.yml') do + it { should_not exist } + end + + # X-Pack is no longer installed as a plugin in elasticsearch + if vars['es_major_version'] == '5.x' + describe file('/usr/share/elasticsearch/plugins') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/usr/share/elasticsearch/plugins/x-pack') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do + its(:exit_status) { should eq 0 } + end + + describe file('/usr/share/elasticsearch/plugins/x-pack') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe 'xpack plugin' do + it 'should be installed with the correct version' do + plugins = curl_json('http://localhost:9200/_nodes/plugins', username='es_admin', password='changeMeAgain') + node, data = plugins['nodes'].first + version = 'plugin not found' + name = 'x-pack' + + data['plugins'].each do |plugin| + if plugin['name'] == name + version = plugin['version'] + end + end + expect(version).to eql(vars['es_version']) + end + end + end + + #Test if x-pack is activated + describe 'x-pack activation' do + it 'should be activated and valid' do + command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain') + expect(command.stdout).to match('"status" : "active"') + expect(command.exit_status).to eq(0) + end + end + + describe file('/etc/elasticsearch/security_node/x-pack') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + for plugin in vars['es_plugins'] + plugin = plugin['plugin'] + + describe file('/usr/share/elasticsearch/plugins/'+plugin) do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do + its(:exit_status) { should eq 0 } + end + end + + #Test users file, users_roles and roles.yml + describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/users_roles') do + it { should be_owned_by 'elasticsearch' } + it { should contain 'admin:es_admin' } + it { should contain 'power_user:testUser' } + end + + describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/users') do + it { should be_owned_by 'elasticsearch' } + it { should contain 'testUser:' } + it { should contain 'es_admin:' } + end + + describe 'security roles' do + it 'should list the security roles' do + roles = curl_json('http://localhost:9200/_xpack/security/role', username='es_admin', password='changeMeAgain') + expect(roles.key?('superuser')) + end + end + + describe file('/etc/elasticsearch/templates') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/templates/basic.json') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe 'Template Installed' do + it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do + command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain') + expect(command.stdout).to match(/basic/) + expect(command.exit_status).to eq(0) + end + end + + #This is possibly subject to format changes in the response across versions so may fail in the future + describe 'Template Contents Correct' do + it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do + template = curl_json('http://localhost:9200/_template/basic', username='es_admin', password='changeMeAgain') + expect(template.key?('basic')) + expect(template['basic']['settings']['index']['number_of_shards']).to eq("1") + expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false) + end + end + + #Test contents of Elasticsearch.yml file + describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do + it { should contain 'security.authc.realms.file1.order: 0' } + it { should contain 'security.authc.realms.file1.type: file' } + it { should contain 'security.authc.realms.native1.order: 1' } + it { should contain 'security.authc.realms.native1.type: native' } + end + + #Test contents of role_mapping.yml + describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/role_mapping.yml') do + it { should be_owned_by 'elasticsearch' } + it { should contain 'power_user:' } + it { should contain '- cn=admins,dc=example,dc=com' } + it { should contain 'user:' } + it { should contain '- cn=admins,dc=example,dc=com' } + end + + #check accounts are correct i.e. we can auth and they have the correct roles + + describe 'kibana4_server access check' do + it 'should be reported as version '+vars['es_version'] do + command = command('curl -s localhost:9200/ -u kibana4_server:changeMe | grep number') + expect(command.stdout).to match(vars['es_version']) + expect(command.exit_status).to eq(0) + end + end + + describe 'security users' do + result = curl_json('http://localhost:9200/_xpack/security/user', username='elastic', password='elasticChanged') + it 'should have the elastic user' do + expect(result['elastic']['username']).to eq('elastic') + expect(result['elastic']['roles']).to eq(['superuser']) + expect(result['elastic']['enabled']).to eq(true) + end + it 'should have the kibana user' do + expect(result['kibana']['username']).to eq('kibana') + expect(result['kibana']['roles']).to eq(['kibana_system']) + expect(result['kibana']['enabled']).to eq(true) + end + it 'should have the kibana_server user' do + expect(result['kibana4_server']['username']).to eq('kibana4_server') + expect(result['kibana4_server']['roles']).to eq(['kibana4_server']) + expect(result['kibana4_server']['enabled']).to eq(true) + end + it 'should have the logstash user' do + expect(result['logstash_system']['username']).to eq('logstash_system') + expect(result['logstash_system']['roles']).to eq(['logstash_system']) + expect(result['logstash_system']['enabled']).to eq(true) + end + end + + describe 'logstash_system access check' do + it 'should be reported as version '+vars['es_version'] do + command = command('curl -s localhost:9200/ -u logstash_system:aNewLogstashPassword | grep number') + expect(command.stdout).to match(vars['es_version']) + expect(command.exit_status).to eq(0) + end + end + + if vars['es_major_version'] == '5.x' # kibana default password has been removed in 6.x + describe 'kibana access check' do + it 'should be reported as version '+vars['es_version'] do + result = curl_json('http://localhost:9200/', username='kibana', password='changeme') + expect(result['version']['number']).to eq(vars['es_version']) + end + end + end +end + diff --git a/test/integration/oss-to-xpack-upgrade.yml b/test/integration/oss-to-xpack-upgrade.yml new file mode 100644 index 00000000..1c74c0f8 --- /dev/null +++ b/test/integration/oss-to-xpack-upgrade.yml @@ -0,0 +1,25 @@ +--- +- name: Standard test for single node setup. Tests idempotence. + hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml + roles: + - { role: elasticsearch, es_instance_name: "node1" } + vars: + es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade + es_enable_xpack: false + es_heap_size: "1g" + +- name: Standard test for single node setup. Tests idempotence. + hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml + roles: + - { role: elasticsearch, es_instance_name: "node1" } + vars: + es_enable_xpack: true + es_api_basic_auth_username: elastic + es_api_basic_auth_password: changeme + es_heap_size: "1g" + es_xpack_features: + - security diff --git a/test/integration/standard/standard.yml b/test/integration/oss-to-xpack-upgrade/oss-to-xpack-upgrade.yml similarity index 100% rename from test/integration/standard/standard.yml rename to test/integration/oss-to-xpack-upgrade/oss-to-xpack-upgrade.yml diff --git a/test/integration/oss-to-xpack-upgrade/serverspec/default_spec.rb b/test/integration/oss-to-xpack-upgrade/serverspec/default_spec.rb new file mode 100644 index 00000000..ff79e845 --- /dev/null +++ b/test/integration/oss-to-xpack-upgrade/serverspec/default_spec.rb @@ -0,0 +1,7 @@ +require 'oss_to_xpack_upgrade_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) + +describe 'oss to xpack upgrade Tests' do + include_examples 'oss_to_xpack_upgrade::init', vars +end diff --git a/test/integration/standard.yml b/test/integration/oss.yml similarity index 100% rename from test/integration/standard.yml rename to test/integration/oss.yml diff --git a/test/integration/xpack-standard/xpack-standard.yml b/test/integration/oss/oss.yml similarity index 100% rename from test/integration/xpack-standard/xpack-standard.yml rename to test/integration/oss/oss.yml diff --git a/test/integration/oss/serverspec/default_spec.rb b/test/integration/oss/serverspec/default_spec.rb new file mode 100644 index 00000000..a5d0f02d --- /dev/null +++ b/test/integration/oss/serverspec/default_spec.rb @@ -0,0 +1,10 @@ +require 'oss_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) + +describe 'OSS Tests' do + include_examples 'oss::init', vars +end + + + diff --git a/test/integration/standard/serverspec/default_spec.rb b/test/integration/standard/serverspec/default_spec.rb deleted file mode 100644 index 449a1c02..00000000 --- a/test/integration/standard/serverspec/default_spec.rb +++ /dev/null @@ -1,10 +0,0 @@ -require 'standard_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'Standard Tests' do - include_examples 'standard::init', vars -end - - - diff --git a/test/integration/xpack-standard.yml b/test/integration/xpack-standard.yml deleted file mode 100644 index 6789a7e2..00000000 --- a/test/integration/xpack-standard.yml +++ /dev/null @@ -1,16 +0,0 @@ -#Tests x-pack is idempotent and works when security is not enabled ---- -- name: Elasticsearch Xpack tests - no security and manual download - hosts: localhost - tasks: - - include: elasticsearch/test/integration/debug.yml - roles: - - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "security_node" } - vars: - es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip" - es_heap_size: 2g - es_enable_xpack: true - es_xpack_features: - - monitoring - - graph - - ml diff --git a/test/integration/xpack-standard/serverspec/default_spec.rb b/test/integration/xpack-standard/serverspec/default_spec.rb deleted file mode 100644 index 771c7c3a..00000000 --- a/test/integration/xpack-standard/serverspec/default_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'xpack_standard_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'Xpack Standard Tests' do - include_examples 'xpack_standard::init', vars -end diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml new file mode 100644 index 00000000..5668dfd0 --- /dev/null +++ b/test/integration/xpack-upgrade.yml @@ -0,0 +1,158 @@ +--- +- name: Elasticsearch Xpack tests initial + hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml + roles: + - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", + "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, + es_instance_name: "security_node" } + vars: + es_heap_size: "1g" + es_templates: true + es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade + es_enable_xpack: true + es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" + es_plugins: + - plugin: ingest-geoip + es_xpack_features: + - security + - alerting + es_api_basic_auth_username: elastic + es_api_basic_auth_password: changeme + es_message_auth_file: system_key + es_role_mapping: + power_user: + - "cn=admins,dc=example,dc=com" + user: + - "cn=users,dc=example,dc=com" + - "cn=admins,dc=example,dc=com" + es_users: + native: + kibana4_server: + password: changeMe + roles: + - kibana4_server + logstash_system: + #this should be successfully modified + password: aNewLogstashPassword + #this will be ignored + roles: + - kibana4_server + elastic: + password: elasticChanged + file: + es_admin: + password: changeMe + roles: + - admin + testUser: + password: changeMeAlso! + roles: + - power_user + - user + es_roles: + file: + admin: + cluster: + - all + indices: + - names: '*' + privileges: + - all + power_user: + cluster: + - monitor + indices: + - names: '*' + privileges: + - all + user: + indices: + - names: '*' + privileges: + - read + kibana4_server: + cluster: + - monitor + indices: + - names: '.kibana' + privileges: + - all + native: + logstash: + cluster: + - manage_index_templates + indices: + - names: 'logstash-*' + privileges: + - write + - delete + - create_index + #this will be ignored - its reserved + logstash_system: + cluster: + - manage_index_templates + indices: + - names: 'logstash-*' + privileges: + - write + - delete + - create_index + +#modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed. +- name: Elasticsearch Xpack modify + hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml + roles: + - role: elasticsearch + es_api_port: 9200 + es_instance_name: "security_node" + es_config: + http.port: 9200 + transport.tcp.port: 9300 + discovery.zen.ping.unicast.hosts: "localhost:9300" + xpack.security.enabled: True + xpack.security.authc.realms.file1.type: "file" + xpack.security.authc.realms.file1.order: 0 + xpack.security.authc.realms.native1.type: "native" + xpack.security.authc.realms.native1.order: 1 + vars: + es_heap_size: "1g" + es_templates: true + es_enable_xpack: true + es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" + es_plugins: + - plugin: ingest-attachment + es_xpack_features: + - security + - alerting + es_api_basic_auth_username: elastic + es_api_basic_auth_password: elasticChanged + es_role_mapping: + power_user: + - "cn=admins,dc=example,dc=com" + user: + - "cn=users,dc=example,dc=com" + - "cn=admins,dc=example,dc=com" + es_users: + native: + kibana4_server: + password: changeMe + roles: + - kibana4_server + logstash_system: + #this will be ignored + roles: + - kibana4_server + file: + es_admin: + password: changeMeAgain + roles: + - admin + testUser: + password: changeMeAlso! + roles: + - power_user + - user diff --git a/test/integration/xpack-upgrade/serverspec/default_spec.rb b/test/integration/xpack-upgrade/serverspec/default_spec.rb new file mode 100644 index 00000000..c2452eec --- /dev/null +++ b/test/integration/xpack-upgrade/serverspec/default_spec.rb @@ -0,0 +1,7 @@ +require 'xpack_upgrade_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) + +describe 'Xpack upgrade Tests' do + include_examples 'xpack_upgrade::init', vars +end diff --git a/test/integration/xpack-upgrade/xpack-upgrade.yml b/test/integration/xpack-upgrade/xpack-upgrade.yml new file mode 100644 index 00000000..a3c37e19 --- /dev/null +++ b/test/integration/xpack-upgrade/xpack-upgrade.yml @@ -0,0 +1,2 @@ +--- +- host: test-kitchen diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 5668dfd0..6789a7e2 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -1,158 +1,16 @@ +#Tests x-pack is idempotent and works when security is not enabled --- -- name: Elasticsearch Xpack tests initial +- name: Elasticsearch Xpack tests - no security and manual download hosts: localhost tasks: - include: elasticsearch/test/integration/debug.yml roles: - - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", - "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, - es_instance_name: "security_node" } + - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "security_node" } vars: - es_heap_size: "1g" - es_templates: true - es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade + es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip" + es_heap_size: 2g es_enable_xpack: true - es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" - es_plugins: - - plugin: ingest-geoip es_xpack_features: - - security - - alerting - es_api_basic_auth_username: elastic - es_api_basic_auth_password: changeme - es_message_auth_file: system_key - es_role_mapping: - power_user: - - "cn=admins,dc=example,dc=com" - user: - - "cn=users,dc=example,dc=com" - - "cn=admins,dc=example,dc=com" - es_users: - native: - kibana4_server: - password: changeMe - roles: - - kibana4_server - logstash_system: - #this should be successfully modified - password: aNewLogstashPassword - #this will be ignored - roles: - - kibana4_server - elastic: - password: elasticChanged - file: - es_admin: - password: changeMe - roles: - - admin - testUser: - password: changeMeAlso! - roles: - - power_user - - user - es_roles: - file: - admin: - cluster: - - all - indices: - - names: '*' - privileges: - - all - power_user: - cluster: - - monitor - indices: - - names: '*' - privileges: - - all - user: - indices: - - names: '*' - privileges: - - read - kibana4_server: - cluster: - - monitor - indices: - - names: '.kibana' - privileges: - - all - native: - logstash: - cluster: - - manage_index_templates - indices: - - names: 'logstash-*' - privileges: - - write - - delete - - create_index - #this will be ignored - its reserved - logstash_system: - cluster: - - manage_index_templates - indices: - - names: 'logstash-*' - privileges: - - write - - delete - - create_index - -#modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed. -- name: Elasticsearch Xpack modify - hosts: localhost - tasks: - - include: elasticsearch/test/integration/debug.yml - roles: - - role: elasticsearch - es_api_port: 9200 - es_instance_name: "security_node" - es_config: - http.port: 9200 - transport.tcp.port: 9300 - discovery.zen.ping.unicast.hosts: "localhost:9300" - xpack.security.enabled: True - xpack.security.authc.realms.file1.type: "file" - xpack.security.authc.realms.file1.order: 0 - xpack.security.authc.realms.native1.type: "native" - xpack.security.authc.realms.native1.order: 1 - vars: - es_heap_size: "1g" - es_templates: true - es_enable_xpack: true - es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" - es_plugins: - - plugin: ingest-attachment - es_xpack_features: - - security - - alerting - es_api_basic_auth_username: elastic - es_api_basic_auth_password: elasticChanged - es_role_mapping: - power_user: - - "cn=admins,dc=example,dc=com" - user: - - "cn=users,dc=example,dc=com" - - "cn=admins,dc=example,dc=com" - es_users: - native: - kibana4_server: - password: changeMe - roles: - - kibana4_server - logstash_system: - #this will be ignored - roles: - - kibana4_server - file: - es_admin: - password: changeMeAgain - roles: - - admin - testUser: - password: changeMeAlso! - roles: - - power_user - - user + - monitoring + - graph + - ml diff --git a/test/integration/xpack/serverspec/default_spec.rb b/test/integration/xpack/serverspec/default_spec.rb index 8a3791a4..5f57e819 100644 --- a/test/integration/xpack/serverspec/default_spec.rb +++ b/test/integration/xpack/serverspec/default_spec.rb @@ -2,6 +2,6 @@ require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) -describe 'Xpack Tests' do +describe 'Xpack upgrade Tests' do include_examples 'xpack::init', vars end diff --git a/test/matrix.yml b/test/matrix.yml index 56740704..91070204 100644 --- a/test/matrix.yml +++ b/test/matrix.yml @@ -7,11 +7,9 @@ OS: - debian-8 - centos-7 TEST_TYPE: - - standard - - package - - config - - multi - - xpack - - xpack-standard - - issue-test + - oss - oss-upgrade + - oss-to-xpack-upgrade + - xpack + - xpack-upgrade + - multi From c8197ee82a5e28809e38d5851b071962e2526c3d Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 19 Jun 2018 12:15:10 +0200 Subject: [PATCH 2/8] Move generic tests into a shared spec file --- .../helpers/serverspec/multi_spec.rb | 85 +-------------- .../helpers/serverspec/shared_spec.rb | 102 ++++++++++++++++++ test/integration/multi.yml | 28 +++-- .../multi/serverspec/default_spec.rb | 9 +- 4 files changed, 132 insertions(+), 92 deletions(-) create mode 100644 test/integration/helpers/serverspec/shared_spec.rb diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index da6daf0a..a9d15e3c 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -4,22 +4,9 @@ shared_examples 'multi::init' do |vars| - describe user('elasticsearch') do - it { should exist } - end - - describe service('node1_elasticsearch') do - it { should be_running } - end - describe service('master_elasticsearch') do it { should be_running } end - - describe package(vars['es_package_name']) do - it { should be_installed } - end - #test configuration parameters have been set - test all appropriately set in config file describe file('/etc/elasticsearch/node1/elasticsearch.yml') do it { should be_file } @@ -63,17 +50,10 @@ end end - describe 'Node listening' do - it 'node should be listening in port 9201' do - expect(port 9201).to be_listening - end - end - #test we started on the correct port was used for master describe 'master started' do it 'master node should be running', :retry => 3, :retry_wait => 10 do - command = command('curl "localhost:9200" | grep name') - #expect(command.stdout).should match '/*master_localhost*/' + expect(json_curl('http://localhost:9200')['name']).to eq('localhost-node1') expect(command.exit_status).to eq(0) end end @@ -87,43 +67,7 @@ end end - describe file('/etc/elasticsearch/templates') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/templates/basic.json') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe 'Template Installed' do - it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - command = command('curl localhost:9200/_template/basic') - expect(command.stdout).to match(/basic/) - expect(command.exit_status).to eq(0) - end - end - - describe 'Template Installed' do - it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - command = command('curl localhost:9201/_template/basic') - expect(command.stdout).to match(/basic/) - expect(command.exit_status).to eq(0) - end - end - #Confirm scripts are on both nodes - describe file('/etc/elasticsearch/node1/scripts') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/node1/scripts/calculate-score.groovy') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - describe file('/etc/elasticsearch/master/scripts') do it { should be_directory } it { should be_owned_by 'elasticsearch' } @@ -195,29 +139,4 @@ it { should be_owned_by 'elasticsearch' } end end - - describe file('/etc/init.d/elasticsearch') do - it { should_not exist } - end - - if ['debian', 'ubuntu'].include?(os[:family]) - describe file('/etc/default/elasticsearch') do - its(:content) { should match '' } - end - end - - if ['centos', 'redhat'].include?(os[:family]) - describe file('/etc/sysconfig/elasticsearch') do - its(:content) { should match '' } - end - end - - describe file('/etc/elasticsearch/elasticsearch.yml') do - it { should_not exist } - end - - describe file('/etc/elasticsearch/logging.yml') do - it { should_not exist } - end -end - +end \ No newline at end of file diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb new file mode 100644 index 00000000..51da553a --- /dev/null +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -0,0 +1,102 @@ +require 'spec_helper' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) + +shared_examples 'shared::init' do |vars| + describe user(vars['es_user']) do + it { should exist } + it { should belong_to_group vars['es_group'] } + it { should have_login_shell '/bin/false' } + it { should have_uid vars['es_user_id'] } if vars.key?('es_user_id') + + its(:encrypted_password) { should eq('*') } + end + + describe package(vars['es_package_name']) do + it { should be_installed } + end + + describe service('node1_elasticsearch') do + it { should be_running } + end + + describe port(vars['es_api_port']) do + it { should be_listening.with('tcp') } + end + + # if vars['es_templates'] + # describe file('/etc/elasticsearch/templates') do + # it { should be_directory } + # it { should be_owned_by vars['es_user'] } + # end + # describe file('/etc/elasticsearch/templates/basic.json') do + # it { should be_file } + # it { should be_owned_by vars['es_user'] } + # end + # describe 'Template Installed' do + # it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do + # command = command('curl localhost:9200/_template/basic') + # expect(command.stdout).to match(/basic/) + # expect(command.exit_status).to eq(0) + # end + # end + # describe 'Template Installed' do + # it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do + # command = command('curl localhost:9201/_template/basic') + # expect(command.stdout).to match(/basic/) + # expect(command.exit_status).to eq(0) + # end + # end + # end + if vars['es_scripts'] + describe file('/etc/elasticsearch/node1/scripts') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + describe file('/etc/elasticsearch/node1/scripts/calculate-score.groovy') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + end + describe file('/etc/init.d/elasticsearch') do + it { should_not exist } + end + if ['debian', 'ubuntu'].include?(os[:family]) + describe file('/etc/default/elasticsearch') do + its(:content) { should match '' } + end + end + + if ['centos', 'redhat'].include?(os[:family]) + describe file('/etc/sysconfig/elasticsearch') do + its(:content) { should match '' } + end + end + + describe file('/etc/elasticsearch/elasticsearch.yml') do + it { should_not exist } + end + + describe file('/etc/elasticsearch/logging.yml') do + it { should_not exist } + end + + vars['es_plugins'].each do |plugin| + name = plugin['plugin'] + url = "http://localhost:#{vars['es_api_port']}" + + describe file('/usr/share/elasticsearch/plugins/'+name) do + it { should be_directory } + it { should be_owned_by vars['es_user'] } + end + it 'should be installed and the right version' do + plugins = curl_json("#{url}/_nodes/plugins") + version = nil + _node, data = plugins['nodes'].first + data['plugins'].each do |p| + version = p['version'] if p['name'] == name + end + expect(version).to eql(vars['es_version']) + end + end +end diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 950ff70e..3dabd9ae 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -1,8 +1,8 @@ --- -#Test ability to deploy multiple instances to a machine +# Test ability to deploy multiple instances to a machine - name: Elasticsearch Multi test - master on 9200 hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml vars: es_enable_xpack: false @@ -13,11 +13,20 @@ es_plugins: - plugin: ingest-geoip roles: - - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } + - role: elasticsearch + es_instance_name: "master" + es_data_dirs: ["/opt/elasticsearch/master"] + es_config: + discovery.zen.ping.unicast.hosts: "localhost:9300" + http.port: 9200 + transport.tcp.port: 9300 + node.data: false + node.master: true + bootstrap.memory_lock: true - name: Elasticsearch Multi test - data on 9201 hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml vars: es_enable_xpack: false @@ -28,5 +37,12 @@ es_plugins: - plugin: ingest-geoip roles: - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } -#Plugins installed for this test are specified in .kitchen.yml under suite + - role: elasticsearch + es_instance_name: "node1" + es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2" + es_config: + discovery.zen.ping.unicast.hosts: "localhost:9300" + http.port: 9201 + transport.tcp.port: 9301 + node.data: true + node.master: false diff --git a/test/integration/multi/serverspec/default_spec.rb b/test/integration/multi/serverspec/default_spec.rb index 94bd05f5..0df6458f 100644 --- a/test/integration/multi/serverspec/default_spec.rb +++ b/test/integration/multi/serverspec/default_spec.rb @@ -1,9 +1,12 @@ require 'multi_spec' +require 'shared_spec' require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) -describe 'Multi Tests' do - include_examples 'multi::init', vars +describe 'Shared Tests' do + include_examples 'shared::init', vars end - +describe 'Multi Tests' do + include_examples 'multi::init', vars +end \ No newline at end of file From f53e6397972fc165a27baeca694a9f0904202591 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 19 Jun 2018 17:17:11 +0200 Subject: [PATCH 3/8] Fix curl_json/json_curl typo --- test/integration/helpers/serverspec/multi_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index a9d15e3c..d66ccac3 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -53,7 +53,7 @@ #test we started on the correct port was used for master describe 'master started' do it 'master node should be running', :retry => 3, :retry_wait => 10 do - expect(json_curl('http://localhost:9200')['name']).to eq('localhost-node1') + expect(curl_json('http://localhost:9200')['name']).to eq('localhost-node1') expect(command.exit_status).to eq(0) end end @@ -139,4 +139,4 @@ it { should be_owned_by 'elasticsearch' } end end -end \ No newline at end of file +end From 433ed7f5e34ab97c73a7c2dc5ecd9e534ca0964a Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 19 Jun 2018 17:20:48 +0200 Subject: [PATCH 4/8] Master is called master and node is called node --- test/integration/helpers/serverspec/multi_spec.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index d66ccac3..6233e5a7 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -53,17 +53,14 @@ #test we started on the correct port was used for master describe 'master started' do it 'master node should be running', :retry => 3, :retry_wait => 10 do - expect(curl_json('http://localhost:9200')['name']).to eq('localhost-node1') - expect(command.exit_status).to eq(0) + expect(curl_json('http://localhost:9200')['name']).to eq('localhost-master') end end #test we started on the correct port was used for node 1 describe 'node1 started' do it 'node should be running', :retry => 3, :retry_wait => 10 do - command = command('curl "localhost:9201" | grep name') - #expect(command.stdout).should match '/*node1_localhost*/' - expect(command.exit_status).to eq(0) + expect(curl_json('http://localhost:9201')['name']).to eq('localhost-node1') end end From 9673fe4679d8b08bef8d150bd1eaecf2554216e1 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 19 Jun 2018 19:30:40 +0200 Subject: [PATCH 5/8] Make sure shared tests work for RedHat and Debian --- .../helpers/serverspec/shared_spec.rb | 52 ++++++++++++++----- test/integration/multi.yml | 4 +- .../multi/serverspec/default_spec.rb | 5 +- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index 51da553a..f973fe2b 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -2,14 +2,37 @@ require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) +families = { + 'Debian' => { + 'shell' => '/bin/false', + 'password' => '*', + 'defaults_path' => '/etc/default/elasticsearch' + }, + 'RedHat' => { + 'shell' => '/sbin/nologin', + 'password' => '!!', + 'defaults_path' => '/etc/sysconfig/elasticsearch' + } +} + +family = families[vars['ansible_os_family']] + +es_api_url = "http://localhost:#{vars['es_api_port']}" + shared_examples 'shared::init' do |vars| + describe 'version check' do + it 'should be reported as version '+vars['es_version'] do + expect(curl_json(es_api_url)['version']['number']).to eq(vars['es_version']) + end + end describe user(vars['es_user']) do it { should exist } it { should belong_to_group vars['es_group'] } - it { should have_login_shell '/bin/false' } it { should have_uid vars['es_user_id'] } if vars.key?('es_user_id') - its(:encrypted_password) { should eq('*') } + it { should have_login_shell family['shell'] } + + its(:encrypted_password) { should eq(family['password']) } end describe package(vars['es_package_name']) do @@ -61,16 +84,9 @@ describe file('/etc/init.d/elasticsearch') do it { should_not exist } end - if ['debian', 'ubuntu'].include?(os[:family]) - describe file('/etc/default/elasticsearch') do - its(:content) { should match '' } - end - end - if ['centos', 'redhat'].include?(os[:family]) - describe file('/etc/sysconfig/elasticsearch') do - its(:content) { should match '' } - end + describe file(family['defaults_path']) do + its(:content) { should match '' } end describe file('/etc/elasticsearch/elasticsearch.yml') do @@ -83,14 +99,13 @@ vars['es_plugins'].each do |plugin| name = plugin['plugin'] - url = "http://localhost:#{vars['es_api_port']}" describe file('/usr/share/elasticsearch/plugins/'+name) do it { should be_directory } it { should be_owned_by vars['es_user'] } end it 'should be installed and the right version' do - plugins = curl_json("#{url}/_nodes/plugins") + plugins = curl_json("#{es_api_url}/_nodes/plugins") version = nil _node, data = plugins['nodes'].first data['plugins'].each do |p| @@ -99,4 +114,15 @@ expect(version).to eql(vars['es_version']) end end + describe file('/etc/elasticsearch/node1/elasticsearch.yml') do + it { should contain 'node.name: localhost-node1' } + it { should contain 'cluster.name: elasticsearch' } + if vars['es_major_version'] == '6.x' + it { should_not contain 'path.conf: /etc/elasticsearch/node1' } + else + it { should contain 'path.conf: /etc/elasticsearch/node1' } + end + # it { should contain 'path.data: /var/lib/elasticsearch/localhost-node1' } + it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' } + end end diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 3dabd9ae..85e916bf 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -39,7 +39,9 @@ roles: - role: elasticsearch es_instance_name: "node1" - es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2" + es_data_dirs: + - "/opt/elasticsearch/data-1" + - "/opt/elasticsearch/data-2" es_config: discovery.zen.ping.unicast.hosts: "localhost:9300" http.port: 9201 diff --git a/test/integration/multi/serverspec/default_spec.rb b/test/integration/multi/serverspec/default_spec.rb index 0df6458f..718b6816 100644 --- a/test/integration/multi/serverspec/default_spec.rb +++ b/test/integration/multi/serverspec/default_spec.rb @@ -3,10 +3,7 @@ require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) -describe 'Shared Tests' do - include_examples 'shared::init', vars -end - describe 'Multi Tests' do + include_examples 'shared::init', vars include_examples 'multi::init', vars end \ No newline at end of file From c325ff79faf31cb13a7f5b390a6d0f247c695af5 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 19 Jun 2018 21:17:10 +0200 Subject: [PATCH 6/8] Move all shared tests into the shared test helper --- test/integration/config.yml | 40 ---- test/integration/config/config.yml | 2 - .../config/serverspec/default_spec.rb | 8 - .../helpers/serverspec/multi_spec.rb | 22 +-- .../helpers/serverspec/oss_spec.rb | 98 +--------- .../serverspec/oss_to_xpack_upgrade_spec.rb | 8 - .../helpers/serverspec/oss_upgrade_spec.rb | 5 - .../helpers/serverspec/shared_spec.rb | 138 +++++++++----- .../helpers/serverspec/xpack_spec.rb | 129 +------------ .../helpers/serverspec/xpack_upgrade_spec.rb | 177 +----------------- test/integration/issue-test.yml | 20 +- test/integration/multi.yml | 46 ++--- test/integration/oss-to-xpack-upgrade.yml | 10 +- .../serverspec/default_spec.rb | 2 + test/integration/oss-upgrade.yml | 4 +- .../oss-upgrade/serverspec/default_spec.rb | 2 + test/integration/oss.yml | 5 +- .../oss/serverspec/default_spec.rb | 2 + test/integration/package.yml | 33 ---- test/integration/package/package.yaml | 2 - .../package/serverspec/default_spec.rb | 7 - test/integration/xpack-upgrade.yml | 42 +++-- .../xpack-upgrade/serverspec/default_spec.rb | 2 + test/integration/xpack.yml | 10 +- .../xpack/serverspec/default_spec.rb | 2 + 25 files changed, 205 insertions(+), 611 deletions(-) delete mode 100644 test/integration/config.yml delete mode 100644 test/integration/config/config.yml delete mode 100644 test/integration/config/serverspec/default_spec.rb delete mode 100644 test/integration/package.yml delete mode 100644 test/integration/package/package.yaml delete mode 100644 test/integration/package/serverspec/default_spec.rb diff --git a/test/integration/config.yml b/test/integration/config.yml deleted file mode 100644 index 9d74e52e..00000000 --- a/test/integration/config.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -#Test explicit setting of parameters and variables -- name: Elasticsearch Config initial - hosts: localhost - tasks: - - include: elasticsearch/test/integration/debug.yml - roles: - #expand to all available parameters - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: false } } - vars: - es_enable_xpack: false - es_scripts: false - es_templates: false - es_version_lock: false - es_heap_size: 1g - es_api_port: 9201 - es_plugins: - - plugin: ingest-geoip - -#Modify the above configuration. Final test should evaluate this configuration. Also tests the plugins are added and removed. -- name: Elasticsearch Config test modify - hosts: localhost - tasks: - - include: elasticsearch/test/integration/debug.yml - roles: - #expand to all available parameters - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: true } } - vars: - es_enable_xpack: false - es_scripts: false - es_templates: false - es_version_lock: false - es_heap_size: 1g - #add a custom log4j file - es_config_log4j2: "./files/logging/log4j2.properties.custom.j2" - es_api_port: 9401 - es_max_threads: 3000 - es_plugins: - - plugin: ingest-attachment - - plugin: ingest-user-agent diff --git a/test/integration/config/config.yml b/test/integration/config/config.yml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/config/config.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/config/serverspec/default_spec.rb b/test/integration/config/serverspec/default_spec.rb deleted file mode 100644 index b76a5098..00000000 --- a/test/integration/config/serverspec/default_spec.rb +++ /dev/null @@ -1,8 +0,0 @@ -require 'config_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'Config Tests' do - include_examples 'config::init', vars -end - diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index 6233e5a7..e2a34aed 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -8,21 +8,21 @@ it { should be_running } end #test configuration parameters have been set - test all appropriately set in config file - describe file('/etc/elasticsearch/node1/elasticsearch.yml') do + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should be_file } it { should contain 'http.port: 9201' } it { should contain 'transport.tcp.port: 9301' } it { should contain 'node.data: true' } it { should contain 'node.master: false' } - it { should contain 'node.name: localhost-node1' } + it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should_not contain 'bootstrap.memory_lock: true' } if vars['es_major_version'] == '6.x' - it { should_not contain 'path.conf: /etc/elasticsearch/node1' } + it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } else - it { should contain 'path.conf: /etc/elasticsearch/node1' } + it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } end - it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' } - it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' } + it { should contain "path.data: /opt/elasticsearch/data-1/localhost-#{vars['es_instance_name']},/opt/elasticsearch/data-2/localhost-#{vars['es_instance_name']}" } + it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" } end @@ -58,9 +58,9 @@ end #test we started on the correct port was used for node 1 - describe 'node1 started' do + describe "#{vars['es_instance_name']} started" do it 'node should be running', :retry => 3, :retry_wait => 10 do - expect(curl_json('http://localhost:9201')['name']).to eq('localhost-node1') + expect(curl_json('http://localhost:9201')['name']).to eq("localhost-#{vars['es_instance_name']}") end end @@ -81,13 +81,13 @@ it { should be_owned_by 'elasticsearch' } end - describe file('/opt/elasticsearch/data-1/localhost-node1') do + describe file("/opt/elasticsearch/data-1/localhost-#{vars['es_instance_name']}") do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - describe file('/opt/elasticsearch/data-2/localhost-node1') do + describe file("/opt/elasticsearch/data-2/localhost-#{vars['es_instance_name']}") do it { should be_directory } it { should be_owned_by 'elasticsearch' } end @@ -99,7 +99,7 @@ end #test to make sure mlock was not applied - describe command('curl -s "localhost:9201/_nodes/localhost-node1/process?pretty=true" | grep mlockall') do + describe command("curl -s 'localhost:9201/_nodes/localhost-#{vars['es_instance_name']}/process?pretty=true' | grep mlockall") do its(:stdout) { should match /false/ } its(:exit_status) { should eq 0 } end diff --git a/test/integration/helpers/serverspec/oss_spec.rb b/test/integration/helpers/serverspec/oss_spec.rb index be2cb7b5..d8d94093 100644 --- a/test/integration/helpers/serverspec/oss_spec.rb +++ b/test/integration/helpers/serverspec/oss_spec.rb @@ -1,105 +1,13 @@ require 'spec_helper' shared_examples 'oss::init' do |vars| - - describe user('elasticsearch') do - it { should exist } - end - - describe service('node1_elasticsearch') do - it { should be_running } - end - - describe package(vars['es_package_name']) do - it { should be_installed } - end - - describe file('/etc/elasticsearch/node1/elasticsearch.yml') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/node1/log4j2.properties') do + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/log4j2.properties") do it { should be_file } it { should be_owned_by 'elasticsearch' } it { should_not contain 'CUSTOM LOG4J FILE' } end - - describe file('/etc/elasticsearch/node1/jvm.options') do + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/jvm.options") do it { should be_file } - it { should be_owned_by 'elasticsearch' } + it { should be_owned_by vars['es_user'] } end - - describe file('/etc/elasticsearch/node1/elasticsearch.yml') do - it { should contain 'node.name: localhost-node1' } - it { should contain 'cluster.name: elasticsearch' } - if vars['es_major_version'] == '6.x' - it { should_not contain 'path.conf: /etc/elasticsearch/node1' } - else - it { should contain 'path.conf: /etc/elasticsearch/node1' } - end - it { should contain 'path.data: /var/lib/elasticsearch/localhost-node1' } - it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' } - end - - describe 'Node listening' do - it 'listening in port 9200' do - expect(port 9200).to be_listening - end - end - - describe 'version check' do - it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200 | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) - end - end - - describe file('/etc/init.d/elasticsearch') do - it { should_not exist } - end - - if ['debian', 'ubuntu'].include?(os[:family]) - describe file('/etc/default/elasticsearch') do - its(:content) { should match '' } - end - end - - if ['centos', 'redhat'].include?(os[:family]) - describe file('/etc/sysconfig/elasticsearch') do - its(:content) { should match '' } - end - end - - describe file('/etc/elasticsearch/elasticsearch.yml') do - it { should_not exist } - end - - describe file('/etc/elasticsearch/logging.yml') do - it { should_not exist } - end - - for plugin in vars['es_plugins'] - name = plugin['plugin'] - - describe file('/usr/share/elasticsearch/plugins/'+name) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - it 'should be installed and the right version' do - plugins = curl_json('http://localhost:9200/_nodes/plugins') - version = nil - node, data = plugins['nodes'].first - data['plugins'].each do |plugin| - if plugin['name'] == name - version = plugin['version'] - end - end - expect(version).to eql(vars['es_version']) - end - end - - end - diff --git a/test/integration/helpers/serverspec/oss_to_xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/oss_to_xpack_upgrade_spec.rb index 8cb5c05a..4eeda4ce 100644 --- a/test/integration/helpers/serverspec/oss_to_xpack_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/oss_to_xpack_upgrade_spec.rb @@ -1,12 +1,4 @@ require 'spec_helper' shared_examples 'oss_to_xpack_upgrade::init' do |vars| - describe 'version check' do - it 'should be reported as version '+vars['es_version'] do - expect(curl_json('http://localhost:9200', username='elastic', password='changeme')['version']['number']).to eq(vars['es_version']) - end - it 'should be be running the standard (xpack) version' do - expect(curl_json('http://localhost:9200/_xpack', username='elastic', password='changeme')['tagline']).to eq('You know, for X') - end - end end diff --git a/test/integration/helpers/serverspec/oss_upgrade_spec.rb b/test/integration/helpers/serverspec/oss_upgrade_spec.rb index db0ea059..f09e937f 100644 --- a/test/integration/helpers/serverspec/oss_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/oss_upgrade_spec.rb @@ -1,9 +1,4 @@ require 'spec_helper' shared_examples 'oss_upgrade::init' do |vars| - describe 'version check' do - it 'should be reported as version '+vars['es_version'] do - expect(curl_json('http://localhost:9200')['version']['number']).to eq(vars['es_version']) - end - end end diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index f973fe2b..664595c3 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -18,11 +18,57 @@ family = families[vars['ansible_os_family']] es_api_url = "http://localhost:#{vars['es_api_port']}" +username = vars['es_api_basic_auth_username'] +password = vars['es_api_basic_auth_password'] shared_examples 'shared::init' do |vars| describe 'version check' do it 'should be reported as version '+vars['es_version'] do - expect(curl_json(es_api_url)['version']['number']).to eq(vars['es_version']) + expect(curl_json(es_api_url, username=username, password=password)['version']['number']).to eq(vars['es_version']) + end + end + describe 'xpack checks' do + if vars['es_enable_xpack'] + it 'should be be running the xpack version' do + expect(curl_json("#{es_api_url}/_xpack", username=username, password=password)['tagline']).to eq('You know, for X') + end + it 'xpack should be activated' do + expect(curl_json("#{es_api_url}/_license", username=username, password=password)['license']['status']).to eq('active') + end + features = curl_json("#{es_api_url}/_xpack", username=username, password=password) + curl_json("#{es_api_url}/_xpack", username=username, password=password)['features'].each do |feature,values| + enabled = vars['es_xpack_features'].include? feature + status = if enabled then 'enabled' else 'disabled' end + it "the xpack feature '#{feature}' to be #{status}" do + expect(values['enabled'] = enabled) + end + end + # X-Pack is no longer installed as a plugin in elasticsearch + if vars['es_major_version'] == '5.x' + describe file('/usr/share/elasticsearch/plugins/x-pack') do + it { should be_directory } + it { should be_owned_by vars['es_user'] } + end + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/x-pack") do + it { should be_directory } + it { should be_owned_by vars['es_user'] } + end + describe 'x-pack-core plugin' do + it 'should be installed with the correct version' do + plugins = curl_json("#{es_api_url}/_nodes/plugins", username=username, password=password) + node, data = plugins['nodes'].first + version = 'plugin not found' + name = 'x-pack' + + data['plugins'].each do |plugin| + if plugin['name'] == name + version = plugin['version'] + end + end + expect(version).to eql(vars['es_version']) + end + end + end end end describe user(vars['es_user']) do @@ -39,7 +85,7 @@ it { should be_installed } end - describe service('node1_elasticsearch') do + describe service("#{vars['es_instance_name']}_elasticsearch") do it { should be_running } end @@ -47,36 +93,31 @@ it { should be_listening.with('tcp') } end - # if vars['es_templates'] - # describe file('/etc/elasticsearch/templates') do - # it { should be_directory } - # it { should be_owned_by vars['es_user'] } - # end - # describe file('/etc/elasticsearch/templates/basic.json') do - # it { should be_file } - # it { should be_owned_by vars['es_user'] } - # end - # describe 'Template Installed' do - # it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - # command = command('curl localhost:9200/_template/basic') - # expect(command.stdout).to match(/basic/) - # expect(command.exit_status).to eq(0) - # end - # end - # describe 'Template Installed' do - # it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - # command = command('curl localhost:9201/_template/basic') - # expect(command.stdout).to match(/basic/) - # expect(command.exit_status).to eq(0) - # end - # end - # end + if vars['es_templates'] + describe file('/etc/elasticsearch/templates') do + it { should be_directory } + it { should be_owned_by vars['es_user'] } + end + describe file('/etc/elasticsearch/templates/basic.json') do + it { should be_file } + it { should be_owned_by vars['es_user'] } + end + #This is possibly subject to format changes in the response across versions so may fail in the future + describe 'Template Contents Correct' do + it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do + template = curl_json("#{es_api_url}/_template/basic", username=username, password=password) + expect(template.key?('basic')) + expect(template['basic']['settings']['index']['number_of_shards']).to eq("1") + expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false) + end + end + end if vars['es_scripts'] - describe file('/etc/elasticsearch/node1/scripts') do + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/scripts") do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - describe file('/etc/elasticsearch/node1/scripts/calculate-score.groovy') do + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/scripts/calculate-score.groovy") do it { should be_file } it { should be_owned_by 'elasticsearch' } end @@ -97,32 +138,33 @@ it { should_not exist } end - vars['es_plugins'].each do |plugin| - name = plugin['plugin'] - - describe file('/usr/share/elasticsearch/plugins/'+name) do - it { should be_directory } - it { should be_owned_by vars['es_user'] } - end - it 'should be installed and the right version' do - plugins = curl_json("#{es_api_url}/_nodes/plugins") - version = nil - _node, data = plugins['nodes'].first - data['plugins'].each do |p| - version = p['version'] if p['name'] == name + if vars.key?('es_plugins') + vars['es_plugins'].each do |plugin| + name = plugin['plugin'] + describe file('/usr/share/elasticsearch/plugins/'+name) do + it { should be_directory } + it { should be_owned_by vars['es_user'] } + end + it 'should be installed and the right version' do + plugins = curl_json("#{es_api_url}/_nodes/plugins", username=username, password=password) + version = nil + _node, data = plugins['nodes'].first + data['plugins'].each do |p| + version = p['version'] if p['name'] == name + end + expect(version).to eql(vars['es_version']) end - expect(version).to eql(vars['es_version']) end end - describe file('/etc/elasticsearch/node1/elasticsearch.yml') do - it { should contain 'node.name: localhost-node1' } + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do + it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should contain 'cluster.name: elasticsearch' } if vars['es_major_version'] == '6.x' - it { should_not contain 'path.conf: /etc/elasticsearch/node1' } + it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } else - it { should contain 'path.conf: /etc/elasticsearch/node1' } + it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } end - # it { should contain 'path.data: /var/lib/elasticsearch/localhost-node1' } - it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' } + its(:content) { should match "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" } + its(:content) { should match "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" } end end diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index f50694ee..926a0137 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -1,138 +1,17 @@ require 'spec_helper' shared_examples 'xpack::init' do |vars| - - describe user('elasticsearch') do - it { should exist } - end - - describe service('security_node_elasticsearch') do - it { should be_running } - end - - describe package(vars['es_package_name']) do - it { should be_installed } - end - - describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/security_node/log4j2.properties') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do - it { should contain 'node.name: localhost-security_node' } + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do + it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should contain 'cluster.name: elasticsearch' } if vars['es_major_version'] == '6.x' it { should_not contain 'path.conf: /etc/elasticsearch/security_node' } else it { should contain 'path.conf: /etc/elasticsearch/security_node' } end - it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' } - it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' } + it { should contain "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" } + it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" } it { should contain 'xpack.security.enabled: false' } it { should contain 'xpack.watcher.enabled: false' } - - end - - describe 'Node listening' do - it 'listening in port 9200' do - expect(port 9200).to be_listening - end - end - - describe 'version check' do - it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200 | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) - end - end - - describe file('/etc/init.d/elasticsearch') do - it { should_not exist } - end - - if ['debian', 'ubuntu'].include?(os[:family]) - describe file('/etc/default/elasticsearch') do - its(:content) { should match '' } - end - end - - if ['centos', 'redhat'].include?(os[:family]) - describe file('/etc/sysconfig/elasticsearch') do - its(:content) { should match '' } - end - end - - describe file('/etc/elasticsearch/elasticsearch.yml') do - it { should_not exist } - end - - describe file('/etc/elasticsearch/logging.yml') do - it { should_not exist } - end - - #Xpack specific tests - describe file('/usr/share/elasticsearch/plugins') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } end - - #Test if x-pack is activated - describe 'x-pack activation' do - it 'should be activated and valid' do - command = command('curl -s localhost:9200/_license?pretty=true') - expect(command.stdout).to match('"status" : "active"') - expect(command.exit_status).to eq(0) - end - end - - # X-Pack is no longer installed as a plugin in elasticsearch - if vars['es_major_version'] == '5.x' - describe file('/usr/share/elasticsearch/plugins/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe 'x-pack-core plugin' do - it 'should be installed with the correct version' do - plugins = curl_json('http://localhost:9200/_nodes/plugins') - node, data = plugins['nodes'].first - version = 'plugin not found' - name = 'x-pack' - - data['plugins'].each do |plugin| - if plugin['name'] == name - version = plugin['version'] - end - end - expect(version).to eql(vars['es_version']) - end - end - - describe file('/etc/elasticsearch/security_node/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - end - - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do - its(:exit_status) { should eq 0 } - end - - describe command('curl -s localhost:9200/_xpack') do - its(:stdout_as_json) { should include('features' => include('security' => include('enabled' => false))) } - its(:stdout_as_json) { should include('features' => include('watcher' => include('enabled' => false))) } - its(:stdout_as_json) { should include('features' => include('graph' => include('enabled' => true))) } - its(:stdout_as_json) { should include('features' => include('monitoring' => include('enabled' => true))) } - its(:stdout_as_json) { should include('features' => include('ml' => include('enabled' => true))) } - end - end - diff --git a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb index a3658858..4afc6226 100644 --- a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb @@ -3,152 +3,26 @@ vars = JSON.parse(File.read('/tmp/vars.json')) shared_examples 'xpack_upgrade::init' do |vars| - - describe user('elasticsearch') do - it { should exist } - end - - describe service('security_node_elasticsearch') do - it { should be_running } - end - - describe package(vars['es_package_name']) do - it { should be_installed } - end - - describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/security_node/log4j2.properties') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do - it { should contain 'node.name: localhost-security_node' } + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do + it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should contain 'cluster.name: elasticsearch' } if vars['es_major_version'] == '6.x' - it { should_not contain 'path.conf: /etc/elasticsearch/security_node' } + it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } else - it { should contain 'path.conf: /etc/elasticsearch/security_node' } - end - it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' } - it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' } - end - - describe 'Node listening' do - it 'listening in port 9200' do - expect(port 9200).to be_listening - end - end - - describe 'version check' do - it 'should be reported as version '+vars['es_version'] do - command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number') - expect(command.stdout).to match(vars['es_version']) - expect(command.exit_status).to eq(0) - end - end - - describe file('/etc/init.d/elasticsearch') do - it { should_not exist } - end - - if ['debian', 'ubuntu'].include?(os[:family]) - describe file('/etc/default/elasticsearch') do - its(:content) { should match '' } - end - end - - if ['centos', 'redhat'].include?(os[:family]) - describe file('/etc/sysconfig/elasticsearch') do - its(:content) { should match '' } - end - end - - describe file('/etc/elasticsearch/elasticsearch.yml') do - it { should_not exist } - end - - describe file('/etc/elasticsearch/logging.yml') do - it { should_not exist } - end - - # X-Pack is no longer installed as a plugin in elasticsearch - if vars['es_major_version'] == '5.x' - describe file('/usr/share/elasticsearch/plugins') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/usr/share/elasticsearch/plugins/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do - its(:exit_status) { should eq 0 } - end - - describe file('/usr/share/elasticsearch/plugins/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe 'xpack plugin' do - it 'should be installed with the correct version' do - plugins = curl_json('http://localhost:9200/_nodes/plugins', username='es_admin', password='changeMeAgain') - node, data = plugins['nodes'].first - version = 'plugin not found' - name = 'x-pack' - - data['plugins'].each do |plugin| - if plugin['name'] == name - version = plugin['version'] - end - end - expect(version).to eql(vars['es_version']) - end - end - end - - #Test if x-pack is activated - describe 'x-pack activation' do - it 'should be activated and valid' do - command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain') - expect(command.stdout).to match('"status" : "active"') - expect(command.exit_status).to eq(0) - end - end - - describe file('/etc/elasticsearch/security_node/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - for plugin in vars['es_plugins'] - plugin = plugin['plugin'] - - describe file('/usr/share/elasticsearch/plugins/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do - its(:exit_status) { should eq 0 } + it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } end + it { should contain "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" } + it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" } end #Test users file, users_roles and roles.yml - describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/users_roles') do + describe file("/etc/elasticsearch/#{vars['es_instance_name']}#{vars['es_xpack_conf_subdir']}/users_roles") do it { should be_owned_by 'elasticsearch' } it { should contain 'admin:es_admin' } it { should contain 'power_user:testUser' } end - describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/users') do + describe file("/etc/elasticsearch/#{vars['es_instance_name']}#{vars['es_xpack_conf_subdir']}/users") do it { should be_owned_by 'elasticsearch' } it { should contain 'testUser:' } it { should contain 'es_admin:' } @@ -161,36 +35,7 @@ end end - describe file('/etc/elasticsearch/templates') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/templates/basic.json') do - it { should be_file } - it { should be_owned_by 'elasticsearch' } - end - - describe 'Template Installed' do - it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain') - expect(command.stdout).to match(/basic/) - expect(command.exit_status).to eq(0) - end - end - - #This is possibly subject to format changes in the response across versions so may fail in the future - describe 'Template Contents Correct' do - it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - template = curl_json('http://localhost:9200/_template/basic', username='es_admin', password='changeMeAgain') - expect(template.key?('basic')) - expect(template['basic']['settings']['index']['number_of_shards']).to eq("1") - expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false) - end - end - - #Test contents of Elasticsearch.yml file - describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do + describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should contain 'security.authc.realms.file1.order: 0' } it { should contain 'security.authc.realms.file1.type: file' } it { should contain 'security.authc.realms.native1.order: 1' } @@ -198,7 +43,7 @@ end #Test contents of role_mapping.yml - describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/role_mapping.yml') do + describe file("/etc/elasticsearch/#{vars['es_instance_name']}#{vars['es_xpack_conf_subdir']}/role_mapping.yml") do it { should be_owned_by 'elasticsearch' } it { should contain 'power_user:' } it { should contain '- cn=admins,dc=example,dc=com' } @@ -207,7 +52,6 @@ end #check accounts are correct i.e. we can auth and they have the correct roles - describe 'kibana4_server access check' do it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9200/ -u kibana4_server:changeMe | grep number') @@ -257,4 +101,3 @@ end end end - diff --git a/test/integration/issue-test.yml b/test/integration/issue-test.yml index 36229fca..30a2cba2 100644 --- a/test/integration/issue-test.yml +++ b/test/integration/issue-test.yml @@ -5,19 +5,19 @@ - name: Simple Example hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - role: elasticsearch - es_instance_name: "security_node" - es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" - es_config: - xpack.security.enabled: True - xpack.security.authc.realms.file1.type: "file" - xpack.security.authc.realms.file1.order: 1 - xpack.security.authc.realms.native1.type: "native" - xpack.security.authc.realms.native1.order: 0 + - elasticsearch vars: + es_instance_name: "security_node" + es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" + es_config: + xpack.security.enabled: True + xpack.security.authc.realms.file1.type: "file" + xpack.security.authc.realms.file1.order: 1 + xpack.security.authc.realms.native1.type: "native" + xpack.security.authc.realms.native1.order: 0 es_heap_size: "1g" es_enable_xpack: true es_plugins: diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 85e916bf..6fda7896 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -4,7 +4,18 @@ hosts: localhost post_tasks: - include: elasticsearch/test/integration/debug.yml + roles: + - elasticsearch vars: + es_instance_name: "master" + es_data_dirs: ["/opt/elasticsearch/master"] + es_config: + discovery.zen.ping.unicast.hosts: "localhost:9300" + http.port: 9200 + transport.tcp.port: 9300 + node.data: false + node.master: true + bootstrap.memory_lock: true es_enable_xpack: false es_scripts: true es_templates: true @@ -12,22 +23,13 @@ es_api_port: 9200 es_plugins: - plugin: ingest-geoip - roles: - - role: elasticsearch - es_instance_name: "master" - es_data_dirs: ["/opt/elasticsearch/master"] - es_config: - discovery.zen.ping.unicast.hosts: "localhost:9300" - http.port: 9200 - transport.tcp.port: 9300 - node.data: false - node.master: true - bootstrap.memory_lock: true - name: Elasticsearch Multi test - data on 9201 hosts: localhost post_tasks: - include: elasticsearch/test/integration/debug.yml + roles: + - role: elasticsearch vars: es_enable_xpack: false es_scripts: true @@ -36,15 +38,13 @@ es_api_port: 9201 es_plugins: - plugin: ingest-geoip - roles: - - role: elasticsearch - es_instance_name: "node1" - es_data_dirs: - - "/opt/elasticsearch/data-1" - - "/opt/elasticsearch/data-2" - es_config: - discovery.zen.ping.unicast.hosts: "localhost:9300" - http.port: 9201 - transport.tcp.port: 9301 - node.data: true - node.master: false + es_instance_name: "node1" + es_data_dirs: + - "/opt/elasticsearch/data-1" + - "/opt/elasticsearch/data-2" + es_config: + discovery.zen.ping.unicast.hosts: "localhost:9300" + http.port: 9201 + transport.tcp.port: 9301 + node.data: true + node.master: false diff --git a/test/integration/oss-to-xpack-upgrade.yml b/test/integration/oss-to-xpack-upgrade.yml index 1c74c0f8..c1ea5586 100644 --- a/test/integration/oss-to-xpack-upgrade.yml +++ b/test/integration/oss-to-xpack-upgrade.yml @@ -1,22 +1,24 @@ --- - name: Standard test for single node setup. Tests idempotence. hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - { role: elasticsearch, es_instance_name: "node1" } + - elasticsearch vars: + es_instance_name: "node1" es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade es_enable_xpack: false es_heap_size: "1g" - name: Standard test for single node setup. Tests idempotence. hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - { role: elasticsearch, es_instance_name: "node1" } + - elasticsearch vars: + es_instance_name: "node1" es_enable_xpack: true es_api_basic_auth_username: elastic es_api_basic_auth_password: changeme diff --git a/test/integration/oss-to-xpack-upgrade/serverspec/default_spec.rb b/test/integration/oss-to-xpack-upgrade/serverspec/default_spec.rb index ff79e845..c633ffaf 100644 --- a/test/integration/oss-to-xpack-upgrade/serverspec/default_spec.rb +++ b/test/integration/oss-to-xpack-upgrade/serverspec/default_spec.rb @@ -1,7 +1,9 @@ require 'oss_to_xpack_upgrade_spec' +require 'shared_spec' require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) describe 'oss to xpack upgrade Tests' do + include_examples 'shared::init', vars include_examples 'oss_to_xpack_upgrade::init', vars end diff --git a/test/integration/oss-upgrade.yml b/test/integration/oss-upgrade.yml index cc3f5301..f98715b8 100644 --- a/test/integration/oss-upgrade.yml +++ b/test/integration/oss-upgrade.yml @@ -1,7 +1,7 @@ --- - name: Standard test for single node setup. Tests idempotence. hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - { role: elasticsearch, es_instance_name: "node1" } @@ -12,7 +12,7 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - { role: elasticsearch, es_instance_name: "node1" } diff --git a/test/integration/oss-upgrade/serverspec/default_spec.rb b/test/integration/oss-upgrade/serverspec/default_spec.rb index ee360e74..939ee16d 100644 --- a/test/integration/oss-upgrade/serverspec/default_spec.rb +++ b/test/integration/oss-upgrade/serverspec/default_spec.rb @@ -1,7 +1,9 @@ require 'oss_upgrade_spec' +require 'shared_spec' require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) describe 'oss upgrade Tests' do include_examples 'oss_upgrade::init', vars + include_examples 'shared::init', vars end diff --git a/test/integration/oss.yml b/test/integration/oss.yml index 86e8cc23..4bcf6b94 100644 --- a/test/integration/oss.yml +++ b/test/integration/oss.yml @@ -1,11 +1,12 @@ --- - name: Standard test for single node setup. Tests idempotence. hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - { role: elasticsearch, es_instance_name: "node1" } + - elasticsearch vars: + es_instance_name: "node1" es_enable_xpack: false es_heap_size: "1g" es_plugins: diff --git a/test/integration/oss/serverspec/default_spec.rb b/test/integration/oss/serverspec/default_spec.rb index a5d0f02d..7b506670 100644 --- a/test/integration/oss/serverspec/default_spec.rb +++ b/test/integration/oss/serverspec/default_spec.rb @@ -1,9 +1,11 @@ require 'oss_spec' +require 'shared_spec' require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) describe 'OSS Tests' do include_examples 'oss::init', vars + include_examples 'shared::init', vars end diff --git a/test/integration/package.yml b/test/integration/package.yml deleted file mode 100644 index c6c7f43c..00000000 --- a/test/integration/package.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Elasticsearch Package test intial - hosts: localhost - tasks: - - include: elasticsearch/test/integration/debug.yml - roles: - - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } - vars: - es_enable_xpack: false - es_scripts: true - es_templates: true - es_heap_size: "1g" - es_api_port: 9200 - es_plugins: - - plugin: ingest-geoip - - -#Tests the plugins have been correctly removed and ES can be upgraded between minor versions. All plugins will be removed and re-installed. -- name: Elasticsearch Package test modify - hosts: localhost - tasks: - - include: elasticsearch/test/integration/debug.yml - roles: - - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } - vars: - es_enable_xpack: false - es_scripts: true - es_templates: true - es_heap_size: "1g" - es_api_port: 9200 - es_plugins: - - plugin: ingest-attachment - - plugin: ingest-geoip diff --git a/test/integration/package/package.yaml b/test/integration/package/package.yaml deleted file mode 100644 index a3c37e19..00000000 --- a/test/integration/package/package.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- host: test-kitchen diff --git a/test/integration/package/serverspec/default_spec.rb b/test/integration/package/serverspec/default_spec.rb deleted file mode 100644 index ee8f77bd..00000000 --- a/test/integration/package/serverspec/default_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'package_spec' -require 'json' -vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'Package Tests' do - include_examples 'package::init', vars -end diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index 5668dfd0..5620489e 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -1,13 +1,21 @@ --- - name: Elasticsearch Xpack tests initial hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", - "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, - es_instance_name: "security_node" } + - elasticsearch vars: + es_instance_name: "node1" + es_api_port: 9200 + es_config: + http.port: 9200 + transport.tcp.port: 9300 + discovery.zen.ping.unicast.hosts: "localhost:9300" + xpack.security.authc.realms.file1.type: "file" + xpack.security.authc.realms.file1.order: 0 + xpack.security.authc.realms.native1.type: "native" + xpack.security.authc.realms.native1.order: 1 es_heap_size: "1g" es_templates: true es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade @@ -103,22 +111,22 @@ #modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed. - name: Elasticsearch Xpack modify hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - role: elasticsearch - es_api_port: 9200 - es_instance_name: "security_node" - es_config: - http.port: 9200 - transport.tcp.port: 9300 - discovery.zen.ping.unicast.hosts: "localhost:9300" - xpack.security.enabled: True - xpack.security.authc.realms.file1.type: "file" - xpack.security.authc.realms.file1.order: 0 - xpack.security.authc.realms.native1.type: "native" - xpack.security.authc.realms.native1.order: 1 + - elasticsearch vars: + es_api_port: 9200 + es_instance_name: "node1" + es_config: + http.port: 9200 + transport.tcp.port: 9300 + discovery.zen.ping.unicast.hosts: "localhost:9300" + xpack.security.enabled: True + xpack.security.authc.realms.file1.type: "file" + xpack.security.authc.realms.file1.order: 0 + xpack.security.authc.realms.native1.type: "native" + xpack.security.authc.realms.native1.order: 1 es_heap_size: "1g" es_templates: true es_enable_xpack: true diff --git a/test/integration/xpack-upgrade/serverspec/default_spec.rb b/test/integration/xpack-upgrade/serverspec/default_spec.rb index c2452eec..043bc5ca 100644 --- a/test/integration/xpack-upgrade/serverspec/default_spec.rb +++ b/test/integration/xpack-upgrade/serverspec/default_spec.rb @@ -1,7 +1,9 @@ require 'xpack_upgrade_spec' +require 'shared_spec' require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) describe 'Xpack upgrade Tests' do + include_examples 'shared::init', vars include_examples 'xpack_upgrade::init', vars end diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 6789a7e2..1f143836 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -2,10 +2,16 @@ --- - name: Elasticsearch Xpack tests - no security and manual download hosts: localhost - tasks: + post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "security_node" } + - role: elasticsearch + es_api_port: 9200 + es_instance_name: "node1" + es_config: + http.port: 9200 + transport.tcp.port: 9300 + discovery.zen.ping.unicast.hosts: "localhost:9300" vars: es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip" es_heap_size: 2g diff --git a/test/integration/xpack/serverspec/default_spec.rb b/test/integration/xpack/serverspec/default_spec.rb index 5f57e819..eaf7a101 100644 --- a/test/integration/xpack/serverspec/default_spec.rb +++ b/test/integration/xpack/serverspec/default_spec.rb @@ -1,7 +1,9 @@ require 'xpack_spec' +require 'shared_spec' require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) describe 'Xpack upgrade Tests' do + include_examples 'shared::init', vars include_examples 'xpack::init', vars end From e1cfb90f5d86b4b7a1ee5d93074506ff3109b3fe Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 19 Jun 2018 21:36:41 +0200 Subject: [PATCH 7/8] Clean up remaining yml test definitions --- test/integration/multi.yml | 2 +- test/integration/oss-upgrade.yml | 6 ++++-- test/integration/xpack.yml | 14 +++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 6fda7896..500bc9a1 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -29,7 +29,7 @@ post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - role: elasticsearch + - elasticsearch vars: es_enable_xpack: false es_scripts: true diff --git a/test/integration/oss-upgrade.yml b/test/integration/oss-upgrade.yml index f98715b8..e90ac823 100644 --- a/test/integration/oss-upgrade.yml +++ b/test/integration/oss-upgrade.yml @@ -4,8 +4,9 @@ post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - { role: elasticsearch, es_instance_name: "node1" } + - elasticsearch vars: + es_instance_name: "node1" es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade es_enable_xpack: false es_heap_size: "1g" @@ -15,7 +16,8 @@ post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - { role: elasticsearch, es_instance_name: "node1" } + - elasticsearch vars: + es_instance_name: "node1" es_enable_xpack: false es_heap_size: "1g" diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 1f143836..25d03af6 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -5,14 +5,14 @@ post_tasks: - include: elasticsearch/test/integration/debug.yml roles: - - role: elasticsearch - es_api_port: 9200 - es_instance_name: "node1" - es_config: - http.port: 9200 - transport.tcp.port: 9300 - discovery.zen.ping.unicast.hosts: "localhost:9300" + - elasticsearch vars: + es_api_port: 9200 + es_instance_name: "node1" + es_config: + http.port: 9200 + transport.tcp.port: 9300 + discovery.zen.ping.unicast.hosts: "localhost:9300" es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip" es_heap_size: 2g es_enable_xpack: true From 95c0bff835d09cd621aaf5d8769c51eb6bfe1a6b Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 19 Jun 2018 22:11:46 +0200 Subject: [PATCH 8/8] Fix path.data content test for when there are multiple paths --- test/integration/helpers/serverspec/shared_spec.rb | 2 +- test/integration/multi.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index 664595c3..d707cde4 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -164,7 +164,7 @@ else it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } end - its(:content) { should match "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" } + its(:content) { should match "path.data: #{vars['data_dirs'].join(',')}" } its(:content) { should match "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" } end end diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 500bc9a1..27fd3df8 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -8,7 +8,8 @@ - elasticsearch vars: es_instance_name: "master" - es_data_dirs: ["/opt/elasticsearch/master"] + es_data_dirs: + - "/opt/elasticsearch/master" es_config: discovery.zen.ping.unicast.hosts: "localhost:9300" http.port: 9200