From cf7b4c832dec8d0e1970134410bf58e439bbf6d9 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Thu, 2 May 2019 15:27:10 +0200 Subject: [PATCH 01/25] [test] update kitchen-ansible configuration - install ansible and jmespath dependencies using os repositories - fix role path --- .kitchen.yml | 48 +++++++---------------- test/integration/issue-test.yml | 4 +- test/integration/multi.yml | 8 ++-- test/integration/oss-to-xpack-upgrade.yml | 8 ++-- test/integration/oss-upgrade.yml | 8 ++-- test/integration/oss.yml | 4 +- test/integration/xpack-upgrade.yml | 8 ++-- test/integration/xpack.yml | 4 +- 8 files changed, 36 insertions(+), 56 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 0a741e3f..ef263a32 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -6,11 +6,9 @@ provisioner: name: ansible_playbook hosts: localhost roles_path: ../ - require_ansible_repo: false + require_ansible_repo: true require_ansible_omnibus: false require_ansible_source: false - require_pip: true - ansible_version: 2.4.3.0 http_proxy: <%= ENV['HTTP_PROXY'] %> https_proxy: <%= ENV['HTTPS_PROXY'] %> no_proxy: localhost,127.0.0.1 @@ -34,10 +32,8 @@ platforms: image: ubuntu:14.04 privileged: true provision_command: - - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible && add-apt-repository -y ppa:openjdk-r/ppa - - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip python-openssl build-essential libssl-dev libffi-dev python-dev locales openjdk-8-jre - - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - - pip install jmespath pyOpenSSL ndg-httpsclient cryptography==1.8.1 + - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible + - apt-get -y -q install ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -48,10 +44,7 @@ platforms: privileged: true provision_command: - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get install -y -q net-tools - - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip locales - - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - - pip install jmespath + - apt-get install -y -q ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -62,9 +55,7 @@ platforms: image: ubuntu:18.04 privileged: true provision_command: - - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip net-tools iproute2 - - pip install jmespath + - apt-get -y -q install ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -75,14 +66,10 @@ platforms: image: debian:8 privileged: true provision_command: - - echo "deb http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list - - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf - - apt-get update && apt-get -y install -t jessie-backports openjdk-8-jre-headless - - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget net-tools - - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config - - pip install jmespath setuptools --upgrade + - apt-get update && apt-get install gnupg2 + - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list + - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 + - apt-get update && apt-get -y install ansible python-jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers @@ -93,12 +80,10 @@ platforms: image: debian:9 privileged: true provision_command: - - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget systemd-sysv - - apt-get install -y -q net-tools - - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config - - pip install jmespath + - apt-get update && apt-get install gnupg2 + - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list + - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 + - apt-get update && apt-get -y install ansible python-jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers @@ -108,13 +93,8 @@ platforms: driver_config: image: centos:7 provision_command: - - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config - yum -y install epel-release - - yum -y install initscripts python-pip - - yum clean all - - pip install jmespath + - yum -y install ansible python2-jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers diff --git a/test/integration/issue-test.yml b/test/integration/issue-test.yml index 30a2cba2..7a1c8de2 100644 --- a/test/integration/issue-test.yml +++ b/test/integration/issue-test.yml @@ -6,9 +6,9 @@ - name: Simple Example hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_instance_name: "security_node" es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 27fd3df8..00459d47 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -3,9 +3,9 @@ - name: Elasticsearch Multi test - master on 9200 hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_instance_name: "master" es_data_dirs: @@ -28,9 +28,9 @@ - name: Elasticsearch Multi test - data on 9201 hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_enable_xpack: false es_scripts: true diff --git a/test/integration/oss-to-xpack-upgrade.yml b/test/integration/oss-to-xpack-upgrade.yml index c1ea5586..b60fe44a 100644 --- a/test/integration/oss-to-xpack-upgrade.yml +++ b/test/integration/oss-to-xpack-upgrade.yml @@ -2,9 +2,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_instance_name: "node1" es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade @@ -14,9 +14,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_instance_name: "node1" es_enable_xpack: true diff --git a/test/integration/oss-upgrade.yml b/test/integration/oss-upgrade.yml index e90ac823..181a90e1 100644 --- a/test/integration/oss-upgrade.yml +++ b/test/integration/oss-upgrade.yml @@ -2,9 +2,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_instance_name: "node1" es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade @@ -14,9 +14,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_instance_name: "node1" es_enable_xpack: false diff --git a/test/integration/oss.yml b/test/integration/oss.yml index 4bcf6b94..baf81ac9 100644 --- a/test/integration/oss.yml +++ b/test/integration/oss.yml @@ -2,9 +2,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_instance_name: "node1" es_enable_xpack: false diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index 5620489e..bb27f538 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -2,9 +2,9 @@ - name: Elasticsearch Xpack tests initial hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_instance_name: "node1" es_api_port: 9200 @@ -112,9 +112,9 @@ - name: Elasticsearch Xpack modify hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_api_port: 9200 es_instance_name: "node1" diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 25d03af6..bf606b73 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -3,9 +3,9 @@ - name: Elasticsearch Xpack tests - no security and manual download hosts: localhost post_tasks: - - include: elasticsearch/test/integration/debug.yml + - include: ansible-elasticsearch/test/integration/debug.yml roles: - - elasticsearch + - ansible-elasticsearch vars: es_api_port: 9200 es_instance_name: "node1" From 86377360e94b2e14e596225225533d8195fbacd5 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Thu, 2 May 2019 15:28:20 +0200 Subject: [PATCH 02/25] [doc] update supported platforms in README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c7b1ff81..9ae0613d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ Ansible role for 6.x/5.x Elasticsearch. Currently this works on Debian and RedH * Ubuntu 14.04 * Ubuntu 16.04 +* Ubuntu 18.04 * Debian 8 +* Debian 9 * CentOS 7 The latest Elasticsearch versions of 6.x and 5.x are actively tested. **Only Ansible versions > 2.4.3.0 are supported, as this is currently the only version tested.** From 0ae58a7c8d605e03b6cee047d6b1663c0884a724 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Thu, 2 May 2019 15:29:49 +0200 Subject: [PATCH 03/25] [7.x] add support for elasticsearch 7.x --- Makefile | 2 +- README.md | 12 ++++++------ defaults/main.yml | 3 +-- tasks/main.yml | 2 ++ test/integration/helpers/serverspec/multi_spec.rb | 4 ++-- test/integration/helpers/serverspec/shared_spec.rb | 2 +- test/integration/helpers/serverspec/xpack_spec.rb | 2 +- .../helpers/serverspec/xpack_upgrade_spec.rb | 2 +- test/matrix.yml | 1 + 9 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index d5e4d146..9b4e59d4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ default: build SHELL:=/bin/bash -eux -VERSION := 6.x +VERSION := 7.x PATTERN := xpack-ubuntu-1604 .PHONY: converge cerify test login destroy list diff --git a/README.md b/README.md index 9ae0613d..81e84184 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ [![Build Status](https://img.shields.io/jenkins/s/https/devops-ci.elastic.co/job/elastic+ansible-elasticsearch+master.svg)](https://devops-ci.elastic.co/job/elastic+ansible-elasticsearch+master/) [![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/) -**THIS ROLE IS FOR 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.** +**THIS ROLE IS FOR 7.x, 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.** -Ansible role for 6.x/5.x Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: +Ansible role for 7.x/6.x/5.x Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: * Ubuntu 14.04 * Ubuntu 16.04 @@ -13,7 +13,7 @@ Ansible role for 6.x/5.x Elasticsearch. Currently this works on Debian and RedH * Debian 9 * CentOS 7 -The latest Elasticsearch versions of 6.x and 5.x are actively tested. **Only Ansible versions > 2.4.3.0 are supported, as this is currently the only version tested.** +The latest Elasticsearch versions of 7.x, 6.x and 5.x are actively tested. **Only Ansible versions > 2.4.3.0 are supported, as this is currently the only version tested.** ##### Dependency This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine. @@ -23,7 +23,7 @@ This role uses the json_query filter which [requires jmespath](https://github.co Create your Ansible playbook with your own tasks, and include the role elasticsearch. You will have to have this repository accessible within the context of playbook. ```sh -ansible-galaxy install elastic.elasticsearch,6.6.0 +ansible-galaxy install elastic.elasticsearch,7.0.0 ``` Then create your playbook yaml adding the role elasticsearch. By default, the user is only required to specify a unique es_instance_name per role application. This should be unique per node. @@ -93,9 +93,9 @@ The `PATTERN` is a kitchen pattern which can match multiple suites. To run all t $ make converge PATTERN=centos-7 ``` -The default version is 6.x If you want to test 5.x you can override it with the `VERSION` variable to test 5.x +The default version is 7.x. If you want to test 6.x or 5.x you can override it with the `VERSION` variable, for example: ```sh -$ make converge VERSION=5.x PATTERN=oss-centos-7 +$ make converge VERSION=6.x PATTERN=oss-centos-7 ``` When you are finished testing you can clean up everything with diff --git a/defaults/main.yml b/defaults/main.yml index 8c4c868b..4aebc153 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,5 @@ --- -es_major_version: "6.x" -es_version: "6.6.0" +es_version: "7.0.0" es_use_snapshot_release: false es_enable_xpack: true es_package_name: "elasticsearch" diff --git a/tasks/main.yml b/tasks/main.yml index cb4735ef..c48e1cc1 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,4 +1,6 @@ --- +- set_fact: "es_major_version={{ es_version[0] }}.x" + - name: os-specific vars include_vars: "{{ansible_os_family}}.yml" tags: diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index e2a34aed..8b328841 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -16,7 +16,7 @@ it { should contain 'node.master: false' } it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should_not contain 'bootstrap.memory_lock: true' } - if vars['es_major_version'] == '6.x' + if vars['es_major_version'] == '6.x' or '7.x' it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } else it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } @@ -35,7 +35,7 @@ it { should contain 'node.master: true' } it { should contain 'node.name: localhost-master' } it { should contain 'bootstrap.memory_lock: true' } - if vars['es_major_version'] == '6.x' + if vars['es_major_version'] == '6.x' or '7.x' it { should_not contain 'path.conf: /etc/elasticsearch/master' } else it { should contain 'path.conf: /etc/elasticsearch/master' } diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index eca66821..a04241b6 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -159,7 +159,7 @@ describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should contain 'cluster.name: elasticsearch' } - if vars['es_major_version'] == '6.x' + if vars['es_major_version'] == '6.x' or '7.x' it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } else it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 2b962ed3..a189390e 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -4,7 +4,7 @@ describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should contain 'cluster.name: elasticsearch' } - if vars['es_major_version'] == '6.x' + if vars['es_major_version'] == '6.x' or '7.x' it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } else it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } diff --git a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb index 4afc6226..076ab7af 100644 --- a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb @@ -6,7 +6,7 @@ describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should contain 'cluster.name: elasticsearch' } - if vars['es_major_version'] == '6.x' + if vars['es_major_version'] == '6.x' or '7.x' it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } else it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } diff --git a/test/matrix.yml b/test/matrix.yml index 4b30ed16..9a29d625 100644 --- a/test/matrix.yml +++ b/test/matrix.yml @@ -1,4 +1,5 @@ VERSION: + - 7.x - 6.x - 5.x OS: From 09f1abd0f752347bcdd9db4c78b118edd9649e89 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Thu, 2 May 2019 15:43:53 +0200 Subject: [PATCH 04/25] Revert "[test] update kitchen-ansible configuration" This reverts commit cf7b4c832dec8d0e1970134410bf58e439bbf6d9. --- .kitchen.yml | 48 ++++++++++++++++------- test/integration/issue-test.yml | 4 +- test/integration/multi.yml | 8 ++-- test/integration/oss-to-xpack-upgrade.yml | 8 ++-- test/integration/oss-upgrade.yml | 8 ++-- test/integration/oss.yml | 4 +- test/integration/xpack-upgrade.yml | 8 ++-- test/integration/xpack.yml | 4 +- 8 files changed, 56 insertions(+), 36 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index ef263a32..0a741e3f 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -6,9 +6,11 @@ provisioner: name: ansible_playbook hosts: localhost roles_path: ../ - require_ansible_repo: true + require_ansible_repo: false require_ansible_omnibus: false require_ansible_source: false + require_pip: true + ansible_version: 2.4.3.0 http_proxy: <%= ENV['HTTP_PROXY'] %> https_proxy: <%= ENV['HTTPS_PROXY'] %> no_proxy: localhost,127.0.0.1 @@ -32,8 +34,10 @@ platforms: image: ubuntu:14.04 privileged: true provision_command: - - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get -y -q install ansible python-jmespath + - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible && add-apt-repository -y ppa:openjdk-r/ppa + - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip python-openssl build-essential libssl-dev libffi-dev python-dev locales openjdk-8-jre + - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 + - pip install jmespath pyOpenSSL ndg-httpsclient cryptography==1.8.1 use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -44,7 +48,10 @@ platforms: privileged: true provision_command: - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get install -y -q ansible python-jmespath + - apt-get install -y -q net-tools + - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip locales + - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 + - pip install jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -55,7 +62,9 @@ platforms: image: ubuntu:18.04 privileged: true provision_command: - - apt-get -y -q install ansible python-jmespath + - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible + - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip net-tools iproute2 + - pip install jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -66,10 +75,14 @@ platforms: image: debian:8 privileged: true provision_command: - - apt-get update && apt-get install gnupg2 - - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list - - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 - - apt-get update && apt-get -y install ansible python-jmespath + - echo "deb http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list + - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf + - apt-get update && apt-get -y install -t jessie-backports openjdk-8-jre-headless + - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget net-tools + - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config + - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config + - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config + - pip install jmespath setuptools --upgrade volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers @@ -80,10 +93,12 @@ platforms: image: debian:9 privileged: true provision_command: - - apt-get update && apt-get install gnupg2 - - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list - - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 - - apt-get update && apt-get -y install ansible python-jmespath + - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget systemd-sysv + - apt-get install -y -q net-tools + - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config + - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config + - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config + - pip install jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers @@ -93,8 +108,13 @@ platforms: driver_config: image: centos:7 provision_command: + - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config + - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config + - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config - yum -y install epel-release - - yum -y install ansible python2-jmespath + - yum -y install initscripts python-pip + - yum clean all + - pip install jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers diff --git a/test/integration/issue-test.yml b/test/integration/issue-test.yml index 7a1c8de2..30a2cba2 100644 --- a/test/integration/issue-test.yml +++ b/test/integration/issue-test.yml @@ -6,9 +6,9 @@ - name: Simple Example hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_instance_name: "security_node" es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 00459d47..27fd3df8 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -3,9 +3,9 @@ - name: Elasticsearch Multi test - master on 9200 hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_instance_name: "master" es_data_dirs: @@ -28,9 +28,9 @@ - name: Elasticsearch Multi test - data on 9201 hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_enable_xpack: false es_scripts: true diff --git a/test/integration/oss-to-xpack-upgrade.yml b/test/integration/oss-to-xpack-upgrade.yml index b60fe44a..c1ea5586 100644 --- a/test/integration/oss-to-xpack-upgrade.yml +++ b/test/integration/oss-to-xpack-upgrade.yml @@ -2,9 +2,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_instance_name: "node1" es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade @@ -14,9 +14,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_instance_name: "node1" es_enable_xpack: true diff --git a/test/integration/oss-upgrade.yml b/test/integration/oss-upgrade.yml index 181a90e1..e90ac823 100644 --- a/test/integration/oss-upgrade.yml +++ b/test/integration/oss-upgrade.yml @@ -2,9 +2,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_instance_name: "node1" es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade @@ -14,9 +14,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_instance_name: "node1" es_enable_xpack: false diff --git a/test/integration/oss.yml b/test/integration/oss.yml index baf81ac9..4bcf6b94 100644 --- a/test/integration/oss.yml +++ b/test/integration/oss.yml @@ -2,9 +2,9 @@ - name: Standard test for single node setup. Tests idempotence. hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_instance_name: "node1" es_enable_xpack: false diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index bb27f538..5620489e 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -2,9 +2,9 @@ - name: Elasticsearch Xpack tests initial hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_instance_name: "node1" es_api_port: 9200 @@ -112,9 +112,9 @@ - name: Elasticsearch Xpack modify hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_api_port: 9200 es_instance_name: "node1" diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index bf606b73..25d03af6 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -3,9 +3,9 @@ - name: Elasticsearch Xpack tests - no security and manual download hosts: localhost post_tasks: - - include: ansible-elasticsearch/test/integration/debug.yml + - include: elasticsearch/test/integration/debug.yml roles: - - ansible-elasticsearch + - elasticsearch vars: es_api_port: 9200 es_instance_name: "node1" From eb168a605bfe9035728307f70fc94960e410490f Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Thu, 2 May 2019 15:47:09 +0200 Subject: [PATCH 05/25] [test] update kitchen-ansible configuration Install ansible and jmespath dependencies using os repositories --- .kitchen.yml | 48 ++++++++++++++---------------------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 0a741e3f..44c6e2d7 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -6,11 +6,9 @@ provisioner: name: ansible_playbook hosts: localhost roles_path: ../ - require_ansible_repo: false + require_ansible_repo: true require_ansible_omnibus: false require_ansible_source: false - require_pip: true - ansible_version: 2.4.3.0 http_proxy: <%= ENV['HTTP_PROXY'] %> https_proxy: <%= ENV['HTTPS_PROXY'] %> no_proxy: localhost,127.0.0.1 @@ -34,10 +32,8 @@ platforms: image: ubuntu:14.04 privileged: true provision_command: - - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible && add-apt-repository -y ppa:openjdk-r/ppa - - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip python-openssl build-essential libssl-dev libffi-dev python-dev locales openjdk-8-jre - - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - - pip install jmespath pyOpenSSL ndg-httpsclient cryptography==1.8.1 + - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible + - apt-get -y -q install ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -48,10 +44,7 @@ platforms: privileged: true provision_command: - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get install -y -q net-tools - - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip locales - - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - - pip install jmespath + - apt-get install -y -q ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -62,9 +55,7 @@ platforms: image: ubuntu:18.04 privileged: true provision_command: - - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip net-tools iproute2 - - pip install jmespath + - apt-get -y -q install ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -75,14 +66,10 @@ platforms: image: debian:8 privileged: true provision_command: - - echo "deb http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list - - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf - - apt-get update && apt-get -y install -t jessie-backports openjdk-8-jre-headless - - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget net-tools - - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config - - pip install jmespath setuptools --upgrade + - apt-get update && apt-get install gnupg2 + - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list + - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 + - apt-get update && apt-get -y install ansible python-jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers @@ -93,12 +80,10 @@ platforms: image: debian:9 privileged: true provision_command: - - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget systemd-sysv - - apt-get install -y -q net-tools - - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config - - pip install jmespath + - apt-get update && apt-get install gnupg2 + - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list + - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 + - apt-get update && apt-get -y install ansible python-jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers @@ -108,13 +93,8 @@ platforms: driver_config: image: centos:7 provision_command: - - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config - - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config - yum -y install epel-release - - yum -y install initscripts python-pip - - yum clean all - - pip install jmespath + - yum -y install ansible python2-jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers From 83f78bfd5efbb6d680861758c8c3435634293ba2 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 11:59:52 +0200 Subject: [PATCH 06/25] [test] replace geoip plugin in tests as this one is now embeded in elasticsearch since 6.7.0 see https://www.elastic.co/guide/en/elasticsearch/plugins/6.7/ingest-geoip.html --- README.md | 12 ++++++------ test/integration/issue-test.yml | 2 +- test/integration/multi.yml | 4 ++-- test/integration/oss.yml | 2 +- test/integration/xpack-upgrade.yml | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 81e84184..327106e7 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ A more complex example: es_plugins_reinstall: false es_api_port: 9201 es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment proxy_host: proxy.example.com proxy_port: 8080 ``` @@ -217,7 +217,7 @@ An example of a two server deployment is shown below. The first server holds th es_version_lock: false ansible_user: ansible es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment - hosts: data_nodes roles: @@ -240,7 +240,7 @@ An example of a two server deployment is shown below. The first server holds th ansible_user: ansible es_api_port: 9200 es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment - hosts: data_nodes roles: @@ -262,7 +262,7 @@ An example of a two server deployment is shown below. The first server holds th es_api_port: 9201 ansible_user: ansible es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment ``` Parameters can additionally be assigned to hosts using the inventory file if desired. @@ -398,7 +398,7 @@ In addition to es_config, the following parameters allow the customization of th * ```es_plugins``` an array of plugin definitions e.g.: ```yaml es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment ``` * ```es_path_repo``` Sets the whitelist for allowing local back-up repositories * ```es_action_auto_create_index ``` Sets the value for auto index creation, use the syntax below for specifying indexes (else true/false): @@ -461,7 +461,7 @@ To define proxy only for a particular plugin during its installation: ```yaml es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment proxy_host: proxy.example.com proxy_port: 8080 ``` diff --git a/test/integration/issue-test.yml b/test/integration/issue-test.yml index 30a2cba2..06d51cc9 100644 --- a/test/integration/issue-test.yml +++ b/test/integration/issue-test.yml @@ -21,7 +21,7 @@ es_heap_size: "1g" es_enable_xpack: true es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment es_xpack_features: - security - alerting diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 27fd3df8..5cc3a17b 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -23,7 +23,7 @@ es_heap_size: "1g" es_api_port: 9200 es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment - name: Elasticsearch Multi test - data on 9201 hosts: localhost @@ -38,7 +38,7 @@ es_heap_size: "1g" es_api_port: 9201 es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment es_instance_name: "node1" es_data_dirs: - "/opt/elasticsearch/data-1" diff --git a/test/integration/oss.yml b/test/integration/oss.yml index 4bcf6b94..4dfdee20 100644 --- a/test/integration/oss.yml +++ b/test/integration/oss.yml @@ -10,6 +10,6 @@ es_enable_xpack: false es_heap_size: "1g" es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment #Do not add tests here. This test is run twice and confirms idempotency. diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index 5620489e..599c641e 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -22,7 +22,7 @@ es_enable_xpack: true es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" es_plugins: - - plugin: ingest-geoip + - plugin: ingest-attachment es_xpack_features: - security - alerting From 3c218eb9eced1a9c8c1d7a631a850ed127ceca54 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 15:17:22 +0200 Subject: [PATCH 07/25] [test] force again UTF-8 locale-gen on ubuntu --- .kitchen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 44c6e2d7..a0ea2b78 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -32,6 +32,7 @@ platforms: image: ubuntu:14.04 privileged: true provision_command: + - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - apt-get -y -q install ansible python-jmespath use_sudo: false @@ -43,8 +44,7 @@ platforms: image: ubuntu:16.04 privileged: true provision_command: - - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get install -y -q ansible python-jmespath + - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json From 4ce8a895fef3471200e532cc3b1daefe99909204 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 15:20:10 +0200 Subject: [PATCH 08/25] [test] ensure every apt command is forced and quiet --- .kitchen.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index a0ea2b78..7644f3f4 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -33,8 +33,8 @@ platforms: privileged: true provision_command: - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get -y -q install ansible python-jmespath + - apt-get update -q && apt-get install -y -q software-properties-common && add-apt-repository -y ppa:ansible/ansible + - apt-get update -q && apt-get -y -q install ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -45,6 +45,8 @@ platforms: privileged: true provision_command: - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 + - apt-get update -q && apt-get install -y -q software-properties-common && add-apt-repository -y ppa:ansible/ansible + - apt-get update -q && apt-get install -y -q ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -55,7 +57,7 @@ platforms: image: ubuntu:18.04 privileged: true provision_command: - - apt-get -y -q install ansible python-jmespath + - apt-get install -y -q ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -66,10 +68,10 @@ platforms: image: debian:8 privileged: true provision_command: - - apt-get update && apt-get install gnupg2 + - apt-get update -q && apt-get install -y -q gnupg2 - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 - - apt-get update && apt-get -y install ansible python-jmespath + - apt-get update -q && apt-get install -y -q ansible python-jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers @@ -80,10 +82,10 @@ platforms: image: debian:9 privileged: true provision_command: - - apt-get update && apt-get install gnupg2 + - apt-get update -q && apt-get install -y -q gnupg2 - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 - - apt-get update && apt-get -y install ansible python-jmespath + - apt-get update -q && apt-get install -y -q ansible python-jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers From 389716b16f3cf86dbb21ab00b94ea57651574762 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 15:22:28 +0200 Subject: [PATCH 09/25] [es] remove 5.x support --- .kitchen.yml | 4 +-- README.md | 11 ++++---- .../helpers/serverspec/shared_spec.rb | 26 ------------------- .../helpers/serverspec/xpack_upgrade_spec.rb | 9 ------- test/matrix.yml | 1 - 5 files changed, 7 insertions(+), 44 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 7644f3f4..9085b3e4 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -18,8 +18,8 @@ provisioner: attributes: extra_vars: es_major_version: "<%= ENV['VERSION'] %>" - <% if ENV['VERSION'] == '5.x' %> - es_version: '5.6.11' + <% if ENV['VERSION'] == '6.x' %> + es_version: '6.7.1' <% end %> <% end %> diff --git a/README.md b/README.md index 327106e7..130908a5 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ [![Build Status](https://img.shields.io/jenkins/s/https/devops-ci.elastic.co/job/elastic+ansible-elasticsearch+master.svg)](https://devops-ci.elastic.co/job/elastic+ansible-elasticsearch+master/) [![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/) -**THIS ROLE IS FOR 7.x, 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.** +**THIS ROLE IS FOR 7.x & 6.x** -Ansible role for 7.x/6.x/5.x Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: +Ansible role for 7.x/6.x Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: * Ubuntu 14.04 * Ubuntu 16.04 @@ -13,7 +13,7 @@ Ansible role for 7.x/6.x/5.x Elasticsearch. Currently this works on Debian and * Debian 9 * CentOS 7 -The latest Elasticsearch versions of 7.x, 6.x and 5.x are actively tested. **Only Ansible versions > 2.4.3.0 are supported, as this is currently the only version tested.** +The latest Elasticsearch versions of 7.x & 6.x are actively tested. **Only Ansible versions > 2.4.3.0 are supported, as this is currently the only version tested.** ##### Dependency This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine. @@ -93,7 +93,7 @@ The `PATTERN` is a kitchen pattern which can match multiple suites. To run all t $ make converge PATTERN=centos-7 ``` -The default version is 7.x. If you want to test 6.x or 5.x you can override it with the `VERSION` variable, for example: +The default version is 7.x. If you want to test 6.x you can override it with the `VERSION` variable, for example: ```sh $ make converge VERSION=6.x PATTERN=oss-centos-7 ``` @@ -387,7 +387,6 @@ These can either be set to a user declared in the file based realm, with admin p In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include: * ```es_enable_xpack``` Default `true`. Setting this to `false` will install the oss release of elasticsearch -* ```es_major_version``` Should be consistent with es_version. For versions >= 5.0 and < 6.0 this must be "5.x". For versions >= 6.0 this must be "6.x". * ```es_version``` (e.g. "6.3.0"). * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200** @@ -473,7 +472,7 @@ To define proxy only for a particular plugin during its installation: * The role assumes the user/group exists on the server. The elasticsearch packages create the default elasticsearch user. If this needs to be changed, ensure the user exists. * The playbook relies on the inventory_name of each host to ensure its directories are unique * Changing an instance_name for a role application will result in the installation of a new component. The previous component will remain. -* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test the latest version of 6.x and 5.x on all supported platforms. +* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test the latest version of 7.x and 6.x on all supported platforms. * The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and Elasticsearch restarted where required. * Systemd is used for Ubuntu versions >= 15, Debian >=8, Centos >=7. All other versions use init for service scripts. * In order to run x-pack tests a license file with security enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index a04241b6..36c00bbf 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -43,32 +43,6 @@ expect(values['enabled'] = enabled) end end - # X-Pack is no longer installed as a plugin in elasticsearch - if vars['es_major_version'] == '5.x' - describe file('/usr/share/elasticsearch/plugins/x-pack') do - it { should be_directory } - it { should be_owned_by vars['es_user'] } - end - describe file("/etc/elasticsearch/#{vars['es_instance_name']}/x-pack") do - it { should be_directory } - it { should be_owned_by vars['es_user'] } - end - describe 'x-pack-core plugin' do - it 'should be installed with the correct version' do - plugins = curl_json("#{es_api_url}/_nodes/plugins", username=username, password=password) - node, data = plugins['nodes'].first - version = 'plugin not found' - name = 'x-pack' - - data['plugins'].each do |plugin| - if plugin['name'] == name - version = plugin['version'] - end - end - expect(version).to eql(vars['es_version']) - end - end - end end end describe user(vars['es_user']) do diff --git a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb index 076ab7af..4318879c 100644 --- a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb @@ -91,13 +91,4 @@ expect(command.exit_status).to eq(0) end end - - if vars['es_major_version'] == '5.x' # kibana default password has been removed in 6.x - describe 'kibana access check' do - it 'should be reported as version '+vars['es_version'] do - result = curl_json('http://localhost:9200/', username='kibana', password='changeme') - expect(result['version']['number']).to eq(vars['es_version']) - end - end - end end diff --git a/test/matrix.yml b/test/matrix.yml index 9a29d625..e27a5ac8 100644 --- a/test/matrix.yml +++ b/test/matrix.yml @@ -1,7 +1,6 @@ VERSION: - 7.x - 6.x - - 5.x OS: - ubuntu-1404 - ubuntu-1604 From f0b2d8dff9a1b96547bf0bd4c06c49d473730856 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 18:34:12 +0200 Subject: [PATCH 10/25] [test] fix packages requirements --- .kitchen.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 9085b3e4..75d60244 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -32,9 +32,9 @@ platforms: image: ubuntu:14.04 privileged: true provision_command: + - apt-get update -q && apt-get install -y -q software-properties-common && add-apt-repository -y ppa:ansible/ansible && add-apt-repository -y ppa:openjdk-r/ppa + - apt-get update -q && apt-get -y -q install ansible openjdk-8-jre python-jmespath - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - - apt-get update -q && apt-get install -y -q software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get update -q && apt-get -y -q install ansible python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -44,9 +44,9 @@ platforms: image: ubuntu:16.04 privileged: true provision_command: - - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 - - apt-get update -q && apt-get install -y -q software-properties-common && add-apt-repository -y ppa:ansible/ansible + - apt-get update -q && apt-get install -y -q iproute locales software-properties-common && add-apt-repository -y ppa:ansible/ansible - apt-get update -q && apt-get install -y -q ansible python-jmespath + - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -57,7 +57,7 @@ platforms: image: ubuntu:18.04 privileged: true provision_command: - - apt-get install -y -q ansible python-jmespath + - apt-get install -y -q ansible iproute2 python-jmespath use_sudo: false volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json @@ -68,10 +68,13 @@ platforms: image: debian:8 privileged: true provision_command: - - apt-get update -q && apt-get install -y -q gnupg2 + - apt-get update -q && apt-get install -y -q gnupg2 python-jmespath + - echo "deb http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list + - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf + - apt-get update && apt-get -y install -t jessie-backports openjdk-8-jre-headless - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 - - apt-get update -q && apt-get install -y -q ansible python-jmespath + - apt-get update -q && apt-get install -y -q ansible volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers @@ -82,10 +85,10 @@ platforms: image: debian:9 privileged: true provision_command: - - apt-get update -q && apt-get install -y -q gnupg2 + - apt-get update -q && apt-get install -y -q gnupg2 python-jmespath systemd-sysv - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 - - apt-get update -q && apt-get install -y -q ansible python-jmespath + - apt-get update -q && apt-get install -y -q ansible volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers @@ -96,7 +99,7 @@ platforms: image: centos:7 provision_command: - yum -y install epel-release - - yum -y install ansible python2-jmespath + - yum -y install ansible iproute python2-jmespath volume: - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers From d087543fb5291657bfa4bdda31695a851e721587 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 18:45:44 +0200 Subject: [PATCH 11/25] [es] update version and remove last 5.x mentions --- .kitchen.yml | 2 +- README.md | 4 ++-- defaults/main.yml | 2 +- templates/jvm.options.j2 | 8 -------- test/integration/oss-to-xpack-upgrade.yml | 2 +- test/integration/oss-upgrade.yml | 2 +- test/integration/xpack-upgrade.yml | 2 +- 7 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 75d60244..1bf6f487 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -19,7 +19,7 @@ provisioner: extra_vars: es_major_version: "<%= ENV['VERSION'] %>" <% if ENV['VERSION'] == '6.x' %> - es_version: '6.7.1' + es_version: '6.7.2' <% end %> <% end %> diff --git a/README.md b/README.md index 130908a5..21256c89 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ This role uses the json_query filter which [requires jmespath](https://github.co Create your Ansible playbook with your own tasks, and include the role elasticsearch. You will have to have this repository accessible within the context of playbook. ```sh -ansible-galaxy install elastic.elasticsearch,7.0.0 +ansible-galaxy install elastic.elasticsearch,7.0.1 ``` Then create your playbook yaml adding the role elasticsearch. By default, the user is only required to specify a unique es_instance_name per role application. This should be unique per node. @@ -387,7 +387,7 @@ These can either be set to a user declared in the file based realm, with admin p In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include: * ```es_enable_xpack``` Default `true`. Setting this to `false` will install the oss release of elasticsearch -* ```es_version``` (e.g. "6.3.0"). +* ```es_version``` (e.g. "7.0.0"). * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200** * ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin. diff --git a/defaults/main.yml b/defaults/main.yml index 4aebc153..99d36e41 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,5 +1,5 @@ --- -es_version: "7.0.0" +es_version: "7.0.1" es_use_snapshot_release: false es_enable_xpack: true es_package_name: "elasticsearch" diff --git a/templates/jvm.options.j2 b/templates/jvm.options.j2 index ad30851a..98327824 100644 --- a/templates/jvm.options.j2 +++ b/templates/jvm.options.j2 @@ -103,14 +103,6 @@ #-XX:+UseGCLogFileRotation #-XX:NumberOfGCLogFiles=32 #-XX:GCLogFileSize=128M - -# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. -# If documents were already indexed with unquoted fields in a previous version -# of Elasticsearch, some operations may throw errors. -# -# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided -# only for migration purposes. -#-Delasticsearch.json.allow_unquoted_field_names=true {% if es_jvm_custom_parameters !='' %} {% for item in es_jvm_custom_parameters %} {{ item }} diff --git a/test/integration/oss-to-xpack-upgrade.yml b/test/integration/oss-to-xpack-upgrade.yml index c1ea5586..71e37985 100644 --- a/test/integration/oss-to-xpack-upgrade.yml +++ b/test/integration/oss-to-xpack-upgrade.yml @@ -7,7 +7,7 @@ - elasticsearch vars: es_instance_name: "node1" - es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade + es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade es_enable_xpack: false es_heap_size: "1g" diff --git a/test/integration/oss-upgrade.yml b/test/integration/oss-upgrade.yml index e90ac823..62c20894 100644 --- a/test/integration/oss-upgrade.yml +++ b/test/integration/oss-upgrade.yml @@ -7,7 +7,7 @@ - elasticsearch vars: es_instance_name: "node1" - es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade + es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade es_enable_xpack: false es_heap_size: "1g" diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index 599c641e..b61ca148 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -18,7 +18,7 @@ xpack.security.authc.realms.native1.order: 1 es_heap_size: "1g" es_templates: true - es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade + es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade es_enable_xpack: true es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" es_plugins: From 643706f12cbd5fbcdb067ed08cf8a29df022cbf6 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 19:33:35 +0200 Subject: [PATCH 12/25] [test] remove deprecated conditions (needed for 5.x) --- test/integration/helpers/serverspec/multi_spec.rb | 12 ++---------- test/integration/helpers/serverspec/shared_spec.rb | 6 +----- test/integration/helpers/serverspec/xpack_spec.rb | 6 +----- .../helpers/serverspec/xpack_upgrade_spec.rb | 6 +----- 4 files changed, 5 insertions(+), 25 deletions(-) diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index 8b328841..2567e650 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -16,11 +16,7 @@ it { should contain 'node.master: false' } it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should_not contain 'bootstrap.memory_lock: true' } - if vars['es_major_version'] == '6.x' or '7.x' - it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } - else - it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } - end + it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } it { should contain "path.data: /opt/elasticsearch/data-1/localhost-#{vars['es_instance_name']},/opt/elasticsearch/data-2/localhost-#{vars['es_instance_name']}" } it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" } end @@ -35,11 +31,7 @@ it { should contain 'node.master: true' } it { should contain 'node.name: localhost-master' } it { should contain 'bootstrap.memory_lock: true' } - if vars['es_major_version'] == '6.x' or '7.x' - it { should_not contain 'path.conf: /etc/elasticsearch/master' } - else - it { should contain 'path.conf: /etc/elasticsearch/master' } - end + it { should_not contain 'path.conf: /etc/elasticsearch/master' } it { should contain 'path.data: /opt/elasticsearch/master/localhost-master' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-master' } end diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index 36c00bbf..325d7cf8 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -133,11 +133,7 @@ describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should contain 'cluster.name: elasticsearch' } - if vars['es_major_version'] == '6.x' or '7.x' - it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } - else - it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } - end + it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } its(:content) { should match "path.data: #{vars['data_dirs'].join(',')}" } its(:content) { should match "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" } end diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index a189390e..77b0a0b5 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -4,11 +4,7 @@ describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should contain 'cluster.name: elasticsearch' } - if vars['es_major_version'] == '6.x' or '7.x' - it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } - else - it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } - end + it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } it { should contain "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" } it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" } it { should contain 'xpack.security.enabled: false' } diff --git a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb index 4318879c..2302f32f 100644 --- a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb @@ -6,11 +6,7 @@ describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should contain "node.name: localhost-#{vars['es_instance_name']}" } it { should contain 'cluster.name: elasticsearch' } - if vars['es_major_version'] == '6.x' or '7.x' - it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } - else - it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } - end + it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" } it { should contain "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" } it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" } end From 2110e0a64f27ff2d10338a1993f31b09d8138388 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 19:37:56 +0200 Subject: [PATCH 13/25] [es] remove deprecated configuration in ES 7.x, discovery.zen.ping.unicast.hosts is replaced by discovery.seed_hosts and transport.tcp.port is replaced by transport.port. As this parameters are set with default values, we can completely remove this parameters from test configurations to avoid managing differents tests with differents values for 6.x and 7.x --- test/integration/multi.yml | 4 ---- test/integration/xpack-upgrade.yml | 4 ---- test/integration/xpack.yml | 2 -- 3 files changed, 10 deletions(-) diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 5cc3a17b..02950e46 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -11,9 +11,7 @@ es_data_dirs: - "/opt/elasticsearch/master" es_config: - discovery.zen.ping.unicast.hosts: "localhost:9300" http.port: 9200 - transport.tcp.port: 9300 node.data: false node.master: true bootstrap.memory_lock: true @@ -44,8 +42,6 @@ - "/opt/elasticsearch/data-1" - "/opt/elasticsearch/data-2" es_config: - discovery.zen.ping.unicast.hosts: "localhost:9300" http.port: 9201 - transport.tcp.port: 9301 node.data: true node.master: false diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index b61ca148..6202931c 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -10,8 +10,6 @@ es_api_port: 9200 es_config: http.port: 9200 - transport.tcp.port: 9300 - discovery.zen.ping.unicast.hosts: "localhost:9300" xpack.security.authc.realms.file1.type: "file" xpack.security.authc.realms.file1.order: 0 xpack.security.authc.realms.native1.type: "native" @@ -120,8 +118,6 @@ es_instance_name: "node1" es_config: http.port: 9200 - transport.tcp.port: 9300 - discovery.zen.ping.unicast.hosts: "localhost:9300" xpack.security.enabled: True xpack.security.authc.realms.file1.type: "file" xpack.security.authc.realms.file1.order: 0 diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 25d03af6..5be3d035 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -11,8 +11,6 @@ es_instance_name: "node1" es_config: http.port: 9200 - transport.tcp.port: 9300 - discovery.zen.ping.unicast.hosts: "localhost:9300" es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip" es_heap_size: 2g es_enable_xpack: true From 175abeabc9877072aa57a4251dc080538cb86101 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 20:05:58 +0200 Subject: [PATCH 14/25] [es] update test and docs for deprecated parameters --- README.md | 24 +++++++++---------- tasks/elasticsearch-parameters.yml | 8 +++---- .../helpers/serverspec/multi_spec.rb | 2 -- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 21256c89..703ed5b2 100644 --- a/README.md +++ b/README.md @@ -125,9 +125,9 @@ The following illustrates applying configuration parameters to an Elasticsearch es_config: node.name: "node1" cluster.name: "custom-cluster" - discovery.zen.ping.unicast.hosts: "localhost:9301" + discovery.seed_hosts: "localhost:9301" http.port: 9201 - transport.tcp.port: 9301 + transport.port: 9301 node.data: false node.master: true bootstrap.memory_lock: true @@ -141,8 +141,8 @@ The following illustrates applying configuration parameters to an Elasticsearch Whilst the role installs Elasticsearch with the default configuration parameters, the following should be configured to ensure a cluster successfully forms: * ```es_config['http.port']``` - the http port for the node -* ```es_config['transport.tcp.port']``` - the transport port for the node -* ```es_config['discovery.zen.ping.unicast.hosts']``` - the unicast discovery list, in the comma separated format ```":,:"``` (typically the clusters dedicated masters) +* ```es_config['transport.port']``` - the transport port for the node +* ```es_config['discovery.seed_hosts']``` - the unicast discovery list, in the comma separated format ```":,:"``` (typically the clusters dedicated masters) * ```es_config['network.host']``` - sets both network.bind_host and network.publish_host to the same host value. The network.bind_host setting allows to control the host different network components will bind on. The network.publish_host setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. @@ -165,9 +165,9 @@ A more complex example: es_config: node.name: "node1" cluster.name: "custom-cluster" - discovery.zen.ping.unicast.hosts: "localhost:9301" + discovery.seed_hosts: "localhost:9301" http.port: 9201 - transport.tcp.port: 9301 + transport.port: 9301 node.data: false node.master: true bootstrap.memory_lock: true @@ -206,9 +206,9 @@ An example of a two server deployment is shown below. The first server holds th es_heap_size: "1g" es_config: cluster.name: "test-cluster" - discovery.zen.ping.unicast.hosts: "elastic02:9300" + discovery.seed_hosts: "elastic02:9300" http.port: 9200 - transport.tcp.port: 9300 + transport.port: 9300 node.data: false node.master: true bootstrap.memory_lock: false @@ -228,9 +228,9 @@ An example of a two server deployment is shown below. The first server holds th - "/opt/elasticsearch" es_config: cluster.name: "test-cluster" - discovery.zen.ping.unicast.hosts: "elastic02:9300" + discovery.seed_hosts: "elastic02:9300" http.port: 9200 - transport.tcp.port: 9300 + transport.port: 9300 node.data: true node.master: false bootstrap.memory_lock: false @@ -249,9 +249,9 @@ An example of a two server deployment is shown below. The first server holds th es_instance_name: "node2" es_api_port: 9201 es_config: - discovery.zen.ping.unicast.hosts: "elastic02:9300" + discovery.seed_hosts: "elastic02:9300" http.port: 9201 - transport.tcp.port: 9301 + transport.port: 9301 node.data: true node.master: false bootstrap.memory_lock: false diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index df1ba0e5..53df8ca9 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -13,12 +13,12 @@ when: es_config['http.port'] is not defined - name: debug message - debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port'" - when: es_config['transport.tcp.port'] is not defined + debug: msg="WARNING - It is recommended you specify the parameter 'transport.port'" + when: es_config['transport.port'] is not defined - name: debug message - debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts'" - when: es_config['discovery.zen.ping.unicast.hosts'] is not defined + debug: msg="WARNING - It is recommended you specify the parameter 'discovery.seed_hosts'" + when: es_config['discovery.seed_hosts'] is not defined #If the user attempts to lock memory they must specify a heap size - name: fail when heap size is not specified when using memory lock diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index 2567e650..c54b4b48 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -11,7 +11,6 @@ describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should be_file } it { should contain 'http.port: 9201' } - it { should contain 'transport.tcp.port: 9301' } it { should contain 'node.data: true' } it { should contain 'node.master: false' } it { should contain "node.name: localhost-#{vars['es_instance_name']}" } @@ -26,7 +25,6 @@ describe file('/etc/elasticsearch/master/elasticsearch.yml') do it { should be_file } it { should contain 'http.port: 9200' } - it { should contain 'transport.tcp.port: 9300' } it { should contain 'node.data: false' } it { should contain 'node.master: true' } it { should contain 'node.name: localhost-master' } From cc73e0351dbaba819ba0d64c525862e61a58ae5d Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 3 May 2019 21:43:05 +0200 Subject: [PATCH 15/25] [es] handle different templates structure for es 7.x and 7.x --- defaults/main.yml | 2 +- files/{templates => templates-6.x}/basic.json | 0 files/templates-7.x/basic.json | 9 +++++++++ test/integration/helpers/serverspec/shared_spec.rb | 6 +++++- 4 files changed, 15 insertions(+), 2 deletions(-) rename files/{templates => templates-6.x}/basic.json (100%) create mode 100644 files/templates-7.x/basic.json diff --git a/defaults/main.yml b/defaults/main.yml index 99d36e41..ec69115c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -5,7 +5,7 @@ es_enable_xpack: true es_package_name: "elasticsearch" es_version_lock: false es_use_repository: true -es_templates_fileglob: "files/templates/*.json" +es_templates_fileglob: "files/templates-{{ es_major_version }}/*.json" es_repo_base: "https://artifacts.elastic.co" es_apt_key: "{{ es_repo_base }}/GPG-KEY-elasticsearch" es_apt_url: "deb {{ es_repo_base }}/packages/{{ es_repo_name }}/apt stable main" diff --git a/files/templates/basic.json b/files/templates-6.x/basic.json similarity index 100% rename from files/templates/basic.json rename to files/templates-6.x/basic.json diff --git a/files/templates-7.x/basic.json b/files/templates-7.x/basic.json new file mode 100644 index 00000000..31796dae --- /dev/null +++ b/files/templates-7.x/basic.json @@ -0,0 +1,9 @@ +{ + "index_patterns" : "te*", + "settings" : { + "number_of_shards" : 1 + }, + "mappings" : { + "_source" : { "enabled" : false } + } +} \ No newline at end of file diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index 325d7cf8..0402dca5 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -82,7 +82,11 @@ template = curl_json("#{es_api_url}/_template/basic", username=username, password=password) expect(template.key?('basic')) expect(template['basic']['settings']['index']['number_of_shards']).to eq("1") - expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false) + if vars['es_major_version'] == '7.x' + expect(template['basic']['mappings']['_source']['enabled']).to eq(false) + else + expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false) + end end end end From 8b856e423d643ce29d36ce918fc736a58fa1a1fd Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Mon, 6 May 2019 15:09:30 +0200 Subject: [PATCH 16/25] [es] allow forcing es_major_version for upgrade tests --- tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tasks/main.yml b/tasks/main.yml index c48e1cc1..c37badba 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,5 +1,10 @@ --- + - set_fact: "es_major_version={{ es_version[0] }}.x" + when: + - not es_major_version + tags: + - always - name: os-specific vars include_vars: "{{ansible_os_family}}.yml" From 2954410e85a1ebe02a29a6192849f3c72c561c22 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Mon, 6 May 2019 15:10:36 +0200 Subject: [PATCH 17/25] [es] remove usage of deprecated security realms settings cf. https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#include-realm-type-in-setting --- .../helpers/serverspec/xpack_upgrade_spec.rb | 6 ++---- test/integration/issue-test.yml | 6 ++---- test/integration/xpack-upgrade.yml | 13 +++++-------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb index 2302f32f..09289c9a 100644 --- a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb @@ -32,10 +32,8 @@ end describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do - it { should contain 'security.authc.realms.file1.order: 0' } - it { should contain 'security.authc.realms.file1.type: file' } - it { should contain 'security.authc.realms.native1.order: 1' } - it { should contain 'security.authc.realms.native1.type: native' } + it { should contain 'security.authc.realms.file.file1.order: 0' } + it { should contain 'security.authc.realms.native.native1.order: 1' } end #Test contents of role_mapping.yml diff --git a/test/integration/issue-test.yml b/test/integration/issue-test.yml index 06d51cc9..aa325347 100644 --- a/test/integration/issue-test.yml +++ b/test/integration/issue-test.yml @@ -14,10 +14,8 @@ es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" es_config: xpack.security.enabled: True - xpack.security.authc.realms.file1.type: "file" - xpack.security.authc.realms.file1.order: 1 - xpack.security.authc.realms.native1.type: "native" - xpack.security.authc.realms.native1.order: 0 + xpack.security.authc.realms.file.file1.order: 1 + xpack.security.authc.realms.native.native1.type: "native" es_heap_size: "1g" es_enable_xpack: true es_plugins: diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index 6202931c..3d19c8cd 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -10,12 +10,11 @@ es_api_port: 9200 es_config: http.port: 9200 - xpack.security.authc.realms.file1.type: "file" - xpack.security.authc.realms.file1.order: 0 - xpack.security.authc.realms.native1.type: "native" - xpack.security.authc.realms.native1.order: 1 + xpack.security.authc.realms.file.file1.order: 0 + xpack.security.authc.realms.native.native1.order: 1 es_heap_size: "1g" es_templates: true + es_major_version: "7.x" es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade es_enable_xpack: true es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" @@ -119,10 +118,8 @@ es_config: http.port: 9200 xpack.security.enabled: True - xpack.security.authc.realms.file1.type: "file" - xpack.security.authc.realms.file1.order: 0 - xpack.security.authc.realms.native1.type: "native" - xpack.security.authc.realms.native1.order: 1 + xpack.security.authc.realms.file.file1.order: 0 + xpack.security.authc.realms.native.native1.order: 1 es_heap_size: "1g" es_templates: true es_enable_xpack: true From f9b2071ff092243b99a5939bf49618abf9987842 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Mon, 6 May 2019 19:10:29 +0200 Subject: [PATCH 18/25] [test] use different for security realms in test es 6.x and test es 7.x --- test/integration/xpack-upgrade.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index 3d19c8cd..000fdb5d 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -8,10 +8,18 @@ vars: es_instance_name: "node1" es_api_port: 9200 - es_config: + es_config_6x: http.port: 9200 + xpack.security.authc.realms.file1.order: 0 + xpack.security.authc.realms.file1.type: file + xpack.security.authc.realms.native1.order: 1 + xpack.security.authc.realms.native1.type: native + es_config_7x: + http.port: 9200 + xpack.security.enabled: True xpack.security.authc.realms.file.file1.order: 0 xpack.security.authc.realms.native.native1.order: 1 + es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" es_heap_size: "1g" es_templates: true es_major_version: "7.x" @@ -115,11 +123,18 @@ vars: es_api_port: 9200 es_instance_name: "node1" - es_config: + es_config_6x: + http.port: 9200 + xpack.security.authc.realms.file1.order: 0 + xpack.security.authc.realms.file1.type: file + xpack.security.authc.realms.native1.order: 1 + xpack.security.authc.realms.native1.type: native + es_config_7x: http.port: 9200 xpack.security.enabled: True xpack.security.authc.realms.file.file1.order: 0 xpack.security.authc.realms.native.native1.order: 1 + es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" es_heap_size: "1g" es_templates: true es_enable_xpack: true From 29def4eab0b5ebc30a7be4b9fbb462625b58e3f3 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Mon, 6 May 2019 20:51:46 +0200 Subject: [PATCH 19/25] [test] fix security setting test --- test/integration/helpers/serverspec/xpack_upgrade_spec.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb index 09289c9a..1f3f59d1 100644 --- a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb @@ -32,8 +32,14 @@ end describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do + if vars['es_major_version'] == '7.x' it { should contain 'security.authc.realms.file.file1.order: 0' } it { should contain 'security.authc.realms.native.native1.order: 1' } + else + it { should contain 'security.authc.realms.file1.order: 0' } + it { should contain 'security.authc.realms.file1.type: file' } + it { should contain 'security.authc.realms.native1.order: 1' } + it { should contain 'security.authc.realms.native1.type: native' } end #Test contents of role_mapping.yml From 1f3c24d46a2d61a07fbb16abaf329b82134dd9e6 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Mon, 6 May 2019 23:21:13 +0200 Subject: [PATCH 20/25] [test] fix serverspec syntax --- .../helpers/serverspec/xpack_upgrade_spec.rb | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb index 1f3f59d1..d9784d13 100644 --- a/test/integration/helpers/serverspec/xpack_upgrade_spec.rb +++ b/test/integration/helpers/serverspec/xpack_upgrade_spec.rb @@ -32,14 +32,15 @@ end describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do - if vars['es_major_version'] == '7.x' - it { should contain 'security.authc.realms.file.file1.order: 0' } - it { should contain 'security.authc.realms.native.native1.order: 1' } - else - it { should contain 'security.authc.realms.file1.order: 0' } - it { should contain 'security.authc.realms.file1.type: file' } - it { should contain 'security.authc.realms.native1.order: 1' } - it { should contain 'security.authc.realms.native1.type: native' } + if vars['es_major_version'] == '7.x' + it { should contain 'security.authc.realms.file.file1.order: 0' } + it { should contain 'security.authc.realms.native.native1.order: 1' } + else + it { should contain 'security.authc.realms.file1.order: 0' } + it { should contain 'security.authc.realms.file1.type: file' } + it { should contain 'security.authc.realms.native1.order: 1' } + it { should contain 'security.authc.realms.native1.type: native' } + end end #Test contents of role_mapping.yml From e2060a5833c86030bdf409e9f5bfee87788ac99c Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Tue, 7 May 2019 16:44:02 +0200 Subject: [PATCH 21/25] fixes after pr review --- tasks/compatibility-variables.yml | 1 + tasks/elasticsearch-parameters.yml | 8 -------- tasks/main.yml | 6 ++---- test/integration/multi.yml | 12 ++++++++++-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tasks/compatibility-variables.yml b/tasks/compatibility-variables.yml index 725317b2..c87ccc6b 100644 --- a/tasks/compatibility-variables.yml +++ b/tasks/compatibility-variables.yml @@ -14,6 +14,7 @@ es_xpack_conf_subdir: "" es_repo_name: "{{ es_major_version }}" es_xpack_users_command: "elasticsearch-users" + es_package_name: "elasticsearch" es_other_package_name: "elasticsearch-oss" es_other_repo_name: "{{ 'oss-' + es_major_version }}" es_other_apt_url: "deb {{ es_repo_base }}/packages/{{ 'oss-' + es_major_version }}/apt stable main" diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 53df8ca9..6cee59f0 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -12,14 +12,6 @@ debug: msg="WARNING - It is recommended you specify the parameter 'http.port'" when: es_config['http.port'] is not defined -- name: debug message - debug: msg="WARNING - It is recommended you specify the parameter 'transport.port'" - when: es_config['transport.port'] is not defined - -- name: debug message - debug: msg="WARNING - It is recommended you specify the parameter 'discovery.seed_hosts'" - when: es_config['discovery.seed_hosts'] is not defined - #If the user attempts to lock memory they must specify a heap size - name: fail when heap size is not specified when using memory lock fail: msg="If locking memory with bootstrap.memory_lock a heap size must be specified" diff --git a/tasks/main.yml b/tasks/main.yml index c37badba..465540c9 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,10 +1,8 @@ --- -- set_fact: "es_major_version={{ es_version[0] }}.x" +- set_fact: "es_major_version={{ es_version.split('.')[0] }}.x" when: - - not es_major_version - tags: - - always + - es_major_version is undefined - name: os-specific vars include_vars: "{{ansible_os_family}}.yml" diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 02950e46..a839b836 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -10,11 +10,19 @@ es_instance_name: "master" es_data_dirs: - "/opt/elasticsearch/master" - es_config: + es_config_6x: http.port: 9200 + transport.tcp.port: 9300 node.data: false node.master: true - bootstrap.memory_lock: true + bootstrap.memory_lock: tru + es_config_7x: + http.port: 9200 + transport.port: 9300 + node.data: false + node.master: true + bootstrap.memory_lock: tru + es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" es_enable_xpack: false es_scripts: true es_templates: true From a80e3d53a750cbd8647d507efdf7f12aae8be693 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Tue, 7 May 2019 16:56:42 +0200 Subject: [PATCH 22/25] fix typo --- test/integration/multi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/multi.yml b/test/integration/multi.yml index a839b836..dec282ab 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -15,13 +15,13 @@ transport.tcp.port: 9300 node.data: false node.master: true - bootstrap.memory_lock: tru + bootstrap.memory_lock: true es_config_7x: http.port: 9200 transport.port: 9300 node.data: false node.master: true - bootstrap.memory_lock: tru + bootstrap.memory_lock: true es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" es_enable_xpack: false es_scripts: true From 5383148ae854e90da064c32d9935f6b9cf17dc9a Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Tue, 7 May 2019 17:10:51 +0200 Subject: [PATCH 23/25] fix test after pr review --- test/integration/helpers/serverspec/multi_spec.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index c54b4b48..78dd5e39 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -11,6 +11,11 @@ describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do it { should be_file } it { should contain 'http.port: 9201' } + if vars['es_major_version'] == '7.x' + it { should contain 'transport.port: 9301' } + else + it { should contain 'transport.tcp.port: 9301' } + end it { should contain 'node.data: true' } it { should contain 'node.master: false' } it { should contain "node.name: localhost-#{vars['es_instance_name']}" } From d652f4ce469057647d0ea8eb31399be25a82f9c8 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Tue, 7 May 2019 18:43:44 +0200 Subject: [PATCH 24/25] [test] fix multi-instances test - add different tests for transport port for 6.x and 7.x - fix nodes discovery for 6.x and 7.x --- test/integration/helpers/serverspec/multi_spec.rb | 5 +++++ test/integration/multi.yml | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index 78dd5e39..278ed42e 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -30,6 +30,11 @@ describe file('/etc/elasticsearch/master/elasticsearch.yml') do it { should be_file } it { should contain 'http.port: 9200' } + if vars['es_major_version'] == '7.x' + it { should contain 'transport.port: 9300' } + else + it { should contain 'transport.tcp.port: 9300' } + end it { should contain 'node.data: false' } it { should contain 'node.master: true' } it { should contain 'node.name: localhost-master' } diff --git a/test/integration/multi.yml b/test/integration/multi.yml index dec282ab..cf0b03ab 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -11,12 +11,14 @@ es_data_dirs: - "/opt/elasticsearch/master" es_config_6x: + discovery.zen.ping.unicast.hosts: "localhost:9300" http.port: 9200 transport.tcp.port: 9300 node.data: false node.master: true bootstrap.memory_lock: true es_config_7x: + discovery.seed_hosts: "localhost:9300" http.port: 9200 transport.port: 9300 node.data: false @@ -46,10 +48,19 @@ es_plugins: - plugin: ingest-attachment es_instance_name: "node1" - es_data_dirs: + es_data_dirs: - "/opt/elasticsearch/data-1" - "/opt/elasticsearch/data-2" - es_config: + es_config_6x: + discovery.zen.ping.unicast.hosts: "localhost:9300" http.port: 9201 + transport.tcp.port: 9301 node.data: true node.master: false + es_config_7x: + discovery.seed_hosts: "localhost:9300" + http.port: 9201 + transport.port: 9301 + node.data: true + node.master: false + es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" From 549a9a7fb21c56da2e6dc49045e5dbbe7028f076 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Tue, 7 May 2019 19:31:38 +0200 Subject: [PATCH 25/25] disable discovery on master nodes to avoid "master_not_discovered_exception" error when creating templates in the same play --- test/integration/multi.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/multi.yml b/test/integration/multi.yml index cf0b03ab..f018bd4f 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -18,7 +18,6 @@ node.master: true bootstrap.memory_lock: true es_config_7x: - discovery.seed_hosts: "localhost:9300" http.port: 9200 transport.port: 9300 node.data: false