Move to ELK 6.5+ as the master branch.

This commit supports the full ELK stack at the latest 6.x series. Among a number of changes here are that Kibana dashboards are now managed via the Kibana API and uploaded via elk_client/filebeat. There are some workarounds here as well, most namely with a breaking change in Filebeat 6.3.0+ and how host namespaces work: https://discuss.elastic.co/t/logstash-errors-after-upgrading-to-filebeat-6-3-0/135984 Fixes: #66
sadsfae · Nov 16, 2018 · b0d0ed6 · b0d0ed6
1 parent 02add7e
commit b0d0ed6
Show file tree

Hide file tree

Showing 24 changed files with 295 additions and 532 deletions.
diff --git a/README.md b/README.md
@@ -1,12 +1,12 @@
 ansible-elk
 ===========
-Ansible Playbook for setting up the 5.x series ELK/EFK Stack and Filebeat client on remote hosts
+Ansible Playbook for setting up the ELK/EFK Stack and Filebeat client on remote hosts
 
 ![ELK](/image/ansible-elk.png?raw=true)
 
 ## What does it do?
-   - Automated deployment of a full ELK or EFK stack (Elasticsearch, Logstash/Fluentd, Kibana)
-     * 5.5+ and 2.4 ELK versions are maintained, use master for 6.x or latest
+   - Automated deployment of a full 6.5+ ELK or EFK stack (Elasticsearch, Logstash/Fluentd, Kibana)
+     * `5.6` and `2.4` ELK versions are maintained as branches and `master` branch will be 6.x currently.
      * Uses Nginx as a reverse proxy for Kibana
      * Generates SSL certificates for Filebeat or Logstash-forwarder
      * Adds either iptables or firewalld rules if firewall is active
@@ -33,7 +33,7 @@ sysctl -p
 ```
 
 ## Notes
-   - Current ELK version is 5.6.x but you can checkout the 2.4 branch if you want that series
+   - Current ELK version is 6.x but you can checkout the 5.6 or 2.4 branch if you want that series
    - Sets the nginx htpasswd to admin/admin initially
    - nginx ports default to 80/8080 for Kibana and SSL cert retrieval (configurable)
    - Uses OpenJDK for Java
@@ -60,7 +60,7 @@ sed -i 's/host-02/elkclient/' hosts
 
 ```
 ansible_system_user: ec2-user
-``` 
+```
 
    - Run the playbook
 ```
@@ -72,11 +72,21 @@ ansible-playbook -i hosts install/elk.yml
       - username: ```admin```
       - password: ```admin```
 
-![ELK](/image/elk-index-5.x-1.png?raw=true "Select @timestamp from drop-down.")
+   - Next you'll login to your Kibana instance and create a Kibana index pattern.
+
+![ELK](/image/elk6-0.png?raw=true "Click Explore on my Own")
+
+   - Note: Sample data can be useful, you can try it later however.
+
+![ELK](/image/elk6-1.png?raw=true "Click Discover")
+
+![ELK](/image/elk6.2.png?raw=true "Create index pattern")
 
-![ELK](/image/elk-index-5.x-2.png?raw=true "Click the blue create button.")
+![ELK](/image/elk6.3.png?raw=true "Select @timestamp from the drop-down and create index pattern")
 
-![ELK](/image/elk-index-5.x-3.png?raw=true "Click Discover")
+![ELK](/image/elk6.4.png?raw=true "Click Discover")
+
+   - At this point you can setup your client(s) to start sending data via Filebeat/SSL
 
 ## ELK Client Instructions
    - Run the client playbook against the generated ``elk_server`` variable
@@ -86,8 +96,15 @@ ansible-playbook -i hosts install/elk-client.yml --extra-vars 'elk_server=X.X.X.
    - Once this completes return to your ELK and you'll see log results come in from ELK/EFK clients via filebeat
 ![ELK](/image/elk-index-5.x-4.png?raw=true "watch the magic")
 
+## 5.6 ELK/EFK (Deprecated)
+   - The 5.6 series of ELK/EFK is also available, to use this just use the 2.4 branch
+```
+git clone https://github.com/sadsfae/ansible-elk
+cd ansible-elk
+git checkout 5.6
+```
 ## 2.4 ELK/EFK (Deprecated)
-   - The 2.4 series of ELK/EFK is also available, to use this just clone the 2.4 branch
+   - The 2.4 series of ELK/EFK is also available, to use this just use the 2.4 branch
 ```
 git clone https://github.com/sadsfae/ansible-elk
 cd ansible-elk
@@ -101,110 +118,6 @@ git checkout 2.4
 ## File Hierarchy
 ```
 .
-├── ansible-elk-6.2-wip
-│   ├── ansible-elk-6.2-wip
-│   ├── hosts
-│   ├── install
-│   │   ├── elk-client.yml
-│   │   ├── elk.retry
-│   │   ├── elk.yml
-│   │   ├── group_vars
-│   │   │   └── all.yml
-│   │   └── roles
-│   │       ├── curator
-│   │       │   ├── files
-│   │       │   │   └── curator.repo
-│   │       │   ├── tasks
-│   │       │   │   └── main.yml
-│   │       │   └── templates
-│   │       │       ├── curator-action.yml.j2
-│   │       │       └── curator-config.yml.j2
-│   │       ├── elasticsearch
-│   │       │   ├── files
-│   │       │   │   ├── elasticsearch.in.sh
-│   │       │   │   └── elasticsearch.repo
-│   │       │   ├── tasks
-│   │       │   │   └── main.yml
-│   │       │   └── templates
-│   │       │       └── elasticsearch.yml.j2
-│   │       ├── elk_client
-│   │       │   ├── files
-│   │       │   │   └── elk.repo
-│   │       │   └── tasks
-│   │       │       └── main.yml
-│   │       ├── filebeat
-│   │       │   ├── meta
-│   │       │   │   └── main.yml
-│   │       │   ├── tasks
-│   │       │   │   └── main.yml
-│   │       │   └── templates
-│   │       │       ├── filebeat.yml.j2
-│   │       │       └── rsyslog-openstack.conf.j2
-│   │       ├── firewall
-│   │       │   └── tasks
-│   │       │       └── main.yml
-│   │       ├── fluentd
-│   │       │   ├── files
-│   │       │   │   ├── filebeat-index-template.json
-│   │       │   │   └── fluentd.repo
-│   │       │   ├── tasks
-│   │       │   │   └── main.yml
-│   │       │   └── templates
-│   │       │       ├── openssl_extras.cnf.j2
-│   │       │       └── td-agent.conf.j2
-│   │       ├── heartbeat
-│   │       │   ├── meta
-│   │       │   │   └── main.yml
-│   │       │   ├── tasks
-│   │       │   │   └── main.yml
-│   │       │   └── templates
-│   │       │       └── heartbeat.yml.j2
-│   │       ├── instructions
-│   │       │   └── tasks
-│   │       │       └── main.yml
-│   │       ├── kibana
-│   │       │   ├── files
-│   │       │   │   ├── filebeat-dashboards.zip
-│   │       │   │   ├── kibana.repo
-│   │       │   │   └── logstash.repo
-│   │       │   └── tasks
-│   │       │       └── main.yml
-│   │       ├── logstash
-│   │       │   ├── files
-│   │       │   │   ├── filebeat-index-template.json
-│   │       │   │   └── logstash.repo
-│   │       │   ├── tasks
-│   │       │   │   └── main.yml
-│   │       │   └── templates
-│   │       │       ├── 02-beats-input.conf.j2
-│   │       │       ├── logstash.conf.j2
-│   │       │       └── openssl_extras.cnf.j2
-│   │       ├── metricbeat
-│   │       │   ├── meta
-│   │       │   │   └── main.yml
-│   │       │   ├── tasks
-│   │       │   │   └── main.yml
-│   │       │   └── templates
-│   │       │       └── metricbeat.yml.j2
-│   │       ├── nginx
-│   │       │   ├── tasks
-│   │       │   │   └── main.yml
-│   │       │   └── templates
-│   │       │       ├── kibana.conf.j2
-│   │       │       └── nginx.conf.j2
-│   │       ├── packetbeat
-│   │       │   ├── meta
-│   │       │   │   └── main.yml
-│   │       │   ├── tasks
-│   │       │   │   └── main.yml
-│   │       │   └── templates
-│   │       │       └── packetbeat.yml.j2
-│   │       └── xpack
-│   │           └── tasks
-│   │               └── main.yml
-│   └── meta
-│       └── main.yml
-├── ansible-elk-6.2-wip.tar
 ├── hosts
 ├── install
 │   ├── elk-client.yml
@@ -308,6 +221,6 @@ git checkout 2.4
 └── meta
     └── main.yml
 
-105 directories, 101 files
+52 directories, 50 files
 
 ```
diff --git a/image/elk-index-5.x-1.png b/image/elk-index-5.x-1.png
diff --git a/image/elk-index-5.x-2.png b/image/elk-index-5.x-2.png
diff --git a/image/elk-index-5.x-3.png b/image/elk-index-5.x-3.png
diff --git a/image/elk-index-5.x-4.png b/image/elk-index-5.x-4.png
diff --git a/image/elk6-0.png b/image/elk6-0.png
diff --git a/image/elk6-1.png b/image/elk6-1.png
diff --git a/image/elk6-2.png b/image/elk6-2.png
diff --git a/image/elk6-3.png b/image/elk6-3.png
diff --git a/image/elk6-4.png b/image/elk6-4.png
diff --git a/install/elk.yml b/install/elk.yml
@@ -2,6 +2,11 @@
 #
 # Playbook to install the ELK or EFK stack
 #
+# versions:
+#  - 2.4 (git branch 2.4)
+#  - 5.6 (git branch 5.6)
+#  - 6.5 (git branch master)
+#
 
 - hosts: elk
   remote_user: "{{ ansible_system_user }}"
@@ -15,5 +20,5 @@
     - { role: curator, when: install_curator_tool }
     - { role: kibana }
     - { role: xpack, when: ((install_elasticsearch_xpack) or (install_kibana_xpack) or (install_logstash_xpack)) }
-    - { role: firewall, when: manage_firewall }  
+    - { role: firewall, when: manage_firewall }
     - { role: instructions }
diff --git a/install/roles/elasticsearch/files/elasticsearch.repo b/install/roles/elasticsearch/files/elasticsearch.repo
@@ -1,6 +1,6 @@
-[elasticsearch-5.x]
-name=Elasticsearch repository for 5.x packages
-baseurl=https://artifacts.elastic.co/packages/5.x/yum
+[elasticsearch-6.x]
+name=Elasticsearch repository for 6.x packages
+baseurl=https://artifacts.elastic.co/packages/6.x/yum
 gpgcheck=1
 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
 enabled=1

diff --git a/install/roles/elasticsearch/tasks/main.yml b/install/roles/elasticsearch/tasks/main.yml
@@ -40,15 +40,15 @@
 - name: Apply heapsize start tuning for systems with greater than 64G memory
   lineinfile:
     path=/etc/elasticsearch/jvm.options
-    regexp='^-Xms2g'
+    regexp='^-Xms1g'
     line='-Xms31g'
   when: ansible_memory_mb.real.total|int >= 65536
   become: true
 
 - name: Apply heapsize end tuning for systems with greater than 64G memory
   lineinfile:
     path=/etc/elasticsearch/jvm.options
-    regexp='^-Xmx2g'
+    regexp='^-Xmx1g'
     line='-Xmx31g'
   when: ansible_memory_mb.real.total|int >= 65536
   register: elasticsearch_updated
@@ -63,15 +63,15 @@
   become: true
   lineinfile:
     path=/etc/elasticsearch/jvm.options
-    regexp='^-Xms2g'
+    regexp='^-Xms1g'
     line='-Xms{{ (ansible_memory_mb.real.total / 2) | int }}m'
   when: ansible_memory_mb.real.total|int < 65536
 
 - name: Update elasticsearch startup with end heap size
   become: true
   lineinfile:
     path=/etc/elasticsearch/jvm.options
-    regexp='^-Xmx2g'
+    regexp='^-Xmx1g'
     line='-Xmx{{ (ansible_memory_mb.real.total / 2) | int }}m'
   when: ansible_memory_mb.real.total|int < 65536
   register: elasticsearch_updated
@@ -93,6 +93,24 @@
     state=link
   become: true
 
+# this seems like a packaging bug for 6.x
+- name: Create log directory structure for ES
+  file:
+    path: /usr/share/elasticsearch/logs
+    state: directory
+    owner: elasticsearch
+    group: elasticsearch
+    mode: 0755
+
+# another packaging bug, this should be done by RPM
+- name: Create datadir for elasticsearch
+  file:
+    path: /usr/share/elasticsearch/data
+    state: directory
+    owner: elasticsearch
+    group: elasticsearch
+    mode: 0755
+
 - name: Start or restart elasticsearch service
   command: systemctl restart elasticsearch.service
   ignore_errors: true

diff --git a/install/roles/elk_client/files/elk.repo b/install/roles/elk_client/files/elk.repo
@@ -1,6 +1,6 @@
 [elk-client]
-name=Elastic repository for 5.x packages
-baseurl=https://artifacts.elastic.co/packages/5.x/yum
+name=Elastic repository for 6.x packages
+baseurl=https://artifacts.elastic.co/packages/6.x/yum
 gpgcheck=1
 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
 enabled=1

diff --git a/install/roles/elk_client/tasks/main.yml b/install/roles/elk_client/tasks/main.yml
@@ -34,7 +34,7 @@
   become: true
   when: (elk_client_ssl_cert_exists != 0)
 
-- name: Install ELK server SSL client certificate
+- name: Install ELK server SSL client key
   get_url:
     url=http://{{ elk_server }}:{{ elk_server_ssl_cert_port }}/beat-forwarder.key
     dest=/etc/beat/beat-forwarder.key

diff --git a/install/roles/filebeat/tasks/main.yml b/install/roles/filebeat/tasks/main.yml
@@ -20,7 +20,9 @@
   register: filebeat_needs_restart
 
 - name: Start filebeat service
-  command: systemctl start filebeat.service
+  service:
+    name: filebeat
+    state: started
   ignore_errors: true
   become: true
   when: ((filebeat_needs_restart != 0) and (logging_backend != 'fluentd'))