Merge branch 'master' into 5.8

.travis.yml

@@ -2,31 +2,22 @@ language: python
 
 python:
   - "2.7"
-  - "3.5"
   - "3.6"
   - "3.7"
+  - "3.8"
 
 env:
-  - ES_VERSION=5.0.2
-  - ES_VERSION=5.1.2
-  - ES_VERSION=5.2.2
-  - ES_VERSION=5.3.3
-  - ES_VERSION=5.4.3
-  - ES_VERSION=5.5.3
   - ES_VERSION=5.6.16
-  - ES_VERSION=6.0.1
-  - ES_VERSION=6.1.3
-  - ES_VERSION=6.2.4
-  - ES_VERSION=6.3.2
-  - ES_VERSION=6.4.3
-  - ES_VERSION=6.5.4
-  - ES_VERSION=6.6.2
-  - ES_VERSION=6.7.2
-  - ES_VERSION=6.8.2
-  - ES_VERSION=7.0.0-linux-x86_64
-  - ES_VERSION=7.1.1-linux-x86_64
+  - ES_VERSION=6.8.13
   - ES_VERSION=7.2.1-linux-x86_64
   - ES_VERSION=7.3.1-linux-x86_64
+  - ES_VERSION=7.4.2-linux-x86_64
+  - ES_VERSION=7.5.2-linux-x86_64
+  - ES_VERSION=7.6.2-linux-x86_64
+  - ES_VERSION=7.7.1-linux-x86_64
+  - ES_VERSION=7.8.1-linux-x86_64
+  - ES_VERSION=7.9.3-linux-x86_64
+  - ES_VERSION=7.10.0-linux-x86_64
 
 os: linux
 

Dockerfile

@@ -1,16 +1,19 @@
-FROM python:3.6-alpine3.6 as builder
+FROM python:3.7-alpine3.7 as builder
 
 RUN apk --no-cache upgrade && apk --no-cache add build-base tar musl-utils openssl-dev
-RUN pip3 install setuptools cx_Freeze==6.0b1
+RUN pip3 install setuptools cx_Freeze
 
 COPY . .
 RUN ln -s /lib/libc.musl-x86_64.so.1 ldd
 RUN ln -s /lib /lib64
 RUN pip3 install -r requirements.txt
 RUN python3 setup.py build_exe
 
-FROM alpine:3.6
-RUN apk --no-cache upgrade && apk --no-cache add ca-certificates && update-ca-certificates && apk --no-cache add wget
-COPY --from=builder build/exe.linux-x86_64-3.6 /curator/
+FROM alpine:3.7
+RUN apk --no-cache upgrade && apk --no-cache add openssl-dev
+COPY --from=builder build/exe.linux-x86_64-3.7 /curator/
+
 USER nobody:nobody
+ENV LD_LIBRARY_PATH /curator/lib:$LD_LIBRARY_PATH
 ENTRYPOINT ["/curator/curator"]
+

curator/actions.py

@@ -2091,15 +2091,24 @@ def __init__(
         self.number_of_shards = number_of_shards
         self.wait_for_active_shards = wait_for_active_shards
         self.shrink_node_name = None
+
         self.body = {
             'settings': {
                 'index.number_of_shards' : number_of_shards,
                 'index.number_of_replicas' : number_of_replicas,
             }
         }
+
         if extra_settings:
             self._merge_extra_settings(extra_settings)
 
+        if utils.get_version(self.client) >= (6, 1, 0):
+            self._merge_extra_settings({
+                'settings': {
+                    'index.routing.allocation.require._name': None,
+                    'index.blocks.write': None
+                }})
+
     def _merge_extra_settings(self, extra_settings):
         self.loggit.debug(
             'Adding extra_settings to shrink body: '

curator/cli_singletons/utils.py

@@ -155,7 +155,7 @@ def config_override(ctx, config_dict):
         elif k == 'host':
             if 'host' in ctx.params and ctx.params['host'] is not None:
                 config_dict['client']['hosts'] = ctx.params[k]
-        elif k in ['loglevel', 'logfile', 'logformat']:
+        elif k in ['loglevel', 'logfile', 'logformat', 'ecs']:
             if k in ctx.params and ctx.params[k] is not None:
                 config_dict['logging'][k] = ctx.params[k]
         else:

curator/defaults/client_defaults.py

@@ -38,6 +38,10 @@ def config_logging():
                 ),
         Optional('logfile', default=None): Any(None, *string_types),
         Optional('logformat', default='default'):
+<<<<<<< HEAD
             Any(None, All(Any(*string_types), Any('default', 'json', 'logstash'))),
+=======
+            Any(None, All(Any(*string_types), Any('default', 'json', 'logstash', 'ecs'))),
+>>>>>>> master
         Optional('blacklist', default=['elasticsearch', 'urllib3']): Any(None, list),
     }

curator/exceptions.py

@@ -64,3 +64,8 @@ class ClientException(CuratorException):
     """
     Exception raised when the Elasticsearch client and/or connection is the source of the problem.
     """
+
+class LoggingException(CuratorException):
+    """
+    Exception raised when Curator cannot either log or configure logging
+    """

curator/logtools.py

@@ -3,30 +3,80 @@
 import json
 import logging
 import time
+from curator.exceptions import LoggingException
 
+def de_dot(dot_string, msg):
+    """Turn message and dotted string into a nested dictionary"""
+    arr = dot_string.split('.')
+    arr.append(msg)
+    retval = None
+    for idx in range(len(arr), 1, -1):
+        if not retval:
+            try:
+                retval = {arr[idx-2]: arr[idx-1]}
+            except Exception as err:
+                raise LoggingException(err)
+        else:
+            try:
+                new_d = {arr[idx-2]: retval}
+                retval = new_d
+            except Exception as err:
+                raise LoggingException(err)
+    return retval
+
+def deepmerge(source, destination):
+    """Merge deeply nested dictionary structures"""
+    for key, value in source.items():
+        if isinstance(value, dict):
+            node = destination.setdefault(key, {})
+            deepmerge(value, node)
+        else:
+            destination[key] = value
+    return destination
 class LogstashFormatter(logging.Formatter):
     """Logstash formatting (JSON)"""
     # The LogRecord attributes we want to carry over to the Logstash message,
     # mapped to the corresponding output key.
-    WANTED_ATTRS = {'levelname': 'loglevel',
-                    'funcName': 'function',
-                    'lineno': 'linenum',
-                    'message': 'message',
-                    'name': 'name'}
-
-    # def converter(self, timevalue):
-    #     return time.gmtime(timevalue)
+    WANTED_ATTRS = {
+        'levelname': 'loglevel',
+        'funcName': 'function',
+        'lineno': 'linenum',
+        'message': 'message',
+        'name': 'name'
+    }
 
     def format(self, record):
         self.converter = time.gmtime
         timestamp = '%s.%03dZ' % (
             self.formatTime(record, datefmt='%Y-%m-%dT%H:%M:%S'), record.msecs)
-        result = {'message': record.getMessage(),
-                  '@timestamp': timestamp}
-        for attribute in set(self.WANTED_ATTRS).intersection(record.__dict__):
-            result[self.WANTED_ATTRS[attribute]] = getattr(record, attribute)
+        result = {'@timestamp': timestamp}
+        available = record.__dict__
+        # This is cleverness because 'message' is NOT a member key of ``record.__dict__``
+        # the ``getMessage()`` method is effectively ``msg % args`` (actual keys)
+        # By manually adding 'message' to ``available``, it simplifies the code
+        available['message'] = record.getMessage()
+        for attribute in set(self.WANTED_ATTRS).intersection(available):
+            result = deepmerge(
+                de_dot(self.WANTED_ATTRS[attribute], getattr(record, attribute)), result
+            )
+        # The following is mostly for the ecs format. You can't have 2x 'message' keys in
+        # WANTED_ATTRS, so we set the value to 'log.original' in ecs, and this code block
+        # guarantees it still appears as 'message' too.
+        if 'message' not in result.items():
+            result['message'] = available['message']
         return json.dumps(result, sort_keys=True)
 
+class ECSFormatter(LogstashFormatter):
+    """Elastic Common Schema formatting (ECS)"""
+    # Overload LogstashFormatter attribute
+    WANTED_ATTRS = {
+        'levelname': 'log.level',
+        'funcName': 'log.origin.function',
+        'lineno': 'log.origin.file.line',
+        'message': 'log.original',
+        'name': 'log.logger'
+    }
+
 class Whitelist(logging.Filter):
     """How to whitelist logs"""
     def __init__(self, *whitelist):
@@ -63,5 +113,7 @@ def __init__(self, cfg):
 
         if cfg['logformat'] == 'json' or cfg['logformat'] == 'logstash':
             self.handler.setFormatter(LogstashFormatter())
+        elif cfg['logformat'] == 'ecs':
+            self.handler.setFormatter(ECSFormatter())
         else:
             self.handler.setFormatter(logging.Formatter(self.format_string))

curator/singletons.py

@@ -56,7 +56,7 @@
 @click.option('--dry-run', is_flag=True, help='Do not perform any changes.')
 @click.option('--loglevel', help='Log level')
 @click.option('--logfile', help='log file')
-@click.option('--logformat', help='Log output format [default|logstash|json].')
+@click.option('--logformat', help='Log output format [default|logstash|json|ecs].')
 @click.version_option(version=__version__)
 @click.pass_context
 def cli(

curator/utils.py

@@ -1164,7 +1164,7 @@ def find_snapshot_tasks(client):
     :rtype: bool
     """
     retval = False
-    tasklist = client.tasks.get()
+    tasklist = client.tasks.list()
     for node in tasklist['nodes']:
         for task in tasklist['nodes'][node]['tasks']:
             activity = tasklist['nodes'][node]['tasks'][task]['action']
@@ -1243,7 +1243,8 @@ def create_repo_body(repo_type=None,
                      max_snapshot_bytes_per_sec=None,
                      location=None,
                      bucket=None, region=None, base_path=None, access_key=None,
-                     secret_key=None, **kwargs):
+                     secret_key=None,
+                     role_arn=None, **kwargs):
     """
     Build the 'body' portion for use in creating a repository.
 
@@ -1270,6 +1271,8 @@ def create_repo_body(repo_type=None,
         Defaults to value of ``cloud.aws.access_key``.
     :arg secret_key: `S3 only.` The secret key to use for authentication.
         Defaults to value of ``cloud.aws.secret_key``.
+    :arg role_arn: `S3 only.` The role arn for snapshot registration.
+        Required.
 
     :returns: A dictionary suitable for creating a repository from the provided
         arguments.
@@ -1286,7 +1289,7 @@ def create_repo_body(repo_type=None,
     settingz = [] # Differentiate from module settings
     maybes = [
         'compress', 'chunk_size', 'max_restore_bytes_per_sec', 'max_snapshot_bytes_per_sec']
-    s3args = ['bucket', 'region', 'base_path', 'access_key', 'secret_key']
+    s3args = ['bucket', 'region', 'base_path', 'access_key', 'secret_key', 'role_arn']
 
     settingz += [i for i in maybes if argdict[i]]
     # Type 'fs'

docs/Changelog.rst

@@ -3,20 +3,50 @@
 Changelog
 =========
 
-5.8.2 (? ? ?)
--------------
+5.8.2 (24 November 2020)
+------------------------
+
+**Announcement**
+
+  * No, Curator isn't going away. But as you can tell, it's not as actively
+    developed as it once was. I am gratified to find there are still users who
+    make it a part of their workflow. I intend to continue development in my
+    spare time. Curator is now a labor of love, not an engineering project I
+    do during work hours.
 
 **New**
 
+  * Testing changes. Only last ES version of 5.x and 6.x are tested, plus the
+    releases of 7.x since 7.2.
   * ``http_auth`` is now deprecated. You can continue to use it, but it will go
     away in the next major release. Moving forward, you should use ``username``
     and ``password``. This should work in ``curator``, ``curator_cli``, and
     ``es_repo_mgr``.
+  * Removed tests for all 5.x branches of Elasticsearch but the final (5.6).
+  * Added tests for missing 7.x branches of Elasticsearch
+  * Remove tests for Python 3.5 
+  * Fix hang of Shrink action in ES 7.x in #1528 (jclegras)
+  * Add ``ecs`` as a ``logformat`` option in #1529 (m1keil)
 
 **Bug Fixes**
 
   * Lots of code cleanup, trying to go PEP-8. All tests are still passing, and
     the APIs are not changed (yet—-that comes in the next major release).
+  * Dockerfile has been updated to produce a working version with Python 3.7
+    and Curator 5.8.1
+  * Pin (for now) Elasticsearch Python module to 7.1.0. This will be updated
+    when an updated release of the module fixes the `cluster.state` API call
+    regression at https://github.com/elastic/elasticsearch-py/issues/1141
+  * Fix ``client.tasks.get`` API call to be ``client.tasks.list`` when no index
+    name is provided.  See
+    https://github.com/elastic/elasticsearch-py/issues/1110
+  * Pin some pip versions to allow urllib3 and boto to coexist. See #1562
+    (sethmlarson).
+
+**Documentation**
+
+  * Add Freeze/Unfreeze documentation in #1497 (lucabelluccini)
+  * Update compatibility matrix in #1522 (jibsonline)
 
 5.8.1 (25 September 2019)
 -------------------------