Fix _check_max_size call

Other docs and minor patches in prep for 5.5.2 release included.

Fixes elastic#1202

.travis.yml

@@ -15,7 +15,7 @@ env:
   - ES_VERSION=5.6.8
   - ES_VERSION=6.0.1
   - ES_VERSION=6.1.3
-  - ES_VERSION=6.2.2
+  - ES_VERSION=6.2.4
 
 os: linux
 
@@ -25,12 +25,6 @@ cache:
 jdk:
   - oraclejdk8
 
-before_install:
-  - cp requirements.txt ..
-  - cd ..
-  - pip install -r requirements.txt
-  - cd -
-
 install:
   - pip install -U setuptools
   - pip install -r requirements.txt

curator/_version.py

@@ -1,2 +1,2 @@
-__version__ = '5.5.1'
+__version__ = '5.5.2'
 

curator/actions.py

@@ -857,13 +857,13 @@ def __init__(
                 '"extra_settings" must be a dictionary or None')
         #: Instance variable.
         #: The Elasticsearch Client object
-        self.client     = client
+        self.client = client
         #: Instance variable.
         #: Internal reference to `conditions`
         self.conditions = self._check_max_size(conditions)
         #: Instance variable.
         #: Internal reference to `extra_settings`
-        self.settings   = extra_settings
+        self.settings = extra_settings
         #: Instance variable.
         #: Internal reference to `new_index`
         self.new_index = parse_date_pattern(new_index) if new_index else new_index
@@ -881,23 +881,20 @@ def __init__(
                     '"{0}". See previous logs for more details.'.format(name)
                 )
 
-    def _check_max_size(self, data):
+    def _check_max_size(self, conditions):
         """
         Ensure that if ``max_size`` is specified, that ``self.client``
         is running 6.1 or higher.
         """
-        try:
-            if 'max_size' in data['conditions']:
-                version = get_version(self.client)
-                if version < (6,1,0):
-                    raise ConfigurationError(
-                        'Your version of elasticsearch ({0}) does not support '
-                        'the max_size rollover condition. It is only supported '
-                        'in versions 6.1.0 and up.'.format(version)
-                    )
-        except KeyError:
-            self.loggit.debug('data does not contain dict key "conditions"')
-        return data
+        if 'max_size' in conditions:
+            version = get_version(self.client)
+            if version < (6,1,0):
+                raise ConfigurationError(
+                    'Your version of elasticsearch ({0}) does not support '
+                    'the max_size rollover condition. It is only supported '
+                    'in versions 6.1.0 and up.'.format(version)
+                )
+        return conditions
 
     def body(self):
         """
@@ -1872,12 +1869,12 @@ def _merge_extra_settings(self, extra_settings):
             try:
                 self.body['settings'].update(settings)
             except Exception as e:
-                raise ConfigurationError('Unable to apply extra settings "{0}" to shrink body'.format({'settings':settings}))
+                raise ConfigurationError('Unable to apply extra settings "{0}" to shrink body. Exception: {1}'.format({'settings':settings}, e))
         if extra_settings:
             try: # Apply any remaining keys, should there be any.
                 self.body.update(extra_settings)
             except Exception as e:
-                raise ConfigurationError('Unable to apply extra settings "{0}" to shrink body'.format(extra_settings))
+                raise ConfigurationError('Unable to apply extra settings "{0}" to shrink body. Exception: {1}'.format(extra_settings, e))
 
     def _data_node(self, node_id):
         roles = node_roles(self.client, node_id)
@@ -1990,7 +1987,6 @@ def _unblock_writes(self, idx):
     def _check_space(self, idx, dry_run=False):
         # Disk watermark calculation is already baked into `available_in_bytes`
         size = index_size(self.client, idx)
-        avail = self.shrink_node_avail
         padded = (size * 2) + (32 * 1024)
         if padded < self.shrink_node_avail:
             self.loggit.debug('Sufficient space available for 2x the size of index "{0}".  Required: {1}, available: {2}'.format(idx, padded, self.shrink_node_avail))
@@ -2142,7 +2138,7 @@ def do_action(self):
                         if self.client.indices.exists(index=target):
                             self.loggit.error('Deleting target index "{0}" due to failure to complete shrink'.format(target))
                             self.client.indices.delete(index=target)
-                        raise ActionError('Unable to shrink index "{0}" -- Error: {1}'.format(index, e))
+                        raise ActionError('Unable to shrink index "{0}" -- Error: {1}'.format(idx, e))
                     self.loggit.info('Index "{0}" successfully shrunk to "{1}"'.format(idx, target))
                     # Do post-shrink steps
                     # Unblock writes on index (just in case)

curator/defaults/client_defaults.py

@@ -1,24 +1,26 @@
-from voluptuous import *
+from six import string_types
+from voluptuous import All, Any, Boolean, Coerce, Optional, Range
 
 # Configuration file: client
+# pylint: disable=no-value-for-parameter
 def config_client():
     return {
-        Optional('hosts', default='127.0.0.1'): Any(None, str, unicode, list),
+        Optional('hosts', default='127.0.0.1'): Any(None, list, *string_types),
         Optional('port', default=9200): Any(
             None, All(Coerce(int), Range(min=1, max=65535))
         ),
-        Optional('url_prefix', default=''): Any(None, str, unicode),
+        Optional('url_prefix', default=''): Any(None, *string_types),
         Optional('use_ssl', default=False): Boolean(),
-        Optional('certificate', default=None): Any(None, str, unicode),
-        Optional('client_cert', default=None): Any(None, str, unicode),
-        Optional('client_key', default=None): Any(None, str, unicode),
-        Optional('aws_key', default=None): Any(None, str, unicode),
-        Optional('aws_secret_key', default=None): Any(None, str, unicode),
-        Optional('aws_token', default=None): Any(None, str, unicode),
+        Optional('certificate', default=None): Any(None, *string_types),
+        Optional('client_cert', default=None): Any(None, *string_types),
+        Optional('client_key', default=None): Any(None, *string_types),
+        Optional('aws_key', default=None): Any(None, *string_types),
+        Optional('aws_secret_key', default=None): Any(None, *string_types),
+        Optional('aws_token', default=None): Any(None, *string_types),
         Optional('aws_sign_request', default=False): Boolean(),
-        Optional('aws_region'): Any(None, str, unicode),
+        Optional('aws_region'): Any(None, *string_types),
         Optional('ssl_no_validate', default=False): Boolean(),
-        Optional('http_auth', default=None): Any(None, str, unicode),
+        Optional('http_auth', default=None): Any(None, *string_types),
         Optional('timeout', default=30): All(
             Coerce(int), Range(min=1, max=86400)),
         Optional('master_only', default=False): Boolean(),
@@ -32,10 +34,10 @@ def config_logging():
             'NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL',
             All(Coerce(int), Any(0, 10, 20, 30, 40, 50))
         ),
-        Optional('logfile', default=None): Any(None, str, unicode),
+        Optional('logfile', default=None): Any(None, *string_types),
         Optional(
             'logformat', default='default'): Any(None, All(
-                Any(str, unicode),
+                Any(*string_types),
                 Any('default', 'json', 'logstash')
             )
         ),

docs/Changelog.rst

@@ -3,6 +3,30 @@
 Changelog
 =========
 
+5.5.2 (11 May 2018)
+-------------------
+
+**Bug Fixes**
+
+  * Allow elasticsearch-py (Python module, not Elasticsearch) versions greater
+    than 5.5.2 and less than 7.0, but exclude 6.0.0, as it had breaking SSL
+    context changes. (untergeek)
+  * Start of pylint cleanup, and use of `six` `string_types`. (untergeek)
+  * `unit_count_pattern` setting can cause indices to mistakenly be included
+    in an index filter. Fixed in #1206 (soenkeliebau)
+  * Fix rollover _check_max_size() call. Reported in #1202 by @diranged
+    (untergeek).
+  * Update tested versions of Elasticsearch. (untergeek).
+  * Update setup.cfg to install dependencies during source install. (untergeek)
+  * Fix reference to unset variable name in log output at
+    https://github.com/elastic/curator/blob/v5.5.1/curator/actions.py#L2145
+    It should be `idx` instead of `index`. (untergeek).
+
+**Documentation**
+
+  * Clarify inclusive filtering for allocated filter. Fixed in #1203 (geekpete)
+  * Fix Kibana filter description. #1199 (quartett-opa)
+
 5.5.1 (22 March 2018)
 ---------------------
 

docs/asciidoc/index.asciidoc

@@ -1,9 +1,9 @@
-:curator_version: 5.5.1
+:curator_version: 5.5.2
 :curator_major: 5
 :curator_doc_tree: 5.5
-:es_py_version: 5.5.2
+:es_py_version: 6.2.0
 :es_doc_tree: 6.2
-:pybuild_ver: 3.6.4
+:pybuild_ver: 3.6.5
 :ref:  http://www.elastic.co/guide/en/elasticsearch/reference/{es_doc_tree}
 
 [[curator-reference]]

docs/asciidoc/installation.asciidoc

@@ -365,7 +365,7 @@ or uncompress and run `python setup.py install`
 Download and install the `certifi` dependency.  Always use the most recent
 version:
 
-. `wget https://github.com/certifi/python-certifi/archive/2018.01.18.tar.gz -O certifi.tar.gz`
+. `wget https://github.com/certifi/python-certifi/archive/2018.04.16.tar.gz -O certifi.tar.gz`
 . `pip install certifi.tar.gz`
 
  

requirements.txt

@@ -1,5 +1,5 @@
 voluptuous>=0.9.3
-elasticsearch==5.5.2
+elasticsearch>=5.5.2,!=6.0.0,<7.0.0
 click>=6.7
 pyyaml>=3.10
-certifi>=2018.1.18
+certifi>=2018.4.16

run_curator.py

@@ -25,6 +25,8 @@
 
 if __name__ == '__main__':
     try:
+        # This is because click uses decorators, and pylint doesn't catch that
+        # pylint: disable=no-value-for-parameter
         cli()
     except RuntimeError as e:
         import sys

setup.cfg

@@ -21,10 +21,18 @@ classifiers =
 [options]
 install_requires = 
     voluptuous>=0.9.3
-    elasticsearch==5.5.2
+    elasticsearch>=5.5.2,!=6.0.0,<7.0.0
     click>=6.7
     pyyaml>=3.10
-    certifi>=2018.1.18
+    certifi>=2018.4.16
+
+setup_requires =
+    voluptuous>=0.9.3
+    elasticsearch>=5.5.2,!=6.0.0,<7.0.0
+    click>=6.7
+    pyyaml>=3.10
+    certifi>=2018.4.16
+
 packages = curator
 include_package_data = True
 tests_require =

setup.py

@@ -22,11 +22,11 @@ def get_version():
     return VERSION
 
 def get_install_requires():
-    res = ['elasticsearch==5.5.2' ]
+    res = ['elasticsearch>=5.5.2,!=6.0.0,<7.0.0' ]
     res.append('click>=6.7')
     res.append('pyyaml>=3.10')
     res.append('voluptuous>=0.9.3')
-    res.append('certifi>=2018.1.18')
+    res.append('certifi>=2018.4.16')
     return res
 
 try:
@@ -107,6 +107,7 @@ def get_install_requires():
         download_url = "https://github.com/elastic/curator/tarball/v" + get_version(),
         license = "Apache License, Version 2.0",
         install_requires = get_install_requires(),
+        setup_requires = get_install_requires(),
         keywords = "elasticsearch time-series indexed index-expiry",
         packages = ["curator"],
         include_package_data=True,

test/integration/test_envvars.py

@@ -80,7 +80,7 @@ def test_do_something_with_int_value(self):
             )
         )
         test = clicktest.CliRunner()
-        result = test.invoke(
+        _ = test.invoke(
                     curator.cli,
                     [
                         '--config', self.args['configfile'],

test/integration/test_es_repo_mgr.py

@@ -102,7 +102,7 @@ def test_delete_repository_success(self):
             testvars.client_conf_logfile.format(host, port, os.devnull)
         )
         test = clicktest.CliRunner()
-        result = test.invoke(
+        _ = test.invoke(
                     curator.repo_mgr_cli,
                     [
                         '--config', self.args['configfile'],