From 710ab50624b16012d54485beeff151ff5940846a Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Mon, 4 Mar 2019 22:38:20 +0300 Subject: [PATCH 01/27] Support parallel work of multiple IPCMEssageSubscribers in one process --- salt/transport/ipc.py | 275 ++++++++++++++++++++++-------------------- 1 file changed, 147 insertions(+), 128 deletions(-) diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index f1f844bca02b..ee3c5d7c43c3 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -18,7 +18,8 @@ import tornado.gen import tornado.netutil import tornado.concurrent -from tornado.locks import Semaphore +import tornado.queues +from tornado.locks import Lock from tornado.ioloop import IOLoop, TimeoutError as TornadoTimeoutError from tornado.iostream import IOStream # Import Salt libs @@ -582,11 +583,116 @@ def __del__(self): self.close() -class IPCMessageSubscriber(IPCClient): +class IPCMessageSubscriberService(IPCClient): + ''' + IPC message subscriber service that is a standalone singleton class starting once for a number + of IPCMessageSubscriber instances feeding all of them with data. It closes automatically when + there are no more subscribers. + + To use this rever to IPCMessageSubscriber documentation. + ''' + def __singleton_init__(self, socket_path, io_loop=None): + super(IPCMessageSubscriberService, self).__singleton_init__( + socket_path, io_loop=io_loop) + self.saved_data = [] + self._read_in_progress = Lock() + self.handlers = weakref.WeakSet() + + def _subscribe(self, handler): + self.handlers.add(handler) + + def unsubscribe(self, handler): + self.handlers.discard(handler) + + def _has_subscribers(self): + return bool(self.handlers) + + def _feed_subscribers(self, data): + for subscriber in self.handlers: + subscriber._feed(data) + + @tornado.gen.coroutine + def _read(self, timeout, callback=None): + try: + yield self._read_in_progress.acquire(timeout=0) + except tornado.gen.TimeoutError: + raise tornado.gen.Return(None) + + log.debug('IPC Subscriber Service is starting reading') + # If timeout is not specified we need to set some here to make the service able to check + # is there any handler waiting for data. + if timeout is None: + timeout = 5 + + read_stream_future = None + while self._has_subscribers(): + if read_stream_future is None: + read_stream_future = self.stream.read_bytes(4096, partial=True) + + try: + wire_bytes = yield FutureWithTimeout(self.io_loop, + read_stream_future, + timeout) + read_stream_future = None + + self.unpacker.feed(wire_bytes) + msgs = [msg['body'] for msg in self.unpacker] + self._feed_subscribers(msgs) + except TornadoTimeoutError: + # Continue checking are there alive waiting handlers + # Keep 'read_stream_future' alive to wait it more in the next loop + continue + except tornado.iostream.StreamClosedError as exc: + log.trace('Subscriber disconnected from IPC %s', self.socket_path) + self._feed_subscribers([None]) + break + except Exception as exc: + log.error('Exception occurred in Subscriber while handling stream: %s', exc) + self._feed_subscribers([exc]) + break + + log.debug('IPC Subscriber Service is stopping due to a lack of subscribers') + self._read_in_progress.release() + raise tornado.gen.Return(None) + + @tornado.gen.coroutine + def read(self, handler, timeout=None): + ''' + Asynchronously read messages and invoke a callback when they are ready. + + :param callback: A callback with the received data + ''' + self._subscribe(handler) + while not self.connected(): + try: + yield self.connect(timeout=5) + except tornado.iostream.StreamClosedError: + log.trace('Subscriber closed stream on IPC %s before connect', self.socket_path) + yield tornado.gen.sleep(1) + except Exception as exc: + log.error('Exception occurred while Subscriber connecting: %s', exc) + yield tornado.gen.sleep(1) + self._read(timeout) + + def close(self): + ''' + Routines to handle any cleanup before the instance shuts down. + Sockets and filehandles should be closed explicitly, to prevent + leaks. + ''' + if not self._closing: + super(IPCMessageSubscriberService, self).close() + + def __del__(self): + if IPCMessageSubscriberService in globals(): + self.close() + + +class IPCMessageSubscriber(object): ''' Salt IPC message subscriber - Create an IPC client to receive messages from IPC publisher + Create or reuse an IPC client to receive messages from IPC publisher An example of a very simple IPCMessageSubscriber connecting to an IPCMessagePublisher. This example assumes an already running IPCMessagePublisher. @@ -615,147 +721,60 @@ class IPCMessageSubscriber(IPCClient): # Wait for some data package = ipc_subscriber.read_sync() ''' - def __singleton_init__(self, socket_path, io_loop=None): - super(IPCMessageSubscriber, self).__singleton_init__( - socket_path, io_loop=io_loop) - self._read_sync_future = None - self._read_stream_future = None - self._sync_ioloop_running = False - self.saved_data = [] - self._sync_read_in_progress = Semaphore() - - @tornado.gen.coroutine - def _read_sync(self, timeout): - yield self._sync_read_in_progress.acquire() - exc_to_raise = None - ret = None - - try: - while True: - if self._read_stream_future is None: - self._read_stream_future = self.stream.read_bytes(4096, partial=True) - - if timeout is None: - wire_bytes = yield self._read_stream_future - else: - future_with_timeout = FutureWithTimeout( - self.io_loop, self._read_stream_future, timeout) - wire_bytes = yield future_with_timeout + def __init__(self, socket_path, io_loop=None): + self.service = IPCMessageSubscriberService(socket_path, io_loop) + self.queue = tornado.queues.Queue() - self._read_stream_future = None + def connected(self): + return self.service.connected() - # Remove the timeout once we get some data or an exception - # occurs. We will assume that the rest of the data is already - # there or is coming soon if an exception doesn't occur. - timeout = None + def connect(self, callback=None, timeout=None): + return self.service.connect(callback=callback, timeout=timeout) - self.unpacker.feed(wire_bytes) - first = True - for framed_msg in self.unpacker: - if first: - ret = framed_msg['body'] - first = False - else: - self.saved_data.append(framed_msg['body']) - if not first: - # We read at least one piece of data - break - except TornadoTimeoutError: - # In the timeout case, just return None. - # Keep 'self._read_stream_future' alive. - ret = None - except tornado.iostream.StreamClosedError as exc: - log.trace('Subscriber disconnected from IPC %s', self.socket_path) - self._read_stream_future = None - exc_to_raise = exc - except Exception as exc: - log.error('Exception occurred in Subscriber while handling stream: %s', exc) - self._read_stream_future = None - exc_to_raise = exc + @tornado.gen.coroutine + def _feed(self, msgs): + for msg in msgs: + yield self.queue.put(msg) - if self._sync_ioloop_running: - # Stop the IO Loop so that self.io_loop.start() will return in - # read_sync(). - self.io_loop.spawn_callback(self.io_loop.stop) + @tornado.gen.coroutine + def read_async(self, callback, timeout=None): + ''' + Asynchronously read messages and invoke a callback when they are ready. - if exc_to_raise is not None: - raise exc_to_raise # pylint: disable=E0702 - self._sync_read_in_progress.release() - raise tornado.gen.Return(ret) + :param callback: A callback with the received data + ''' + self.service.read(self) + while True: + try: + if timeout is not None: + deadline = time.time() + timeout + else: + deadline = None + data = yield self.queue.get(timeout=deadline) + except tornado.gen.TimeoutError: + raise tornado.gen.Return(None) + if data is None: + break + elif isinstance(data, Exception): + raise data + elif callback: + self.service.io_loop.spawn_callback(callback, data) + else: + raise tornado.gen.Return(data) def read_sync(self, timeout=None): ''' Read a message from an IPC socket - The socket must already be connected. The associated IO Loop must NOT be running. :param int timeout: Timeout when receiving message :return: message data if successful. None if timed out. Will raise an exception for all other error conditions. ''' - if self.saved_data: - return self.saved_data.pop(0) - - self._sync_ioloop_running = True - self._read_sync_future = self._read_sync(timeout) - self.io_loop.start() - self._sync_ioloop_running = False - - ret_future = self._read_sync_future - self._read_sync_future = None - return ret_future.result() - - @tornado.gen.coroutine - def _read_async(self, callback): - while not self.stream.closed(): - try: - self._read_stream_future = self.stream.read_bytes(4096, partial=True) - wire_bytes = yield self._read_stream_future - self._read_stream_future = None - self.unpacker.feed(wire_bytes) - for framed_msg in self.unpacker: - body = framed_msg['body'] - self.io_loop.spawn_callback(callback, body) - except tornado.iostream.StreamClosedError: - log.trace('Subscriber disconnected from IPC %s', self.socket_path) - break - except Exception as exc: - log.error('Exception occurred while Subscriber handling stream: %s', exc) - - @tornado.gen.coroutine - def read_async(self, callback): - ''' - Asynchronously read messages and invoke a callback when they are ready. - - :param callback: A callback with the received data - ''' - while not self.connected(): - try: - yield self.connect(timeout=5) - except tornado.iostream.StreamClosedError: - log.trace('Subscriber closed stream on IPC %s before connect', self.socket_path) - yield tornado.gen.sleep(1) - except Exception as exc: - log.error('Exception occurred while Subscriber connecting: %s', exc) - yield tornado.gen.sleep(1) - yield self._read_async(callback) + return self.service.io_loop.run_sync(lambda: self.read_async(None, timeout)) def close(self): - ''' - Routines to handle any cleanup before the instance shuts down. - Sockets and filehandles should be closed explicitly, to prevent - leaks. - ''' - if not self._closing: - IPCClient.close(self) - # This will prevent this message from showing up: - # '[ERROR ] Future exception was never retrieved: - # StreamClosedError' - if self._read_sync_future is not None and self._read_sync_future.done(): - self._read_sync_future.exception() - if self._read_stream_future is not None and self._read_stream_future.done(): - self._read_stream_future.exception() + self.service.unsubscribe(self) def __del__(self): - if IPCMessageSubscriber in globals(): - self.close() + self.close() From 7656331fd2dddf84669a0d54137c794151770962 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 8 Mar 2019 11:02:02 -0700 Subject: [PATCH 02/27] Change default language from 1033 to en_US The integer was causing the .replace('_', '-') function to fail --- salt/modules/win_lgpo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 42a40ff637bf..fadab5f40758 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -133,7 +133,7 @@ # Default to `en-US` (1033) windll = ctypes.windll.kernel32 INSTALL_LANGUAGE = locale.windows_locale.get( - windll.GetSystemDefaultUILanguage(), 1033).replace('_', '-') + windll.GetSystemDefaultUILanguage(), 'en_US').replace('_', '-') except ImportError: HAS_WINDOWS_MODULES = False From 4b99afa8192f103ca28a6e16cafe3db3bceecf03 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 13 Mar 2019 17:27:39 -0700 Subject: [PATCH 03/27] Fixing the output when there are changes for the ACL state so the permissions are shown and not the octal number. --- salt/states/linux_acl.py | 13 +++++++++++-- tests/unit/states/test_linux_acl.py | 18 +++++++++--------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/salt/states/linux_acl.py b/salt/states/linux_acl.py index 38ff1c083016..7f68f3ebc5b5 100644 --- a/salt/states/linux_acl.py +++ b/salt/states/linux_acl.py @@ -29,6 +29,7 @@ # Import Python libs from __future__ import absolute_import, print_function, unicode_literals +import logging import os # Import salt libs @@ -36,6 +37,8 @@ from salt.exceptions import CommandExecutionError import salt.utils.path +log = logging.getLogger(__name__) + __virtualname__ = 'acl' @@ -60,6 +63,11 @@ def present(name, acl_type, acl_name='', perms='', recurse=False): 'comment': ''} _octal = {'r': 4, 'w': 2, 'x': 1, '-': 0} + _octal_lookup = {'1': 'x', + '2': 'w', + '4': 'r', + '5': 'rx', + '7': 'rwx'} if not os.path.exists(name): ret['comment'] = '{0} does not exist'.format(name) @@ -111,18 +119,19 @@ def present(name, acl_type, acl_name='', perms='', recurse=False): if not need_refresh: ret['comment'] = 'Permissions are in the desired state' else: + new_perms = _octal_lookup[six.text_type(user[_search_name]['octal'])] changes = {'new': {'acl_name': acl_name, 'acl_type': acl_type, 'perms': perms}, 'old': {'acl_name': acl_name, 'acl_type': acl_type, - 'perms': six.text_type(user[_search_name]['octal'])}} + 'perms': new_perms}} if __opts__['test']: ret.update({'comment': 'Updated permissions will be applied for ' '{0}: {1} -> {2}'.format( acl_name, - six.text_type(user[_search_name]['octal']), + new_perms, perms), 'result': None, 'pchanges': changes}) return ret diff --git a/tests/unit/states/test_linux_acl.py b/tests/unit/states/test_linux_acl.py index 168ce4b76251..cf4c51005936 100644 --- a/tests/unit/states/test_linux_acl.py +++ b/tests/unit/states/test_linux_acl.py @@ -43,17 +43,17 @@ def test_present(self): perms = 'rwx' mock = MagicMock(side_effect=[{name: {acl_type: [{acl_name: - {'octal': 'A'}}]}}, + {'octal': 5}}]}}, {name: {acl_type: [{acl_name: - {'octal': 'A'}}]}}, + {'octal': 5}}]}}, {name: {acl_type: [{acl_name: - {'octal': 'A'}}]}}, + {'octal': 5}}]}}, {name: {acl_type: [{}]}}, {name: {acl_type: [{}]}}, {name: {acl_type: [{}]}}, { name: {acl_type: [{acl_name: {'octal': 7}}]}, - name+"/foo": {acl_type: [{acl_name: {'octal': 'A'}}]} + name+"/foo": {acl_type: [{acl_name: {'octal': 5}}]} }, { name: {acl_type: [{acl_name: {'octal': 7}}]}, @@ -65,7 +65,7 @@ def test_present(self): with patch.dict(linux_acl.__salt__, {'acl.getfacl': mock}): # Update - test=True with patch.dict(linux_acl.__opts__, {'test': True}): - comt = ('Updated permissions will be applied for {0}: A -> {1}' + comt = ('Updated permissions will be applied for {0}: rx -> {1}' ''.format(acl_name, perms)) ret = {'name': name, 'comment': comt, @@ -75,7 +75,7 @@ def test_present(self): 'perms': perms}, 'old': {'acl_name': acl_name, 'acl_type': acl_type, - 'perms': 'A'}}, + 'perms': 'rx'}}, 'result': None} self.assertDictEqual(linux_acl.present(name, acl_type, acl_name, @@ -91,7 +91,7 @@ def test_present(self): 'perms': perms}, 'old': {'acl_name': acl_name, 'acl_type': acl_type, - 'perms': 'A'}}, + 'perms': 'rx'}}, 'pchanges': {}, 'result': True} self.assertDictEqual(linux_acl.present(name, acl_type, @@ -159,7 +159,7 @@ def test_present(self): with patch.dict(linux_acl.__salt__, {'acl.getfacl': mock}): # Update - test=True with patch.dict(linux_acl.__opts__, {'test': True}): - comt = ('Updated permissions will be applied for {0}: 7 -> {1}' + comt = ('Updated permissions will be applied for {0}: rwx -> {1}' ''.format(acl_name, perms)) ret = {'name': name, 'comment': comt, @@ -169,7 +169,7 @@ def test_present(self): 'perms': perms}, 'old': {'acl_name': acl_name, 'acl_type': acl_type, - 'perms': '7'}}, + 'perms': 'rwx'}}, 'result': None} self.assertDictEqual(linux_acl.present(name, acl_type, acl_name, From 24c907be017e01da6047eb8fdef94b6b2f860456 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 13 Mar 2019 17:48:31 -0700 Subject: [PATCH 04/27] Adding additional permissions to the lookup. --- salt/states/linux_acl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/states/linux_acl.py b/salt/states/linux_acl.py index 7f68f3ebc5b5..76b1c86c8bed 100644 --- a/salt/states/linux_acl.py +++ b/salt/states/linux_acl.py @@ -63,10 +63,13 @@ def present(name, acl_type, acl_name='', perms='', recurse=False): 'comment': ''} _octal = {'r': 4, 'w': 2, 'x': 1, '-': 0} - _octal_lookup = {'1': 'x', + _octal_lookup = {'0': '-', + '1': 'x', '2': 'w', + '3': 'wx', '4': 'r', '5': 'rx', + '6': 'rw', '7': 'rwx'} if not os.path.exists(name): From bfdb6691ffb02e71f8c35969c0ede69f278424d3 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 14 Mar 2019 10:21:42 -0700 Subject: [PATCH 05/27] Updating the reverse octal lookup dictionary. Updating tests. --- salt/states/linux_acl.py | 14 +++++--------- tests/unit/states/test_linux_acl.py | 6 +++--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/salt/states/linux_acl.py b/salt/states/linux_acl.py index 76b1c86c8bed..72fa1266d766 100644 --- a/salt/states/linux_acl.py +++ b/salt/states/linux_acl.py @@ -63,14 +63,7 @@ def present(name, acl_type, acl_name='', perms='', recurse=False): 'comment': ''} _octal = {'r': 4, 'w': 2, 'x': 1, '-': 0} - _octal_lookup = {'0': '-', - '1': 'x', - '2': 'w', - '3': 'wx', - '4': 'r', - '5': 'rx', - '6': 'rw', - '7': 'rwx'} + _octal_lookup = {0: '-', 1: 'r', 2: 'w', 4: 'x'} if not os.path.exists(name): ret['comment'] = '{0} does not exist'.format(name) @@ -122,7 +115,10 @@ def present(name, acl_type, acl_name='', perms='', recurse=False): if not need_refresh: ret['comment'] = 'Permissions are in the desired state' else: - new_perms = _octal_lookup[six.text_type(user[_search_name]['octal'])] + _num = user[_search_name]['octal'] + new_perms = '{}{}{}'.format(_octal_lookup[_num&1], + _octal_lookup[_num&2], + _octal_lookup[_num&4]) changes = {'new': {'acl_name': acl_name, 'acl_type': acl_type, 'perms': perms}, diff --git a/tests/unit/states/test_linux_acl.py b/tests/unit/states/test_linux_acl.py index cf4c51005936..8d60f9c80660 100644 --- a/tests/unit/states/test_linux_acl.py +++ b/tests/unit/states/test_linux_acl.py @@ -65,7 +65,7 @@ def test_present(self): with patch.dict(linux_acl.__salt__, {'acl.getfacl': mock}): # Update - test=True with patch.dict(linux_acl.__opts__, {'test': True}): - comt = ('Updated permissions will be applied for {0}: rx -> {1}' + comt = ('Updated permissions will be applied for {0}: r-x -> {1}' ''.format(acl_name, perms)) ret = {'name': name, 'comment': comt, @@ -75,7 +75,7 @@ def test_present(self): 'perms': perms}, 'old': {'acl_name': acl_name, 'acl_type': acl_type, - 'perms': 'rx'}}, + 'perms': 'r-x'}}, 'result': None} self.assertDictEqual(linux_acl.present(name, acl_type, acl_name, @@ -91,7 +91,7 @@ def test_present(self): 'perms': perms}, 'old': {'acl_name': acl_name, 'acl_type': acl_type, - 'perms': 'rx'}}, + 'perms': 'r-x'}}, 'pchanges': {}, 'result': True} self.assertDictEqual(linux_acl.present(name, acl_type, From 889660f9840ad5c679bd84f221571312e4e47dd6 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 14 Mar 2019 16:21:47 -0700 Subject: [PATCH 06/27] Fixing lint. --- salt/states/linux_acl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/states/linux_acl.py b/salt/states/linux_acl.py index 72fa1266d766..f7408514b75d 100644 --- a/salt/states/linux_acl.py +++ b/salt/states/linux_acl.py @@ -116,9 +116,9 @@ def present(name, acl_type, acl_name='', perms='', recurse=False): ret['comment'] = 'Permissions are in the desired state' else: _num = user[_search_name]['octal'] - new_perms = '{}{}{}'.format(_octal_lookup[_num&1], - _octal_lookup[_num&2], - _octal_lookup[_num&4]) + new_perms = '{}{}{}'.format(_octal_lookup[_num & 1], + _octal_lookup[_num & 2], + _octal_lookup[_num & 4]) changes = {'new': {'acl_name': acl_name, 'acl_type': acl_type, 'perms': perms}, From dce4ffab3501af897571f8a58f044cfc6ec2c667 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 18 Mar 2019 15:30:18 +0000 Subject: [PATCH 07/27] Avoid a traceback on tornado.testing test classes https://gist.github.com/s0undt3ch/9298a69a3492404d89a832de9efb1e68 This only happens when XML reporting is enabled. Why only now, I have no clue. --- tests/support/xmlunit.py | 48 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/tests/support/xmlunit.py b/tests/support/xmlunit.py index 61294b9e5b59..82300e61ca89 100644 --- a/tests/support/xmlunit.py +++ b/tests/support/xmlunit.py @@ -16,6 +16,8 @@ from __future__ import absolute_import import io import sys +import time +import inspect import logging # Import 3rd-party libs @@ -27,6 +29,7 @@ try: import xmlrunner.runner import xmlrunner.result + import xmlrunner.unittest HAS_XMLRUNNER = True class _DelegateIO(object): @@ -56,9 +59,48 @@ def __getattr__(self, attr): class _XMLTestResult(xmlrunner.result._XMLTestResult): def startTest(self, test): - log.debug('>>>>> START >>>>> {0}'.format(test.id())) + log.debug('>>>>> START >>>>> %s', test.id()) # xmlrunner classes are NOT new-style classes - xmlrunner.result._XMLTestResult.startTest(self, test) + # xmlrunner.result._XMLTestResult.startTest(self, test) + + # ----- Re-Implement startTest --------------------------------------------------------------------------> + # The reason being that _XMLTestResult does not like tornado testing wrapping it's test class + # https://gist.github.com/s0undt3ch/9298a69a3492404d89a832de9efb1e68 + self.start_time = time.time() + xmlrunner.unittest.TestResult.startTest(self, test) + + try: + if getattr(test, '_dt_test', None) is not None: + # doctest.DocTestCase + self.filename = test._dt_test.filename + self.lineno = test._dt_test.lineno + else: + # regular unittest.TestCase? + test_method = getattr(test, test._testMethodName) + test_class = type(test) + # Note: inspect can get confused with decorators, so use class. + self.filename = inspect.getsourcefile(test_class) + # Handle partial and partialmethod objects. + test_method = getattr(test_method, 'func', test_method) + + # ----- Code which avoids the inspect tracebacks ------------------------------------------------> + try: + from tornado.testing import _TestMethodWrapper + if isinstance(test_method, _TestMethodWrapper): + test_method = test_method.orig_method + except (ImportError, AttributeError): + pass + # <---- Code which avoids the inspect tracebacks ------------------------------------------------- + _, self.lineno = inspect.getsourcelines(test_method) + finally: + pass + + if self.showAll: + self.stream.write(' ' + self.getDescription(test)) + self.stream.write(" ... ") + self.stream.flush() + # <---- Re-Implement startTest --------------------------------------------------------------------------- + if self.buffer: # Let's override the values of self._stdXXX_buffer # We want a similar sys.stdXXX file like behaviour @@ -68,7 +110,7 @@ def startTest(self, test): sys.stdout = self._stdout_buffer def stopTest(self, test): - log.debug('<<<<< END <<<<<<< {0}'.format(test.id())) + log.debug('<<<<< END <<<<<<< %s', test.id()) # xmlrunner classes are NOT new-style classes return xmlrunner.result._XMLTestResult.stopTest(self, test) From 8c0f46baba9cb319e5f0cb85309d726c136408b4 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 18 Mar 2019 10:05:32 -0700 Subject: [PATCH 08/27] Use windows state to bootstrap windows builds This is also removing the use of dev_*.ps1 scripts in the windows builds --- .kitchen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kitchen.yml b/.kitchen.yml index 0f1a3c968e42..4a97283470e8 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -54,7 +54,7 @@ provisioner: base: "os:Windows": - match: grain - - prep_windows + - windows "*": - <%= ENV['KITCHEN_STATE'] || 'git.salt' %> pillars: From 7263956d8cd17e555451dec1c7b65dfda4d7448f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 18 Mar 2019 17:47:40 +0000 Subject: [PATCH 09/27] Lock coverage and xml-unittest-reporting versions --- noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index ab0f2a659f2c..9d5d8e7fa1a4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -127,7 +127,7 @@ def _install_requirements(session, *extra_requirements): def _run_with_coverage(session, *test_cmd): - session.install('coverage') + session.install('coverage==4.5.3') session.run('coverage', 'erase') python_path_env_var = os.environ.get('PYTHONPATH') or None if python_path_env_var is None: @@ -149,7 +149,7 @@ def _run_with_coverage(session, *test_cmd): @nox.parametrize('coverage', [False, True]) def runtests(session, coverage): # Install requirements - _install_requirements(session, 'unittest-xml-reporting') + _install_requirements(session, 'unittest-xml-reporting<2.4.0') # Create required artifacts directories _create_ci_directories() From b039cec21adacdbc71eeb3b90162389b80677cde Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 20 Mar 2019 14:02:51 +0000 Subject: [PATCH 10/27] Windows now has a static requirements file and no longer needs hacks --- noxfile.py | 11 ++-- requirements/static/windows.in | 37 +++++++++++ requirements/static/windows.txt | 90 +++++++++++++++++++++++++ tests/support/nox-windows-setup.py | 102 ----------------------------- 4 files changed, 132 insertions(+), 108 deletions(-) create mode 100644 requirements/static/windows.in create mode 100644 requirements/static/windows.txt delete mode 100644 tests/support/nox-windows-setup.py diff --git a/noxfile.py b/noxfile.py index 9d5d8e7fa1a4..46b2d39e8bba 100644 --- a/noxfile.py +++ b/noxfile.py @@ -56,7 +56,11 @@ def _install_requirements(session, *extra_requirements): # Install requirements distro_requirements = None - if not IS_WINDOWS: + if IS_WINDOWS: + _distro_requirements = os.path.join(REPO_ROOT, 'requirements', 'static', 'windows.txt') + if os.path.exists(_distro_requirements): + distro_requirements = _distro_requirements + else: # The distro package doesn't output anything for Windows session.install('distro') output = session.run('distro', '-j', silent=True) @@ -120,11 +124,6 @@ def _install_requirements(session, *extra_requirements): if extra_requirements: session.install(*extra_requirements) - if IS_WINDOWS: - # Windows hacks :/ - nox_windows_setup = os.path.join(REPO_ROOT, 'tests', 'support', 'nox-windows-setup.py') - session.run('python', nox_windows_setup) - def _run_with_coverage(session, *test_cmd): session.install('coverage==4.5.3') diff --git a/requirements/static/windows.in b/requirements/static/windows.in new file mode 100644 index 000000000000..4673bbe3006a --- /dev/null +++ b/requirements/static/windows.in @@ -0,0 +1,37 @@ +apache-libcloud==1.0.0 +boto3 +boto>=2.46.0 +dmidecode +dnspython +docker==2.7.0 +ioflo +jsonschema<=2.6.0 +keyring==5.7.1 +kubernetes<4.0 +mock<1.1.0 +more-itertools==5.0.0 +msgpack-python >= 0.4.2, != 0.5.5 +patch +psutil +pycryptodomex +pyopenssl +python-etcd==0.4.2 +python-gnupg +pyvmomi +rfc3987 +salttesting==2017.6.1 +sed +setproctitle +strict_rfc3339 +supervisor; python_version < '3' +timelib +tornado<5.0 +virtualenv + +# If running under windows, please uncomment the following 2 requirements before running +# pip-compile -o requirements/static/windows.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in +# +# On non windows, please copy the following 2 requirements to the generated windows.txt, un-commented + +#pywin32==223 +#wmi==1.4.9 diff --git a/requirements/static/windows.txt b/requirements/static/windows.txt new file mode 100644 index 000000000000..566bf3537f83 --- /dev/null +++ b/requirements/static/windows.txt @@ -0,0 +1,90 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile -o requirements/static/windows.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in +# +apache-libcloud==1.0.0 +asn1crypto==0.24.0 # via cryptography +atomicwrites==1.3.0 # via pytest +attrs==19.1.0 # via pytest +backports-abc==0.5 # via tornado +backports.ssl-match-hostname==3.7.0.1 # via docker, websocket-client +boto3==1.9.117 +boto==2.49.0 +botocore==1.12.117 # via boto3, s3transfer +cachetools==3.1.0 # via google-auth +certifi==2019.3.9 # via kubernetes, requests, tornado +cffi==1.12.2 # via cryptography +chardet==3.0.4 # via requests +coverage==4.5.3 # via pytest-cov +cryptography==2.6.1 # via pyopenssl +dmidecode==0.9.0 +dnspython==1.16.0 +docker-pycreds==0.4.0 # via docker +docker==2.7.0 +docutils==0.14 # via botocore +enum34==1.1.6 # via cryptography, raet +funcsigs==1.0.2 # via pytest +functools32==3.2.3.post2 # via jsonschema +futures==3.2.0 ; python_version < "3.0" +google-auth==1.6.3 # via kubernetes +idna==2.8 # via requests +ioflo==1.7.5 +ipaddress==1.0.22 # via cryptography, docker, kubernetes +jinja2==2.10 +jmespath==0.9.4 # via boto3, botocore +jsonschema==2.6.0 +keyring==5.7.1 +kubernetes==3.0.0 +libnacl==1.6.1 +markupsafe==1.1.1 +meld3==1.0.2 # via supervisor +mock==1.0.1 +more-itertools==5.0.0 +msgpack-python==0.5.6 +msgpack==0.6.1 +patch==1.16 +pathlib2==2.3.3 # via pytest +pluggy==0.9.0 # via pytest +psutil==5.6.1 +py==1.8.0 # via pytest +pyasn1-modules==0.2.4 # via google-auth +pyasn1==0.4.5 # via pyasn1-modules, rsa +pycparser==2.19 # via cffi +pycrypto==2.6.1 +pycryptodomex==3.7.3 +pyopenssl==19.0.0 +pytest-cov==2.6.1 +pytest-helpers-namespace==2019.1.8 +pytest-salt-runtests-bridge==2019.1.30 +pytest-salt==2018.12.8 +pytest-tempdir==2018.8.11 +pytest-timeout==1.3.3 +pytest==4.3.1 +python-dateutil==2.8.0 # via botocore, kubernetes +python-etcd==0.4.2 +python-gnupg==0.4.4 +pyvmomi==6.7.1.2018.12 +pyyaml==3.13 +pyzmq==18.0.1 ; python_version != "3.4" +raet==0.6.8 +requests==2.21.0 +rfc3987==1.3.8 +rsa==4.0 # via google-auth +s3transfer==0.2.0 # via boto3 +salttesting==2017.6.1 +scandir==1.10.0 # via pathlib2 +sed==0.3.1 +setproctitle==1.1.10 +singledispatch==3.4.0.3 # via tornado +six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, pathlib2, pyopenssl, pytest, python-dateutil, pyvmomi, raet, salttesting, singledispatch, websocket-client +strict-rfc3339==0.7 +supervisor==3.3.5 ; python_version < "3" +timelib==0.2.4 +tornado==4.5.3 ; python_version < "3" +urllib3==1.24.1 # via botocore, kubernetes, python-etcd, requests +virtualenv==16.4.3 +websocket-client==0.40.0 # via docker, kubernetes +pywin32==223 +wmi==1.4.9 diff --git a/tests/support/nox-windows-setup.py b/tests/support/nox-windows-setup.py deleted file mode 100644 index 9409884ef3ea..000000000000 --- a/tests/support/nox-windows-setup.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- -''' - tests.support.nox-windows-setup - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - This script is meant to run under the nox virtualenv to take care of required - windows procedures -''' -# pylint: disable=resource-leakage - -from __future__ import absolute_import, print_function, unicode_literals -import os -import re -import sys -import site -import shutil - -try: - import site - SITE_PACKAGES = site.getsitepackages() - PYTHON_EXECUTABLE_DIRECTORY = os.path.dirname(sys.executable) - PYTHON_SCRIPTS_DIR = os.path.join(PYTHON_EXECUTABLE_DIRECTORY, 'Scripts') -except AttributeError: - # The site module does not have the getsitepackages function when running within a virtualenv - # But the site-packages directory WILL be on sys.path - SITE_PACKAGES = None - for entry in sys.path: - if 'site-packages' in entry: - SITE_PACKAGES = entry - break - # Under a virtualenv, the python "binary" is under Scripts already. - # Well, not the binary, but the Python DLLs - PYTHON_EXECUTABLE_DIRECTORY = PYTHON_SCRIPTS_DIR = os.path.dirname(sys.executable) - -# Requests is a Salt dependency, it's safe to import, but... -try: - import requests - HAS_REQUESTS = True -except ImportError: - HAS_REQUESTS = False - -IS_64_BITS = sys.maxsize > 2**32 -SALT_REPO_URL = 'https://repo.saltstack.com/windows/dependencies/{}'.format(IS_64_BITS and 64 or 32) -DLLS = ("libeay32.dll", "ssleay32.dll", "OpenSSL_License.txt", "msvcr120.dll", "libsodium.dll") - -for dll in DLLS: - outfile = os.path.join(PYTHON_EXECUTABLE_DIRECTORY, dll) - if os.path.exists(outfile): - continue - src_url = '{}/{}'.format(SALT_REPO_URL, dll) - if HAS_REQUESTS: - print('Downloading {} to {}'.format(src_url, outfile)) - request = requests.get(src_url, allow_redirects=True) - with open(outfile, 'wb') as wfh: - wfh.write(request.content) - else: - print('ATTENTION: The python requests package is not installed, can\'t download {}'.format(src_url)) - -PYWIN32_SYSTEM32_DIR = os.path.join(SITE_PACKAGES, 'pywin32_system32') -if os.path.exists(PYWIN32_SYSTEM32_DIR): - for fname in os.listdir(PYWIN32_SYSTEM32_DIR): - if not fname.endswith('.dll'): - continue - spath = os.path.join(PYWIN32_SYSTEM32_DIR, fname) - dpath = spath.replace('pywin32_system32', 'win32') - print('Moving {} to {}'.format(spath, dpath)) - shutil.move(spath, dpath) - - print('Deleting {}'.format(PYWIN32_SYSTEM32_DIR)) - shutil.rmtree(PYWIN32_SYSTEM32_DIR, ignore_errors=True) - - -if os.path.exists(PYTHON_SCRIPTS_DIR): - print('Searching for pywin32 scripts to delete') - for fname in os.listdir(PYTHON_SCRIPTS_DIR): - if not fname.startswith('pywin32_'): - continue - fpath = os.path.join(PYTHON_SCRIPTS_DIR, fname) - print('Deleting {}'.format(fpath)) - os.unlink(fpath) - - -PYTHONWIN_DIR = os.path.join(SITE_PACKAGES, 'pythonwin') -if os.path.exists(PYTHONWIN_DIR): - print('Deleting {}'.format(PYTHONWIN_DIR)) - shutil.rmtree(PYTHONWIN_DIR, ignore_errors=True) - -PYCRPTO_NT_FILE = os.path.join(SITE_PACKAGES, 'Crypto', 'Random', 'OSRNG', 'nt.py') -if os.path.exists(PYCRPTO_NT_FILE): - with open(PYCRPTO_NT_FILE, 'r') as rfh: - contents = rfh.read() - new_contents = re.sub( - r'^import winrandom$', - 'from Crypto.Random.OSRNG import winrandom', - contents, - count=1, - flags=re.MULTILINE - ) - if contents != new_contents: - print('Patching {}'.format(PYCRPTO_NT_FILE)) - with open(PYCRPTO_NT_FILE, 'w') as wfh: - wfh.write(new_contents) From 981335a177d8d459779c4c86fcf21275d5f98b95 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 20 Mar 2019 14:09:45 +0000 Subject: [PATCH 11/27] Revert "Avoid a traceback on tornado.testing test classes" This reverts commit dce4ffab3501af897571f8a58f044cfc6ec2c667. --- tests/support/xmlunit.py | 48 +++------------------------------------- 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/tests/support/xmlunit.py b/tests/support/xmlunit.py index 82300e61ca89..61294b9e5b59 100644 --- a/tests/support/xmlunit.py +++ b/tests/support/xmlunit.py @@ -16,8 +16,6 @@ from __future__ import absolute_import import io import sys -import time -import inspect import logging # Import 3rd-party libs @@ -29,7 +27,6 @@ try: import xmlrunner.runner import xmlrunner.result - import xmlrunner.unittest HAS_XMLRUNNER = True class _DelegateIO(object): @@ -59,48 +56,9 @@ def __getattr__(self, attr): class _XMLTestResult(xmlrunner.result._XMLTestResult): def startTest(self, test): - log.debug('>>>>> START >>>>> %s', test.id()) + log.debug('>>>>> START >>>>> {0}'.format(test.id())) # xmlrunner classes are NOT new-style classes - # xmlrunner.result._XMLTestResult.startTest(self, test) - - # ----- Re-Implement startTest --------------------------------------------------------------------------> - # The reason being that _XMLTestResult does not like tornado testing wrapping it's test class - # https://gist.github.com/s0undt3ch/9298a69a3492404d89a832de9efb1e68 - self.start_time = time.time() - xmlrunner.unittest.TestResult.startTest(self, test) - - try: - if getattr(test, '_dt_test', None) is not None: - # doctest.DocTestCase - self.filename = test._dt_test.filename - self.lineno = test._dt_test.lineno - else: - # regular unittest.TestCase? - test_method = getattr(test, test._testMethodName) - test_class = type(test) - # Note: inspect can get confused with decorators, so use class. - self.filename = inspect.getsourcefile(test_class) - # Handle partial and partialmethod objects. - test_method = getattr(test_method, 'func', test_method) - - # ----- Code which avoids the inspect tracebacks ------------------------------------------------> - try: - from tornado.testing import _TestMethodWrapper - if isinstance(test_method, _TestMethodWrapper): - test_method = test_method.orig_method - except (ImportError, AttributeError): - pass - # <---- Code which avoids the inspect tracebacks ------------------------------------------------- - _, self.lineno = inspect.getsourcelines(test_method) - finally: - pass - - if self.showAll: - self.stream.write(' ' + self.getDescription(test)) - self.stream.write(" ... ") - self.stream.flush() - # <---- Re-Implement startTest --------------------------------------------------------------------------- - + xmlrunner.result._XMLTestResult.startTest(self, test) if self.buffer: # Let's override the values of self._stdXXX_buffer # We want a similar sys.stdXXX file like behaviour @@ -110,7 +68,7 @@ def startTest(self, test): sys.stdout = self._stdout_buffer def stopTest(self, test): - log.debug('<<<<< END <<<<<<< %s', test.id()) + log.debug('<<<<< END <<<<<<< {0}'.format(test.id())) # xmlrunner classes are NOT new-style classes return xmlrunner.result._XMLTestResult.stopTest(self, test) From 684bf584f68bef5d1965e81494dfbd00f5c46542 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 20 Mar 2019 21:04:49 +0300 Subject: [PATCH 12/27] Update doc conf with the new import `tornado.queues` --- doc/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/conf.py b/doc/conf.py index 4814536bf60a..7b2e3b444646 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -131,6 +131,7 @@ def inner(fn, *iargs, **ikwargs): # pylint: disable=unused-argument 'tornado.ioloop', 'tornado.iostream', 'tornado.netutil', + 'tornado.queues', 'tornado.simple_httpclient', 'tornado.stack_context', 'tornado.web', From bf9c55e6bea04e31226a7303c50ff8004c91bc0e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Mar 2019 08:29:50 +0000 Subject: [PATCH 13/27] Previously have setuptools-git installed if ioflo is to be installed --- noxfile.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/noxfile.py b/noxfile.py index 46b2d39e8bba..5536b0c02e75 100644 --- a/noxfile.py +++ b/noxfile.py @@ -59,6 +59,12 @@ def _install_requirements(session, *extra_requirements): if IS_WINDOWS: _distro_requirements = os.path.join(REPO_ROOT, 'requirements', 'static', 'windows.txt') if os.path.exists(_distro_requirements): + with open(_distro_requirements) as rfh: + if 'ioflo' in rfh.read(): + # Because we still install ioflo, which requires setuptools-git, which fails with a + # weird SSL certificate issue(weird because the requirements file requirements install + # fine), let's previously have setuptools-git installed + session.install('setuptools-git') distro_requirements = _distro_requirements else: # The distro package doesn't output anything for Windows From cc6fb4662ab33e8e15b264d58c00a443ffaac35e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Mar 2019 11:32:39 +0000 Subject: [PATCH 14/27] Additionally ignore files in nox virtualenvs and CI artifacts directories --- tests/unit/test_doc.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_doc.py b/tests/unit/test_doc.py index c7b53e03ba39..f619c397673a 100644 --- a/tests/unit/test_doc.py +++ b/tests/unit/test_doc.py @@ -58,12 +58,13 @@ def test_check_for_doc_inline_markup(self): regex = re.compile(r':(?!\\)') key, val = regex.split(line, 1) - # Don't test man pages, this file, - # the tox virtualenv files, the page - # that documents to not use ":doc:", - # or the doc/conf.py file + # Don't test man pages, this file, the tox or nox virtualenv files, + # the page that documents to not use ":doc:", the doc/conf.py file + # or the artifacts directory on nox CI test runs if 'man' in key \ or '.tox/' in key \ + or '.nox/' in key \ + or 'artifacts/' in key \ or key.endswith('test_doc.py') \ or key.endswith(os.sep.join(['doc', 'conf.py'])) \ or key.endswith(os.sep.join(['conventions', 'documentation.rst'])) \ From 57348cad9d0c0fd14aa28248632c5d2fc7da4c8b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Mar 2019 11:42:49 +0000 Subject: [PATCH 15/27] Add static requirements for Arch linux --- noxfile.py | 1 + requirements/static/arch.in | 34 +++++++++++ requirements/static/arch.txt | 111 +++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 requirements/static/arch.in create mode 100644 requirements/static/arch.txt diff --git a/noxfile.py b/noxfile.py index 46b2d39e8bba..ce9806756293 100644 --- a/noxfile.py +++ b/noxfile.py @@ -67,6 +67,7 @@ def _install_requirements(session, *extra_requirements): distro = json.loads(output.strip()) session.log('Distro information:\n%s', pprint.pformat(distro)) distro_keys = [ + '{id}'.format(**distro), '{id}-{version}'.format(**distro), '{id}-{version_parts[major]}'.format(**distro) ] diff --git a/requirements/static/arch.in b/requirements/static/arch.in new file mode 100644 index 000000000000..7a44d1a70758 --- /dev/null +++ b/requirements/static/arch.in @@ -0,0 +1,34 @@ +# This is a compilation of requirements installed on salt-jenkins git.salt state run +apache-libcloud==1.0.0 +boto3 +boto>=2.46.0 +cffi +cherrypy==17.3.0 +dnspython +docker +futures>=2.0; python_version < '3.0' +GitPython +ioflo +jsonschema<=2.6.0 +keyring==5.7.1 +kubernetes<4.0 +mock<1.1.0 +more-itertools==5.0.0 +moto +msgpack-python >= 0.4.2, != 0.5.5 +psutil +pycrypto>=2.6.1 +pyinotify +pyopenssl +python-etcd==0.4.2 +python-gnupg +pyvmomi +pyzmq +requests +rfc3987 +salttesting==2017.6.1 +setproctitle +strict_rfc3339 +timelib +tornado<5.0 +virtualenv diff --git a/requirements/static/arch.txt b/requirements/static/arch.txt new file mode 100644 index 000000000000..192cd18e66cf --- /dev/null +++ b/requirements/static/arch.txt @@ -0,0 +1,111 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile -o requirements/static/arch.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/arch.in +# +apache-libcloud==1.0.0 +asn1crypto==0.24.0 # via cryptography +atomicwrites==1.3.0 # via pytest +attrs==19.1.0 # via pytest +aws-xray-sdk==0.95 # via moto +backports-abc==0.5 # via tornado +backports.functools-lru-cache==1.5 # via cheroot, jaraco.functools +backports.ssl-match-hostname==3.7.0.1 # via docker, websocket-client +backports.tempfile==1.0 # via moto +backports.weakref==1.0.post1 # via backports.tempfile +boto3==1.9.118 +boto==2.49.0 +botocore==1.12.118 # via boto3, moto, s3transfer +cachetools==3.1.0 # via google-auth +certifi==2019.3.9 # via kubernetes, requests, tornado +cffi==1.12.2 +chardet==3.0.4 # via requests +cheroot==6.5.4 # via cherrypy +cherrypy==17.3.0 +contextlib2==0.5.5 # via cherrypy +cookies==2.2.1 # via responses +coverage==4.5.3 # via pytest-cov +cryptography==2.6.1 # via moto, pyopenssl +dnspython==1.16.0 +docker-pycreds==0.4.0 # via docker +docker==3.7.1 +docutils==0.14 # via botocore +ecdsa==0.13 # via python-jose +enum34==1.1.6 # via cryptography, raet +funcsigs==1.0.2 # via pytest +functools32==3.2.3.post2 # via jsonschema +future==0.17.1 # via python-jose +futures==3.2.0 ; python_version < "3.0" +gitdb2==2.0.5 # via gitpython +gitpython==2.1.11 +google-auth==1.6.3 # via kubernetes +idna==2.8 # via requests +ioflo==1.7.5 +ipaddress==1.0.22 # via cryptography, docker, kubernetes +jaraco.functools==2.0 # via tempora +jinja2==2.10 +jmespath==0.9.4 # via boto3, botocore +jsondiff==1.1.1 # via moto +jsonpickle==1.1 # via aws-xray-sdk +jsonschema==2.6.0 +keyring==5.7.1 +kubernetes==3.0.0 +libnacl==1.6.1 +markupsafe==1.1.1 +mock==1.0.1 +more-itertools==5.0.0 +moto==1.3.7 +msgpack-python==0.5.6 +msgpack==0.6.1 +pathlib2==2.3.3 # via pytest +pluggy==0.9.0 # via pytest +portend==2.3 # via cherrypy +psutil==5.6.1 +py==1.8.0 # via pytest +pyaml==18.11.0 # via moto +pyasn1-modules==0.2.4 # via google-auth +pyasn1==0.4.5 # via pyasn1-modules, rsa +pycparser==2.19 # via cffi +pycrypto==2.6.1 +pycryptodome==3.7.3 # via python-jose +pyinotify==0.9.6 +pyopenssl==19.0.0 +pytest-cov==2.6.1 +pytest-helpers-namespace==2019.1.8 +pytest-salt-runtests-bridge==2019.1.30 +pytest-salt==2018.12.8 +pytest-tempdir==2018.8.11 +pytest-timeout==1.3.3 +pytest==4.3.1 +python-dateutil==2.8.0 # via botocore, kubernetes, moto +python-etcd==0.4.2 +python-gnupg==0.4.4 +python-jose==2.0.2 # via moto +pytz==2018.9 # via moto, tempora +pyvmomi==6.7.1.2018.12 +pyyaml==3.13 +pyzmq==18.0.1 ; python_version != "3.4" +raet==0.6.8 +requests==2.21.0 +responses==0.10.6 # via moto +rfc3987==1.3.8 +rsa==4.0 # via google-auth +s3transfer==0.2.0 # via boto3 +salttesting==2017.6.1 +scandir==1.10.0 # via pathlib2 +setproctitle==1.1.10 +singledispatch==3.4.0.3 # via tornado +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, moto, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +smmap2==2.0.5 # via gitdb2 +strict-rfc3339==0.7 +tempora==1.14 # via portend +timelib==0.2.4 +tornado==4.5.3 ; python_version < "3" +urllib3==1.24.1 # via botocore, kubernetes, python-etcd, requests +virtualenv==16.4.3 +websocket-client==0.40.0 # via docker, kubernetes +werkzeug==0.15.0 # via moto +wrapt==1.11.1 # via aws-xray-sdk +xmltodict==0.12.0 # via moto +zc.lockfile==1.4 # via cherrypy From 36ed50d771ce6b9c54ba94e4a162bb62e0f5d698 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Mar 2019 11:51:48 +0000 Subject: [PATCH 16/27] Lock to unittest-xml-reporting 2.2.1 After this version, the library started taking into account expected failures which Jenkins treats as regular failures. --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index ce9806756293..c09d57924a71 100644 --- a/noxfile.py +++ b/noxfile.py @@ -149,7 +149,7 @@ def _run_with_coverage(session, *test_cmd): @nox.parametrize('coverage', [False, True]) def runtests(session, coverage): # Install requirements - _install_requirements(session, 'unittest-xml-reporting<2.4.0') + _install_requirements(session, 'unittest-xml-reporting==2.2.1') # Create required artifacts directories _create_ci_directories() From 1f99e2855033789dd3c96810f4fafe6a465466ee Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Mar 2019 19:26:48 +0000 Subject: [PATCH 17/27] Let's not include raet by default --- requirements/static/arch.txt | 8 +++----- requirements/static/centos-6.txt | 8 +++----- requirements/static/centos-7.txt | 8 +++----- requirements/static/debian-8.txt | 8 +++----- requirements/static/debian-9.txt | 8 +++----- requirements/static/fedora-28.txt | 8 +++----- requirements/static/fedora-29.txt | 8 +++----- requirements/static/opensuse-42.txt | 8 +++----- requirements/static/opensuse-leap-15.txt | 8 +++----- requirements/static/ubuntu-14.04.txt | 8 +++----- requirements/static/ubuntu-16.04.txt | 8 +++----- requirements/static/ubuntu-18.04.txt | 8 +++----- requirements/static/windows.txt | 8 +++----- 13 files changed, 39 insertions(+), 65 deletions(-) diff --git a/requirements/static/arch.txt b/requirements/static/arch.txt index 192cd18e66cf..a0ad0126e167 100644 --- a/requirements/static/arch.txt +++ b/requirements/static/arch.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/arch.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/arch.in +# pip-compile -o requirements/static/arch.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/arch.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -32,7 +32,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.1 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -51,7 +51,6 @@ jsonpickle==1.1 # via aws-xray-sdk jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 markupsafe==1.1.1 mock==1.0.1 more-itertools==5.0.0 @@ -86,7 +85,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.6 # via moto rfc3987==1.3.8 @@ -96,7 +94,7 @@ salttesting==2017.6.1 scandir==1.10.0 # via pathlib2 setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, moto, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, moto, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14 # via portend diff --git a/requirements/static/centos-6.txt b/requirements/static/centos-6.txt index 1d88a5ab47b9..a21a3ac8fd5a 100644 --- a/requirements/static/centos-6.txt +++ b/requirements/static/centos-6.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/centos-6.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/centos-6.in +# pip-compile -o requirements/static/centos-6.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/centos-6.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -33,7 +33,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -54,7 +54,6 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 lxml==4.3.2 # via junos-eznc, ncclient markupsafe==1.1.1 meld3==1.0.2 # via supervisor @@ -96,7 +95,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -108,7 +106,7 @@ scp==0.13.1 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap==0.9.0 # via gitdb strict-rfc3339==0.7 supervisor==3.3.5 ; python_version < "3" diff --git a/requirements/static/centos-7.txt b/requirements/static/centos-7.txt index 7534e24aa952..cde8d51ee818 100644 --- a/requirements/static/centos-7.txt +++ b/requirements/static/centos-7.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/centos-7.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/centos-7.in +# pip-compile -o requirements/static/centos-7.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/centos-7.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -33,7 +33,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -54,7 +54,6 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 lxml==4.3.2 # via junos-eznc, ncclient markupsafe==1.1.1 meld3==1.0.2 # via supervisor @@ -96,7 +95,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -108,7 +106,7 @@ scp==0.13.1 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 supervisor==3.3.5 ; python_version < "3" diff --git a/requirements/static/debian-8.txt b/requirements/static/debian-8.txt index 035d9715bfdf..573d062eec22 100644 --- a/requirements/static/debian-8.txt +++ b/requirements/static/debian-8.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/debian-8.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/debian-8.in +# pip-compile -o requirements/static/debian-8.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/debian-8.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -32,7 +32,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -53,7 +53,6 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 lxml==4.3.2 # via junos-eznc, ncclient markupsafe==1.1.1 mock==1.0.1 @@ -93,7 +92,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -105,7 +103,7 @@ scp==0.13.1 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14 # via portend diff --git a/requirements/static/debian-9.txt b/requirements/static/debian-9.txt index 640f2b662de9..55435505fc6e 100644 --- a/requirements/static/debian-9.txt +++ b/requirements/static/debian-9.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/debian-9.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/debian-9.in +# pip-compile -o requirements/static/debian-9.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/debian-9.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -32,7 +32,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -53,7 +53,6 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 lxml==4.3.2 # via junos-eznc, ncclient markupsafe==1.1.1 mock==1.0.1 @@ -93,7 +92,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -105,7 +103,7 @@ scp==0.13.1 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14 # via portend diff --git a/requirements/static/fedora-28.txt b/requirements/static/fedora-28.txt index 1ff78a38d632..71f53a9b5b76 100644 --- a/requirements/static/fedora-28.txt +++ b/requirements/static/fedora-28.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/fedora-28.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/fedora-28.in +# pip-compile -o requirements/static/fedora-28.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/fedora-28.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -33,7 +33,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -54,7 +54,6 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 lxml==4.3.2 # via junos-eznc, ncclient markupsafe==1.1.1 mock==1.0.1 @@ -95,7 +94,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -107,7 +105,7 @@ scp==0.13.1 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14 # via portend diff --git a/requirements/static/fedora-29.txt b/requirements/static/fedora-29.txt index 7049c8db4333..57753ce405de 100644 --- a/requirements/static/fedora-29.txt +++ b/requirements/static/fedora-29.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/fedora-29.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/fedora-29.in +# pip-compile -o requirements/static/fedora-29.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/fedora-29.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -33,7 +33,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -54,7 +54,6 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 lxml==4.3.2 # via junos-eznc, ncclient markupsafe==1.1.1 mock==1.0.1 @@ -95,7 +94,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -107,7 +105,7 @@ scp==0.13.1 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14 # via portend diff --git a/requirements/static/opensuse-42.txt b/requirements/static/opensuse-42.txt index 53df3ccdd8d6..a8ea73e414cc 100644 --- a/requirements/static/opensuse-42.txt +++ b/requirements/static/opensuse-42.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/opensuse-42.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/opensuse-42.in +# pip-compile -o requirements/static/opensuse-42.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/opensuse-42.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -32,7 +32,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -52,7 +52,6 @@ jsonpickle==1.1 # via aws-xray-sdk jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 markupsafe==1.1.1 meld3==1.0.2 # via supervisor mock==1.0.1 @@ -88,7 +87,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -99,7 +97,7 @@ scandir==1.10.0 # via pathlib2 setproctitle==1.1.10 setuptools-scm==3.2.0 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, moto, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, moto, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 supervisor==3.3.5 ; python_version < "3" diff --git a/requirements/static/opensuse-leap-15.txt b/requirements/static/opensuse-leap-15.txt index 26f78a6fafed..2d56a9f92875 100644 --- a/requirements/static/opensuse-leap-15.txt +++ b/requirements/static/opensuse-leap-15.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/opensuse-leap-15.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/opensuse-leap-15.in +# pip-compile -o requirements/static/opensuse-leap-15.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/opensuse-leap-15.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -32,7 +32,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -52,7 +52,6 @@ jsonpickle==1.1 # via aws-xray-sdk jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 markupsafe==1.1.1 mock==1.0.1 more-itertools==5.0.0 @@ -87,7 +86,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -98,7 +96,7 @@ scandir==1.10.0 # via pathlib2 setproctitle==1.1.10 setuptools-scm==3.2.0 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, moto, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, moto, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14 # via portend diff --git a/requirements/static/ubuntu-14.04.txt b/requirements/static/ubuntu-14.04.txt index d59e4176d971..8efda726a2f5 100644 --- a/requirements/static/ubuntu-14.04.txt +++ b/requirements/static/ubuntu-14.04.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/ubuntu-14.04.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/ubuntu-14.04.in +# pip-compile -o requirements/static/ubuntu-14.04.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/ubuntu-14.04.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -32,7 +32,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -53,7 +53,6 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 lxml==4.3.2 # via junos-eznc, ncclient markupsafe==1.1.1 mock==1.0.1 @@ -93,7 +92,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -105,7 +103,7 @@ scp==0.13.1 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14 # via portend diff --git a/requirements/static/ubuntu-16.04.txt b/requirements/static/ubuntu-16.04.txt index 1647182c16e1..8e7a08f6ef01 100644 --- a/requirements/static/ubuntu-16.04.txt +++ b/requirements/static/ubuntu-16.04.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/ubuntu-16.04.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/ubuntu-16.04.in +# pip-compile -o requirements/static/ubuntu-16.04.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/ubuntu-16.04.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -32,7 +32,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -53,7 +53,6 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 lxml==4.3.2 # via junos-eznc, ncclient markupsafe==1.1.1 mock==1.0.1 @@ -93,7 +92,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -105,7 +103,7 @@ scp==0.13.1 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14 # via portend diff --git a/requirements/static/ubuntu-18.04.txt b/requirements/static/ubuntu-18.04.txt index 49837a33505d..9d3547a56fad 100644 --- a/requirements/static/ubuntu-18.04.txt +++ b/requirements/static/ubuntu-18.04.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/ubuntu-18.04.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/ubuntu-18.04.in +# pip-compile -o requirements/static/ubuntu-18.04.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/ubuntu-18.04.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -32,7 +32,7 @@ docker-pycreds==0.4.0 # via docker docker==3.7.0 docutils==0.14 # via botocore ecdsa==0.13 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose @@ -53,7 +53,6 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 lxml==4.3.2 # via junos-eznc, ncclient markupsafe==1.1.1 mock==1.0.1 @@ -93,7 +92,6 @@ pytz==2018.9 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 responses==0.10.5 # via moto rfc3987==1.3.8 @@ -105,7 +103,7 @@ scp==0.13.1 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, more-itertools, moto, ncclient, pathlib2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14 # via portend diff --git a/requirements/static/windows.txt b/requirements/static/windows.txt index 566bf3537f83..01d70c39103a 100644 --- a/requirements/static/windows.txt +++ b/requirements/static/windows.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/windows.txt requirements/zeromq.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in +# pip-compile -o requirements/static/windows.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/windows.in # apache-libcloud==1.0.0 asn1crypto==0.24.0 # via cryptography @@ -24,7 +24,7 @@ dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==2.7.0 docutils==0.14 # via botocore -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 # via cryptography funcsigs==1.0.2 # via pytest functools32==3.2.3.post2 # via jsonschema futures==3.2.0 ; python_version < "3.0" @@ -37,7 +37,6 @@ jmespath==0.9.4 # via boto3, botocore jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 -libnacl==1.6.1 markupsafe==1.1.1 meld3==1.0.2 # via supervisor mock==1.0.1 @@ -68,7 +67,6 @@ python-gnupg==0.4.4 pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" -raet==0.6.8 requests==2.21.0 rfc3987==1.3.8 rsa==4.0 # via google-auth @@ -78,7 +76,7 @@ scandir==1.10.0 # via pathlib2 sed==0.3.1 setproctitle==1.1.10 singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, pathlib2, pyopenssl, pytest, python-dateutil, pyvmomi, raet, salttesting, singledispatch, websocket-client +six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, more-itertools, pathlib2, pyopenssl, pytest, python-dateutil, pyvmomi, salttesting, singledispatch, websocket-client strict-rfc3339==0.7 supervisor==3.3.5 ; python_version < "3" timelib==0.2.4 From cbe5f7d26777e7d830a01d62be52c1111d4e278b Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Thu, 21 Mar 2019 17:26:38 -0600 Subject: [PATCH 18/27] Update to allow for malformed CPE_NAME from some OS's --- salt/grains/core.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 9758275f2c53..d3711edf3360 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1495,6 +1495,10 @@ def _parse_cpe_name(cpe): Info: https://csrc.nist.gov/projects/security-content-automation-protocol/scap-specifications/cpe + Note: cpe:2.3:part:vendor:product:version:update:edition:lang:sw_edition:target_sw:target_hw:other + however some OS's do not have the full 13 elements, for example: + CPE_NAME="cpe:2.3:o:amazon:amazon_linux:2" + :param cpe: :return: ''' @@ -1510,7 +1514,11 @@ def _parse_cpe_name(cpe): ret['vendor'], ret['product'], ret['version'] = cpe[2:5] ret['phase'] = cpe[5] if len(cpe) > 5 else None ret['part'] = part.get(cpe[1][1:]) - elif len(cpe) == 13 and cpe[1] == '2.3': # WFN to a string + elif len(cpe) == 6 and cpe[1] == '2.3': # WFN to a string + ret['vendor'], ret['product'], ret['version'] = [x if x != '*' else None for x in cpe[3:6]] + ret['phase'] = None + ret['part'] = part.get(cpe[2]) + elif len(cpe) > 7 and len(cpe) <= 13 and cpe[1] == '2.3': # WFN to a string ret['vendor'], ret['product'], ret['version'], ret['phase'] = [x if x != '*' else None for x in cpe[3:7]] ret['part'] = part.get(cpe[2]) From 0b1eb00dc44b939d6d9d8c03d163e458e1e2b928 Mon Sep 17 00:00:00 2001 From: Alan Cugler Date: Mon, 25 Mar 2019 12:09:48 -0500 Subject: [PATCH 19/27] Replaced `test.ping` with `test.version` Result of issue #52277 Any where `test.ping` still exists in the docs was left because of the context in those docs. --- doc/ref/cli/salt-unity.rst | 4 ++-- doc/ref/cli/salt.rst | 4 ++-- doc/ref/configuration/index.rst | 16 ++++++------- doc/ref/executors/index.rst | 4 ++-- doc/ref/peer.rst | 6 ++--- doc/ref/returners/index.rst | 24 +++++++++---------- doc/ref/runners/index.rst | 2 +- doc/topics/blackout/index.rst | 4 ++-- doc/topics/cloud/aws.rst | 2 +- doc/topics/cloud/azure.rst | 2 +- doc/topics/cloud/azurearm.rst | 2 +- doc/topics/cloud/deploy.rst | 2 +- doc/topics/cloud/gce.rst | 2 +- doc/topics/cloud/libvirt.rst | 2 +- doc/topics/cloud/linode.rst | 2 +- doc/topics/cloud/opennebula.rst | 2 +- doc/topics/cloud/parallels.rst | 2 +- doc/topics/cloud/proxmox.rst | 2 +- doc/topics/cloud/saltify.rst | 4 ++-- doc/topics/cloud/softlayer.rst | 6 ++--- doc/topics/cloud/vagrant.rst | 2 +- doc/topics/cloud/vexxhost.rst | 2 +- doc/topics/cloud/xen.rst | 2 +- doc/topics/development/architecture.rst | 2 +- doc/topics/development/hacking.rst | 2 +- doc/topics/eauth/access_control.rst | 4 ++-- doc/topics/eauth/index.rst | 4 ++-- doc/topics/event/master_events.rst | 4 ++-- doc/topics/installation/index.rst | 4 ++-- doc/topics/installation/windows.rst | 2 +- doc/topics/proxyminion/demo.rst | 2 +- doc/topics/ssh/index.rst | 6 ++--- doc/topics/targeting/batch.rst | 4 ++-- doc/topics/targeting/compound.rst | 10 ++++---- doc/topics/targeting/globbing.rst | 18 +++++++------- doc/topics/targeting/grains.rst | 2 +- doc/topics/targeting/index.rst | 4 ++-- doc/topics/targeting/ipcidr.rst | 6 ++--- doc/topics/targeting/nodegroups.rst | 2 +- doc/topics/targeting/pillar.rst | 4 ++-- doc/topics/targeting/range.rst | 4 ++-- doc/topics/thorium/index.rst | 4 ++-- doc/topics/topology/syndic.rst | 10 ++++---- doc/topics/tutorials/cloud_controller.rst | 2 +- doc/topics/tutorials/docker_sls.rst | 2 +- doc/topics/tutorials/esxi_proxy_minion.rst | 11 +++++---- doc/topics/tutorials/firewall.rst | 2 +- doc/topics/tutorials/modules.rst | 16 ++++++------- doc/topics/tutorials/multimaster_pki.rst | 2 +- doc/topics/tutorials/walkthrough.rst | 26 ++++++++++----------- doc/topics/tutorials/walkthrough_macosx.rst | 4 ++-- 51 files changed, 131 insertions(+), 132 deletions(-) diff --git a/doc/ref/cli/salt-unity.rst b/doc/ref/cli/salt-unity.rst index 2144acb07ffc..db746038c647 100644 --- a/doc/ref/cli/salt-unity.rst +++ b/doc/ref/cli/salt-unity.rst @@ -9,7 +9,7 @@ Synopsis .. code-block:: bash - salt-unity salt '*' test.ping + salt-unity salt '*' test.version Description =========== @@ -35,4 +35,4 @@ See also :manpage:`salt-minion(1)` :manpage:`salt-run(1)` :manpage:`salt-ssh(1)` -:manpage:`salt-syndic(1)` \ No newline at end of file +:manpage:`salt-syndic(1)` diff --git a/doc/ref/cli/salt.rst b/doc/ref/cli/salt.rst index 1cbad32ba6f7..da6de812b0bd 100644 --- a/doc/ref/cli/salt.rst +++ b/doc/ref/cli/salt.rst @@ -11,9 +11,9 @@ Synopsis salt -E '.*' [ options ] sys.doc cmd - salt -G 'os:Arch.*' [ options ] test.ping + salt -G 'os:Arch.*' [ options ] test.version - salt -C 'G@os:Arch.* and webserv* or G@kernel:FreeBSD' [ options ] test.ping + salt -C 'G@os:Arch.* and webserv* or G@kernel:FreeBSD' [ options ] test.version Description =========== diff --git a/doc/ref/configuration/index.rst b/doc/ref/configuration/index.rst index b57f25ccd6d1..b54d7053d53f 100644 --- a/doc/ref/configuration/index.rst +++ b/doc/ref/configuration/index.rst @@ -199,28 +199,28 @@ Sending Commands ================ Communication between the Master and a Minion may be verified by running -the ``test.ping`` command: +the ``test.version`` command: .. code-block:: bash - [root@master ~]# salt alpha test.ping + [root@master ~]# salt alpha test.version alpha: - True + 2018.3.4 Communication between the Master and all Minions may be tested in a similar way: .. code-block:: bash - [root@master ~]# salt '*' test.ping + [root@master ~]# salt '*' test.version alpha: - True + 2018.3.4 bravo: - True + 2018.3.4 charlie: - True + 2018.3.4 delta: - True + 2018.3.4 Each of the Minions should send a ``True`` response as shown above. diff --git a/doc/ref/executors/index.rst b/doc/ref/executors/index.rst index f7a0e33a8412..6aaf8755fb2e 100644 --- a/doc/ref/executors/index.rst +++ b/doc/ref/executors/index.rst @@ -30,7 +30,7 @@ The same could be done by command line: .. code-block:: bash - salt -t 40 --module-executors='[splay, direct_call]' --executor-opts='{splaytime: 30}' '*' test.ping + salt -t 40 --module-executors='[splay, direct_call]' --executor-opts='{splaytime: 30}' '*' test.version And the same command called via netapi will look like this: @@ -43,7 +43,7 @@ And the same command called via netapi will look like this: -d '[{ "client": "local", "tgt": "*", - "fun": "test.ping", + "fun": "test.version", "module_executors": ["splay", "direct_call"], "executor_opts": {"splaytime": 10} }]' diff --git a/doc/ref/peer.rst b/doc/ref/peer.rst index 83118d8b7b30..3e4068d93bbc 100644 --- a/doc/ref/peer.rst +++ b/doc/ref/peer.rst @@ -107,11 +107,11 @@ comes with a number of functions to execute peer communication in different ways. Currently there are three functions in the publish module. These examples will show how to test the peer system via the salt-call command. -To execute test.ping on all minions: +To execute test.version on all minions: .. code-block:: bash - # salt-call publish.publish \* test.ping + # salt-call publish.publish \* test.version To execute the manage.up runner: @@ -123,7 +123,7 @@ To match minions using other matchers, use ``tgt_type``: .. code-block:: bash - # salt-call publish.publish 'webserv* and not G@os:Ubuntu' test.ping tgt_type='compound' + # salt-call publish.publish 'webserv* and not G@os:Ubuntu' test.version tgt_type='compound' .. note:: In pre-2017.7.0 releases, use ``expr_form`` instead of ``tgt_type``. diff --git a/doc/ref/returners/index.rst b/doc/ref/returners/index.rst index 36176f42aecb..a5601a66f99a 100644 --- a/doc/ref/returners/index.rst +++ b/doc/ref/returners/index.rst @@ -31,7 +31,7 @@ Specifying what returners to use is done when the command is invoked: .. code-block:: bash - salt '*' test.ping --return redis_return + salt '*' test.version --return redis_return This command will ensure that the redis_return returner is used. @@ -39,10 +39,10 @@ It is also possible to specify multiple returners: .. code-block:: bash - salt '*' test.ping --return mongo_return,redis_return,cassandra_return + salt '*' test.version --return mongo_return,redis_return,cassandra_return In this scenario all three returners will be called and the data from the -test.ping command will be sent out to the three named returners. +test.version command will be sent out to the three named returners. Writing a Returner ================== @@ -61,13 +61,13 @@ Other optional functions can be included to add support for ``returner`` The ``returner`` function must accept a single argument. The argument contains return data from the called minion function. If the minion - function ``test.ping`` is called, the value of the argument will be a + function ``test.version`` is called, the value of the argument will be a dictionary. Run the following command from a Salt master to get a sample of the dictionary: .. code-block:: bash - salt-call --local --metadata test.ping --out=pprint + salt-call --local --metadata test.version --out=pprint .. code-block:: python @@ -246,12 +246,12 @@ Sample: "master_minion": { "fun_args": [], "jid": "20150330121011408195", - "return": true, + "return": "2018.3.4", "retcode": 0, "success": true, "cmd": "_return", "_stamp": "2015-03-30T12:10:12.708663", - "fun": "test.ping", + "fun": "test.version", "id": "master_minion" } } @@ -267,9 +267,9 @@ Sample: { "local": { - "minion1": "test.ping", - "minion3": "test.ping", - "minion2": "test.ping" + "minion1": "test.version", + "minion3": "test.version", + "minion2": "test.version" } } @@ -344,7 +344,7 @@ Testing the Returner The ``returner``, ``prep_jid``, ``save_load``, ``get_load``, and ``event_return`` functions can be tested by configuring the :conf_master:`master_job_cache` and `Event Returners`_ in the master config -file and submitting a job to ``test.ping`` each minion from the master. +file and submitting a job to ``test.version`` each minion from the master. Once you have successfully exercised the Master Job Cache functions, test the External Job Cache functions using the ``ret`` execution module. @@ -352,7 +352,7 @@ External Job Cache functions using the ``ret`` execution module. .. code-block:: bash salt-call ret.get_jids cassandra_cql --output=json - salt-call ret.get_fun cassandra_cql test.ping --output=json + salt-call ret.get_fun cassandra_cql test.version --output=json salt-call ret.get_minions cassandra_cql --output=json salt-call ret.get_jid cassandra_cql 20150330121011408195 --output=json diff --git a/doc/ref/runners/index.rst b/doc/ref/runners/index.rst index 4d38a3ef2a03..92f07c26737d 100644 --- a/doc/ref/runners/index.rst +++ b/doc/ref/runners/index.rst @@ -122,6 +122,6 @@ responding to Salt calls could look like this: Print a list of all of the minions that are up ''' client = salt.client.LocalClient(__opts__['conf_file']) - minions = client.cmd('*', 'test.ping', timeout=1) + minions = client.cmd('*', 'test.version', timeout=1) for minion in sorted(minions): print minion diff --git a/doc/topics/blackout/index.rst b/doc/topics/blackout/index.rst index 2ddf9e82baf8..c1b41b334b36 100644 --- a/doc/topics/blackout/index.rst +++ b/doc/topics/blackout/index.rst @@ -22,5 +22,5 @@ allowed during blackout. This is configured with the special pillar key .. code-block:: yaml minion_blackout_whitelist: - - test.ping - - pillar.get + - test.version + - pillar.get diff --git a/doc/topics/cloud/aws.rst b/doc/topics/cloud/aws.rst index 376edc6115a8..ba7ea8a166b3 100644 --- a/doc/topics/cloud/aws.rst +++ b/doc/topics/cloud/aws.rst @@ -324,7 +324,7 @@ it can be verified with Salt: .. code-block:: bash - # salt 'ami.example.com' test.ping + # salt 'ami.example.com' test.version Required Settings diff --git a/doc/topics/cloud/azure.rst b/doc/topics/cloud/azure.rst index 3066c39d6ee3..0426668f3cec 100644 --- a/doc/topics/cloud/azure.rst +++ b/doc/topics/cloud/azure.rst @@ -120,7 +120,7 @@ it can be verified with Salt: .. code-block:: bash - salt newinstance test.ping + salt newinstance test.version Profile Options diff --git a/doc/topics/cloud/azurearm.rst b/doc/topics/cloud/azurearm.rst index f77895475a24..481e77edf7c3 100644 --- a/doc/topics/cloud/azurearm.rst +++ b/doc/topics/cloud/azurearm.rst @@ -121,7 +121,7 @@ it can be verified with Salt: .. code-block:: bash - salt newinstance test.ping + salt newinstance test.version Profile Options diff --git a/doc/topics/cloud/deploy.rst b/doc/topics/cloud/deploy.rst index 4a75d36c72aa..c84e2a449d0d 100644 --- a/doc/topics/cloud/deploy.rst +++ b/doc/topics/cloud/deploy.rst @@ -123,7 +123,7 @@ Post-Deploy Commands Once a minion has been deployed, it has the option to run a salt command. Normally, this would be the :py:func:`state.apply `, which would finish provisioning the VM. Another common option (for testing) is -to use :py:func:`test.ping `. This is configured in the +to use :py:func:`test.version `. This is configured in the main cloud config file: .. code-block:: yaml diff --git a/doc/topics/cloud/gce.rst b/doc/topics/cloud/gce.rst index 58efe8196082..2d52ef1ee7a8 100644 --- a/doc/topics/cloud/gce.rst +++ b/doc/topics/cloud/gce.rst @@ -170,7 +170,7 @@ it can be verified with Salt: .. code-block:: bash - salt gce-instance test.ping + salt gce-instance test.version GCE Specific Settings diff --git a/doc/topics/cloud/libvirt.rst b/doc/topics/cloud/libvirt.rst index 79e88831c28f..33099b0b334a 100644 --- a/doc/topics/cloud/libvirt.rst +++ b/doc/topics/cloud/libvirt.rst @@ -84,7 +84,7 @@ it can be verified with Salt: .. code-block:: bash - # salt my-centos7-clone test.ping + # salt my-centos7-clone test.version Required Settings diff --git a/doc/topics/cloud/linode.rst b/doc/topics/cloud/linode.rst index c0ce064a0788..093100c28336 100644 --- a/doc/topics/cloud/linode.rst +++ b/doc/topics/cloud/linode.rst @@ -62,7 +62,7 @@ it can be verified with Salt: .. code-block:: bash - salt linode-instance test.ping + salt linode-instance test.version Listing Sizes diff --git a/doc/topics/cloud/opennebula.rst b/doc/topics/cloud/opennebula.rst index a453c475f2e8..cd081244fe5a 100644 --- a/doc/topics/cloud/opennebula.rst +++ b/doc/topics/cloud/opennebula.rst @@ -96,7 +96,7 @@ Once the instance has been created with salt-minion installed, connectivity to i .. code-block:: bash - salt my-new-vm test.ping + salt my-new-vm test.version OpenNebula uses an image --> template --> virtual machine paradigm where the template draws on the image, or disk, and virtual machines are created from templates. Because of this, there is no need to define a ``size`` in the cloud diff --git a/doc/topics/cloud/parallels.rst b/doc/topics/cloud/parallels.rst index 9031d4b3a65b..33a2b3a4ec47 100644 --- a/doc/topics/cloud/parallels.rst +++ b/doc/topics/cloud/parallels.rst @@ -91,7 +91,7 @@ it can be verified with Salt: .. code-block:: bash - # salt myubuntu test.ping + # salt myubuntu test.version Required Settings diff --git a/doc/topics/cloud/proxmox.rst b/doc/topics/cloud/proxmox.rst index 643b6a7fbecd..c790e462d6d7 100644 --- a/doc/topics/cloud/proxmox.rst +++ b/doc/topics/cloud/proxmox.rst @@ -91,7 +91,7 @@ it can be verified with Salt: .. code-block:: bash - # salt myubuntu test.ping + # salt myubuntu test.version Required Settings diff --git a/doc/topics/cloud/saltify.rst b/doc/topics/cloud/saltify.rst index aed77baec8fe..eb04cc5bae2e 100644 --- a/doc/topics/cloud/saltify.rst +++ b/doc/topics/cloud/saltify.rst @@ -195,7 +195,7 @@ Connectivity to the new "Salted" instances can now be verified with Salt: .. code-block:: bash - salt 'my-instance-*' test.ping + salt 'my-instance-*' test.version Credential Verification ======================= @@ -203,7 +203,7 @@ Credential Verification Because the Saltify driver does not actually create VM's, unlike other salt-cloud drivers, it has special behaviour when the ``deploy`` option is set to ``False``. When the cloud configuration specifies ``deploy: False``, the -Saltify driver will attept to authenticate to the target node(s) and return +Saltify driver will attempt to authenticate to the target node(s) and return ``True`` for each one that succeeds. This can be useful to verify ports, protocols, services and credentials are correctly configured before a live deployment. diff --git a/doc/topics/cloud/softlayer.rst b/doc/topics/cloud/softlayer.rst index d073f39b9823..8283789b8e32 100644 --- a/doc/topics/cloud/softlayer.rst +++ b/doc/topics/cloud/softlayer.rst @@ -221,9 +221,9 @@ with its short hostname, ``my-vm``): Rejected Keys: # # - # salt my-vm.example.com test.ping + # salt my-vm.example.com test.version my-vm.example.com: - True + 2018.3.4 # # # salt-cloud -d my-vm.example.com @@ -334,7 +334,7 @@ it can be verified with Salt: .. code-block:: bash - # salt 'myserver.example.com' test.ping + # salt 'myserver.example.com' test.version Dedicated Host ~~~~~~~~~~~~~~ diff --git a/doc/topics/cloud/vagrant.rst b/doc/topics/cloud/vagrant.rst index e1905396288e..790c7140ae5f 100644 --- a/doc/topics/cloud/vagrant.rst +++ b/doc/topics/cloud/vagrant.rst @@ -119,7 +119,7 @@ to it can be verified with Salt: .. code-block:: bash - salt my-id test.ping + salt my-id test.version .. _host provisioning example: diff --git a/doc/topics/cloud/vexxhost.rst b/doc/topics/cloud/vexxhost.rst index 98054e10641e..f672ead61a6e 100644 --- a/doc/topics/cloud/vexxhost.rst +++ b/doc/topics/cloud/vexxhost.rst @@ -111,7 +111,7 @@ the following command: .. code-block:: bash - # salt vh_instance1 test.ping + # salt vh_instance1 test.version You can now continue to provision new instances and they will all automatically be set up as minions of the master you've defined in the configuration file. diff --git a/doc/topics/cloud/xen.rst b/doc/topics/cloud/xen.rst index 85f464b95e26..dd87e19169ff 100644 --- a/doc/topics/cloud/xen.rst +++ b/doc/topics/cloud/xen.rst @@ -160,7 +160,7 @@ it can be verified with Salt: .. code-block:: bash - salt xenvm02 test.ping + salt xenvm02 test.version Listing Sizes diff --git a/doc/topics/development/architecture.rst b/doc/topics/development/architecture.rst index 85f7695e22b2..1c717092f830 100644 --- a/doc/topics/development/architecture.rst +++ b/doc/topics/development/architecture.rst @@ -109,7 +109,7 @@ against the command target. The typical lifecycle of a salt job from the perspective of the master might be as follows: -1) A command is issued on the CLI. For example, 'salt my_minion test.ping'. +1) A command is issued on the CLI. For example, 'salt my_minion test.version'. 2) The 'salt' command uses LocalClient to generate a request to the salt master by connecting to the ReqServer on TCP:4506 and issuing the job. diff --git a/doc/topics/development/hacking.rst b/doc/topics/development/hacking.rst index 3631134c9c54..0037cffe878a 100644 --- a/doc/topics/development/hacking.rst +++ b/doc/topics/development/hacking.rst @@ -160,7 +160,7 @@ installation is working: salt-minion -c ./etc/salt -d salt-key -c ./etc/salt -L salt-key -c ./etc/salt -A - salt -c ./etc/salt '*' test.ping + salt -c ./etc/salt '*' test.version Running the master and minion in debug mode can be helpful when developing. To do this, add ``-l debug`` to the calls to ``salt-master`` and ``salt-minion``. diff --git a/doc/topics/eauth/access_control.rst b/doc/topics/eauth/access_control.rst index 6c0373b707c8..76f8149a2630 100644 --- a/doc/topics/eauth/access_control.rst +++ b/doc/topics/eauth/access_control.rst @@ -67,7 +67,7 @@ other minions based on standard targets (all matchers are supported except the c external_auth: pam: dave: - - test.ping + - test.version - mongo\*: - network.* - log\*: @@ -78,7 +78,7 @@ other minions based on standard targets (all matchers are supported except the c steve: - .* -The above allows for all minions to be hit by test.ping by dave, and adds a +The above allows for all minions to be hit by test.version by dave, and adds a few functions that dave can execute on other minions. It also allows steve unrestricted access to salt commands. diff --git a/doc/topics/eauth/index.rst b/doc/topics/eauth/index.rst index 39249ba7b353..623656f4f70e 100644 --- a/doc/topics/eauth/index.rst +++ b/doc/topics/eauth/index.rst @@ -180,7 +180,7 @@ any user on the same system as the master with the ``-a`` option: .. code-block:: bash - $ salt -a pam web\* test.ping + $ salt -a pam web\* test.version The system will ask the user for the credentials required by the authentication system and then publish the command. @@ -198,7 +198,7 @@ adding a ``-T`` option when authenticating: .. code-block:: bash - $ salt -T -a pam web\* test.ping + $ salt -T -a pam web\* test.version Now a token will be created that has an expiration of 12 hours (by default). This token is stored in a file named ``salt_token`` in the active user's home diff --git a/doc/topics/event/master_events.rst b/doc/topics/event/master_events.rst index 1c11928224e9..420021faa486 100644 --- a/doc/topics/event/master_events.rst +++ b/doc/topics/event/master_events.rst @@ -69,7 +69,7 @@ Job events ``G@os_family:RedHat``, etc. :var tgt_type: The type of targeting used: ``glob``, ``grain``, ``compound``, etc. - :var fun: The function to run on minions: ``test.ping``, + :var fun: The function to run on minions: ``test.version``, ``network.interfaces``, etc. :var arg: A list of arguments to pass to the function that will be called. @@ -85,7 +85,7 @@ Job events :var id: The minion ID. :var jid: The job ID. :var retcode: The return code for the job. - :var fun: The function the minion ran. E.g., ``test.ping``. + :var fun: The function the minion ran. E.g., ``test.version``. :var return: The data returned from the execution module. .. salt:event:: salt/job//prog// diff --git a/doc/topics/installation/index.rst b/doc/topics/installation/index.rst index cbcbb5b93433..3261c184f790 100644 --- a/doc/topics/installation/index.rst +++ b/doc/topics/installation/index.rst @@ -25,12 +25,12 @@ The general installation process is as follows: 4. Accept the Salt :ref:`minion keys ` after the Salt minion connects. -After this, you should be able to run a simple command and receive returns from +After this, you should be able to run a simple command and receive salt version returns from all connected Salt minions. .. code-block:: bash - salt '*' test.ping + salt '*' test.version Quick Install ------------- diff --git a/doc/topics/installation/windows.rst b/doc/topics/installation/windows.rst index f100fdff9d38..fef6d7240d89 100644 --- a/doc/topics/installation/windows.rst +++ b/doc/topics/installation/windows.rst @@ -516,7 +516,7 @@ Testing the Salt minion .. code-block:: bash - sudo salt '*' test.ping + sudo salt '*' test.version You should get the following response: ``{'your minion hostname': True}`` diff --git a/doc/topics/proxyminion/demo.rst b/doc/topics/proxyminion/demo.rst index e72cc28e5603..87c585e484b2 100644 --- a/doc/topics/proxyminion/demo.rst +++ b/doc/topics/proxyminion/demo.rst @@ -98,7 +98,7 @@ the 'url' key above should say ``url: http://127.0.0.1:8000`` .. code-block:: bash - salt p8000 test.ping + salt p8000 test.version 8. The REST service implements a degenerately simple pkg and service provider as well as a small set of grains. To "install" a package, use a standard diff --git a/doc/topics/ssh/index.rst b/doc/topics/ssh/index.rst index 95a2bdecdbfa..f931f4e149db 100644 --- a/doc/topics/ssh/index.rst +++ b/doc/topics/ssh/index.rst @@ -127,7 +127,7 @@ command: .. code-block:: bash - salt-ssh '*' test.ping + salt-ssh '*' test.version Commands with ``salt-ssh`` follow the same syntax as the ``salt`` command. @@ -217,8 +217,8 @@ YAML contents: ssh_wipe: True Instead of having to call -``salt-ssh --config-dir=path/to/config/dir --max-procs=30 --wipe \* test.ping`` you -can call ``salt-ssh \* test.ping``. +``salt-ssh --config-dir=path/to/config/dir --max-procs=30 --wipe \* test.version`` you +can call ``salt-ssh \* test.version``. Boolean-style options should be specified in their YAML representation. diff --git a/doc/topics/targeting/batch.rst b/doc/topics/targeting/batch.rst index f8e0aae7936d..41a6f2e26a1e 100644 --- a/doc/topics/targeting/batch.rst +++ b/doc/topics/targeting/batch.rst @@ -9,11 +9,11 @@ supported. .. code-block:: bash - salt '*' -b 10 test.ping + salt '*' -b 10 test.version salt -G 'os:RedHat' --batch-size 25% apache.signal restart -This will only run test.ping on 10 of the targeted minions at a time and then +This will only run test.version on 10 of the targeted minions at a time and then restart apache on 25% of the minions matching ``os:RedHat`` at a time and work through them all until the task is complete. This makes jobs like rolling web server restarts behind a load balancer or doing maintenance on BSD firewalls diff --git a/doc/topics/targeting/compound.rst b/doc/topics/targeting/compound.rst index 59fa619b4c14..cf91b578bdf8 100644 --- a/doc/topics/targeting/compound.rst +++ b/doc/topics/targeting/compound.rst @@ -31,7 +31,7 @@ matches the :mod:`regular expression ` ``web-dc1-srv.*``: .. code-block:: bash - salt -C 'webserv* and G@os:Debian or E@web-dc1-srv.*' test.ping + salt -C 'webserv* and G@os:Debian or E@web-dc1-srv.*' test.version That same example expressed in a :term:`top file` looks like the following: @@ -48,20 +48,20 @@ Excluding a minion based on its ID is also possible: .. code-block:: bash - salt -C 'not web-dc1-srv' test.ping + salt -C 'not web-dc1-srv' test.version Versions prior to 2015.8.0 a leading ``not`` was not supported in compound matches. Instead, something like the following was required: .. code-block:: bash - salt -C '* and not G@kernel:Darwin' test.ping + salt -C '* and not G@kernel:Darwin' test.version Excluding a minion based on its ID was also possible: .. code-block:: bash - salt -C '* and not web-dc1-srv' test.ping + salt -C '* and not web-dc1-srv' test.version Precedence Matching ------------------- @@ -70,7 +70,7 @@ Matchers can be grouped together with parentheses to explicitly declare preceden .. code-block:: bash - salt -C '( ms-1 or G@id:ms-3 ) and G@id:ms-3' test.ping + salt -C '( ms-1 or G@id:ms-3 ) and G@id:ms-3' test.version .. note:: diff --git a/doc/topics/targeting/globbing.rst b/doc/topics/targeting/globbing.rst index 5ced1962f965..fd59f0bfcf97 100644 --- a/doc/topics/targeting/globbing.rst +++ b/doc/topics/targeting/globbing.rst @@ -31,39 +31,39 @@ Match all minions: .. code-block:: bash - salt '*' test.ping + salt '*' test.version Match all minions in the example.net domain or any of the example domains: .. code-block:: bash - salt '*.example.net' test.ping - salt '*.example.*' test.ping + salt '*.example.net' test.version + salt '*.example.*' test.version Match all the ``webN`` minions in the example.net domain (``web1.example.net``, ``web2.example.net`` … ``webN.example.net``): .. code-block:: bash - salt 'web?.example.net' test.ping + salt 'web?.example.net' test.version Match the ``web1`` through ``web5`` minions: .. code-block:: bash - salt 'web[1-5]' test.ping + salt 'web[1-5]' test.version Match the ``web1`` and ``web3`` minions: .. code-block:: bash - salt 'web[1,3]' test.ping + salt 'web[1,3]' test.version Match the ``web-x``, ``web-y``, and ``web-z`` minions: .. code-block:: bash - salt 'web-[x-z]' test.ping + salt 'web-[x-z]' test.version .. note:: @@ -81,7 +81,7 @@ Match both ``web1-prod`` and ``web1-devel`` minions: .. code-block:: bash - salt -E 'web1-(prod|devel)' test.ping + salt -E 'web1-(prod|devel)' test.version When using regular expressions in a State's :term:`top file`, you must specify the matcher as the first option. The following example executes the contents of @@ -102,4 +102,4 @@ At the most basic level, you can specify a flat list of minion IDs: .. code-block:: bash - salt -L 'web1,web2,web3' test.ping \ No newline at end of file + salt -L 'web1,web2,web3' test.version diff --git a/doc/topics/targeting/grains.rst b/doc/topics/targeting/grains.rst index 6f081f9c4835..c38adabb3a13 100644 --- a/doc/topics/targeting/grains.rst +++ b/doc/topics/targeting/grains.rst @@ -10,7 +10,7 @@ For example, the following matches all CentOS minions: .. code-block:: bash - salt -G 'os:CentOS' test.ping + salt -G 'os:CentOS' test.version Match all minions with 64-bit CPUs, and return number of CPU cores for each matching minion: diff --git a/doc/topics/targeting/index.rst b/doc/topics/targeting/index.rst index 4c3ff964a38f..ffdf47b0bcce 100644 --- a/doc/topics/targeting/index.rst +++ b/doc/topics/targeting/index.rst @@ -37,7 +37,7 @@ the target is the grain key followed by a glob expression: "os:Arch*". .. code-block:: bash - salt -G 'os:Fedora' test.ping + salt -G 'os:Fedora' test.version Will return True from all of the minions running Fedora. @@ -62,7 +62,7 @@ This is well defined with an example: .. code-block:: bash - salt -C 'G@os:Debian and webser* or E@db.*' test.ping + salt -C 'G@os:Debian and webser* or E@db.*' test.version In this example any minion who's id starts with ``webser`` and is running Debian, or any minion who's id starts with db will be matched. diff --git a/doc/topics/targeting/ipcidr.rst b/doc/topics/targeting/ipcidr.rst index 40247bc246d6..2adf0f6f773d 100644 --- a/doc/topics/targeting/ipcidr.rst +++ b/doc/topics/targeting/ipcidr.rst @@ -9,14 +9,14 @@ notation). .. code-block:: bash - salt -S 192.168.40.20 test.ping - salt -S 2001:db8::/64 test.ping + salt -S 192.168.40.20 test.version + salt -S 2001:db8::/64 test.version Ipcidr matching can also be used in compound matches .. code-block:: bash - salt -C 'S@10.0.0.0/24 and G@os:Debian' test.ping + salt -C 'S@10.0.0.0/24 and G@os:Debian' test.version It is also possible to use in both pillar and state-matching diff --git a/doc/topics/targeting/nodegroups.rst b/doc/topics/targeting/nodegroups.rst index 6043d959af19..a4cfacca55e2 100644 --- a/doc/topics/targeting/nodegroups.rst +++ b/doc/topics/targeting/nodegroups.rst @@ -57,7 +57,7 @@ To match a nodegroup on the CLI, use the ``-N`` command-line option: .. code-block:: bash - salt -N group1 test.ping + salt -N group1 test.version .. note:: diff --git a/doc/topics/targeting/pillar.rst b/doc/topics/targeting/pillar.rst index a86b0bd8f2e4..ab79cf535a08 100644 --- a/doc/topics/targeting/pillar.rst +++ b/doc/topics/targeting/pillar.rst @@ -21,7 +21,7 @@ Example: .. code-block:: bash - salt -I 'somekey:specialvalue' test.ping + salt -I 'somekey:specialvalue' test.version Like with :ref:`Grains `, it is possible to use globbing as well as match nested values in Pillar, by adding colons for each level that @@ -31,4 +31,4 @@ is being traversed. The below example would match minions with a pillar named .. code-block:: bash - salt -I 'foo:bar:baz*' test.ping + salt -I 'foo:bar:baz*' test.version diff --git a/doc/topics/targeting/range.rst b/doc/topics/targeting/range.rst index e5bd3f9dab91..b3780c60546d 100644 --- a/doc/topics/targeting/range.rst +++ b/doc/topics/targeting/range.rst @@ -72,11 +72,11 @@ One might target host1 through host100 in the test.com domain with Salt as follo .. code-block:: bash - salt --range %test:CLUSTER test.ping + salt --range %test:CLUSTER test.version The following salt command would target three hosts: ``frontend``, ``backend``, and ``mysql``: .. code-block:: bash - salt --range %test:APPS test.ping + salt --range %test:APPS test.version diff --git a/doc/topics/thorium/index.rst b/doc/topics/thorium/index.rst index f37d3edbf865..0eb602d7a09b 100644 --- a/doc/topics/thorium/index.rst +++ b/doc/topics/thorium/index.rst @@ -285,12 +285,12 @@ event bus, and returns ``True`` if that event's tag matches. For example: run_remote_ex: local.cmd: - tgt: '*' - - func: test.ping + - func: test.version - require: - check: salt/foo/*/bar This formula will look for an event whose tag is ``salt/foo//bar`` and -if it comes in, issue a ``test.ping`` to all minions. +if it comes in, issue a ``test.version`` to all minions. Register Persistence diff --git a/doc/topics/topology/syndic.rst b/doc/topics/topology/syndic.rst index da598343f11c..f51cdf1fe61e 100644 --- a/doc/topics/topology/syndic.rst +++ b/doc/topics/topology/syndic.rst @@ -131,15 +131,15 @@ On the Master node: Unaccepted Keys: Rejected Keys: - # salt '*' test.ping + # salt '*' test.version minion_1: - True + 2018.3.4 minion_2: - True + 2018.3.4 minion_4: - True + 2018.3.4 minion_3: - True + 2018.3.4 Topology ======== diff --git a/doc/topics/tutorials/cloud_controller.rst b/doc/topics/tutorials/cloud_controller.rst index 74dfe813195b..6bbdd81764bd 100644 --- a/doc/topics/tutorials/cloud_controller.rst +++ b/doc/topics/tutorials/cloud_controller.rst @@ -277,7 +277,7 @@ This command will return data about all of the hypervisors and respective virtual machines. Now that the new VM is booted it should have contacted the Salt Master, a -``test.ping`` will reveal if the new VM is running. +``test.version`` will reveal if the new VM is running. QEMU copy on write support diff --git a/doc/topics/tutorials/docker_sls.rst b/doc/topics/tutorials/docker_sls.rst index c7e3762f10bc..0e075cb8240a 100644 --- a/doc/topics/tutorials/docker_sls.rst +++ b/doc/topics/tutorials/docker_sls.rst @@ -81,7 +81,7 @@ simple `salt-call` command: .. code-block:: bash - salt-call --local dockerng.call test test.ping + salt-call --local dockerng.call test test.version salt-call --local dockerng.call test network.interfaces salt-call --local dockerng.call test disk.usage salt-call --local dockerng.call test pkg.list_pkgs diff --git a/doc/topics/tutorials/esxi_proxy_minion.rst b/doc/topics/tutorials/esxi_proxy_minion.rst index bce78b5f41db..3cbe2d847e8d 100644 --- a/doc/topics/tutorials/esxi_proxy_minion.rst +++ b/doc/topics/tutorials/esxi_proxy_minion.rst @@ -227,9 +227,10 @@ This allows you to use any number of potential fallback passwords. This scenario is especially true, and even slower, when the proxy minion first starts. If the correct password is not the first password - on the list, it may take up to a minute for ``test.ping`` to respond - with a ``True`` result. Once the initial authorization is complete, the - responses for commands will be a little faster. + on the list, it may take up to a minute for ``test.version`` to respond + with salt's version installed (Example: ``2018.3.4``. Once the initial + authorization is complete, the responses for commands will be a little + faster. To avoid these longer waiting periods, SaltStack recommends moving the correct password to the top of the list and restarting the proxy minion @@ -366,7 +367,7 @@ proxy processes! .. code-block:: bash - # salt 'esxi-*' test.ping + # salt 'esxi-*' test.version esxi-1: True esxi-3: @@ -377,7 +378,7 @@ Executing Commands ================== Now that you've configured your Proxy Minions and have them responding successfully -to a ``test.ping``, we can start executing commands against the ESXi hosts via Salt. +to a ``test.version``, we can start executing commands against the ESXi hosts via Salt. It's important to understand how this particular proxy works, and there are a couple of important pieces to be aware of in order to start running remote execution and diff --git a/doc/topics/tutorials/firewall.rst b/doc/topics/tutorials/firewall.rst index 31fb3c84189b..eaa476c51ce5 100644 --- a/doc/topics/tutorials/firewall.rst +++ b/doc/topics/tutorials/firewall.rst @@ -231,5 +231,5 @@ be set on the Master: needs to communicate with the listening network socket of ``salt-master`` on the *loopback* interface. Without this you will see no outgoing Salt traffic from the master, even for a simple - ``salt '*' test.ping``, because the ``salt`` client never reached + ``salt '*' test.version``, because the ``salt`` client never reached the ``salt-master`` to tell it to carry out the execution. diff --git a/doc/topics/tutorials/modules.rst b/doc/topics/tutorials/modules.rst index e1dbde7d764f..4d6e03b3c978 100644 --- a/doc/topics/tutorials/modules.rst +++ b/doc/topics/tutorials/modules.rst @@ -27,14 +27,14 @@ following function. The default filter is a glob on the minion id. For example: .. code-block:: bash - salt '*' test.ping - salt '*.example.org' test.ping + salt '*' test.version + salt '*.example.org' test.version Targets can be based on minion system information using the Grains system: .. code-block:: bash - salt -G 'os:Ubuntu' test.ping + salt -G 'os:Ubuntu' test.version .. seealso:: :ref:`Grains system ` @@ -42,19 +42,19 @@ Targets can be filtered by regular expression: .. code-block:: bash - salt -E 'virtmach[0-9]' test.ping + salt -E 'virtmach[0-9]' test.version Targets can be explicitly specified in a list: .. code-block:: bash - salt -L 'foo,bar,baz,quo' test.ping + salt -L 'foo,bar,baz,quo' test.version Or Multiple target types can be combined in one command: .. code-block:: bash - salt -C 'G@os:Ubuntu and webser* or E@database.*' test.ping + salt -C 'G@os:Ubuntu and webser* or E@database.*' test.version function @@ -74,7 +74,7 @@ Show all currently available minions: .. code-block:: bash - salt '*' test.ping + salt '*' test.version Run an arbitrary shell command: @@ -99,4 +99,4 @@ Optional, keyword arguments are also supported: salt '*' pip.install salt timeout=5 upgrade=True -They are always in the form of ``kwarg=argument``. \ No newline at end of file +They are always in the form of ``kwarg=argument``. diff --git a/doc/topics/tutorials/multimaster_pki.rst b/doc/topics/tutorials/multimaster_pki.rst index 0dc9dbb5b94f..b647c88789b2 100644 --- a/doc/topics/tutorials/multimaster_pki.rst +++ b/doc/topics/tutorials/multimaster_pki.rst @@ -238,7 +238,7 @@ The minion will connect to the first master from its master list [DEBUG ] Decrypting the current master AES key -A test.ping on the master the minion is currently connected to should be run to +A test.version on the master the minion is currently connected to should be run to test connectivity. If successful, that master should be turned off. A firewall-rule denying the diff --git a/doc/topics/tutorials/walkthrough.rst b/doc/topics/tutorials/walkthrough.rst index 9f691c5cc08b..62be2f04f071 100644 --- a/doc/topics/tutorials/walkthrough.rst +++ b/doc/topics/tutorials/walkthrough.rst @@ -238,16 +238,16 @@ start with looks like this: .. code-block:: bash - salt '*' test.ping + salt '*' test.version The ``*`` is the target, which specifies all minions. -``test.ping`` tells the minion to run the :py:func:`test.ping -` function. +``test.version`` tells the minion to run the :py:func:`test.version +` function. -In the case of ``test.ping``, ``test`` refers to a :ref:`execution module -`. ``ping`` refers to the :py:func:`ping -` function contained in the aforementioned ``test`` +In the case of ``test.version``, ``test`` refers to a :ref:`execution module +`. ``version`` refers to the :py:func:`version +` function contained in the aforementioned ``test`` module. .. note:: @@ -257,12 +257,10 @@ module. services. The result of running this command will be the master instructing all of the -minions to execute :py:func:`test.ping ` in parallel -and return the result. - -This is not an actual ICMP ping, but rather a simple function which returns ``True``. -Using :py:func:`test.ping ` is a good way of confirming that a minion is -connected. +minions to execute :py:func:`test.version ` in parallel +and return the result. Using :py:func:`test.version ` +is a good way of confirming that a minion is connected, and reaffirm to the user +the salt version(s) they have installed on the minions. .. note:: @@ -271,7 +269,7 @@ connected. well by using the :conf_minion:`id` parameter. Of course, there are hundreds of other modules that can be called just as -``test.ping`` can. For example, the following would return disk usage on all +``test.version`` can. For example, the following would return disk usage on all targeted minions: .. code-block:: bash @@ -591,7 +589,7 @@ This formula can be referenced via the following command: .. note:: :py:func:`state.apply ` is just another remote - execution function, just like :py:func:`test.ping ` + execution function, just like :py:func:`test.version ` or :py:func:`disk.usage `. It simply takes the name of an SLS file as an argument. diff --git a/doc/topics/tutorials/walkthrough_macosx.rst b/doc/topics/tutorials/walkthrough_macosx.rst index 00d6c31febda..f1e5de026a61 100644 --- a/doc/topics/tutorials/walkthrough_macosx.rst +++ b/doc/topics/tutorials/walkthrough_macosx.rst @@ -412,9 +412,9 @@ following: .. code-block:: bash - sudo salt '*' test.ping + sudo salt '*' test.version -You should see your minion answering the ping. It's now time to do some +You should see your minion answering with its salt version. It's now time to do some configuration. From e8c8dba161ffee6508571b55a994dcff73fbbece Mon Sep 17 00:00:00 2001 From: Alan Cugler Date: Mon, 25 Mar 2019 16:10:29 -0500 Subject: [PATCH 20/27] Added in an explaination of the --zone flag approved in #52251 --- doc/topics/tutorials/firewall.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/topics/tutorials/firewall.rst b/doc/topics/tutorials/firewall.rst index 31fb3c84189b..b704964488f5 100644 --- a/doc/topics/tutorials/firewall.rst +++ b/doc/topics/tutorials/firewall.rst @@ -28,8 +28,11 @@ FirewallD use the command line client ``firewall-cmd``. firewall-cmd --permanent --zone= --add-port=4505-4506/tcp -Please choose the desired zone according to your setup. Don't forget to reload -after you made your changes. +A network zone defines the security level of trust for the the network. +The user should choose an appropriate zone value for their setup. +Possible values include: drop, block, public, external, dmz, work, home, internal, trusted. + +Don't forget to reload after you made your changes. .. code-block:: bash From c04ba6e35efaf3b4f9646e24b2716aacc2f1fe17 Mon Sep 17 00:00:00 2001 From: Alan Cugler Date: Tue, 26 Mar 2019 09:58:26 -0500 Subject: [PATCH 21/27] Changed a text artifact from the ping function to supporting text for the version function. --- doc/ref/configuration/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/ref/configuration/index.rst b/doc/ref/configuration/index.rst index b54d7053d53f..b5098a4978ba 100644 --- a/doc/ref/configuration/index.rst +++ b/doc/ref/configuration/index.rst @@ -222,7 +222,8 @@ similar way: delta: 2018.3.4 -Each of the Minions should send a ``True`` response as shown above. +Each of the Minions should send a ``2018.3.4`` response as shown above, +or any other salt version installed. What's Next? ============ From 8255901b33f0c782459223f36c81552a594ed8f4 Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Wed, 27 Mar 2019 12:14:07 -0400 Subject: [PATCH 22/27] document `regex_replace` Jinja filter (#52326) Fixes #48952. --- doc/topics/jinja/index.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 5f2b46d56a44..011cbaeac4eb 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -405,6 +405,29 @@ Returns: None +.. jinja_ref:: regex_replace + +``regex_replace`` +----------------- + +.. versionadded:: 2017.7.0 + +Searches for a pattern and replaces with a sequence of characters. + +Example: + +.. code-block:: jinja + + {% set my_text = 'yes, this is a TEST' %} + {{ my_text | regex_replace(' ([a-z])', '__\\1', ignorecase=True) }} + +Returns: + +.. code-block:: text + + yes,__this__is__a__TEST + + .. jinja_ref:: uuid ``uuid`` From 364ef065e8d4dd89942e764558378ee4243861dd Mon Sep 17 00:00:00 2001 From: Alan Cugler Date: Thu, 21 Feb 2019 13:52:28 -0600 Subject: [PATCH 23/27] Added mention of map file alternative for minion configuration options. Fixes: #51127 --- doc/topics/cloud/config.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/topics/cloud/config.rst b/doc/topics/cloud/config.rst index 1d124fb95e8f..f50593e8c88f 100644 --- a/doc/topics/cloud/config.rst +++ b/doc/topics/cloud/config.rst @@ -27,10 +27,12 @@ cloud is operating on. Minion Configuration ==================== -The default minion configuration is set up in this file. Minions created by -salt-cloud derive their configuration from this file. Almost all parameters -found in :ref:`Configuring the Salt Minion ` can -be used here. +The default minion configuration is set up in this file or alternatively used in +any ``.conf`` file listed in the ``/etc/salt/cloud.maps.d/`` +:ref:`map file ` directory. Minions created by salt-cloud +traditionally derive their configuration from this file. Almost all parameters +found in :ref:`Configuring the Salt Minion ` can be +used here. .. code-block:: yaml From 9ca0f72308a431c4d8120dfa6128413aed0d9f04 Mon Sep 17 00:00:00 2001 From: Alan Cugler Date: Wed, 27 Mar 2019 12:26:52 -0500 Subject: [PATCH 24/27] Changed where to mention the map files compatibility for minion configuration options. This is a result of @Ch3LL suggestion on the PR itself for this doc change. --- doc/topics/cloud/config.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/topics/cloud/config.rst b/doc/topics/cloud/config.rst index f50593e8c88f..ffe619049f7d 100644 --- a/doc/topics/cloud/config.rst +++ b/doc/topics/cloud/config.rst @@ -27,10 +27,8 @@ cloud is operating on. Minion Configuration ==================== -The default minion configuration is set up in this file or alternatively used in -any ``.conf`` file listed in the ``/etc/salt/cloud.maps.d/`` -:ref:`map file ` directory. Minions created by salt-cloud -traditionally derive their configuration from this file. Almost all parameters +The default minion configuration is set up in this file. Minions created by +salt-cloud derive their configuration from this file. Almost all parameters found in :ref:`Configuring the Salt Minion ` can be used here. @@ -46,7 +44,7 @@ and its listening port, if the port is not set to the default. Similar to most other settings, Minion configuration settings are inherited across configuration files. For example, the master setting might be contained in the main ``cloud`` configuration file as demonstrated above, but additional -settings can be placed in the provider or profile: +settings can be placed in the provider, profile or map configuration files: .. code-block:: yaml From 23733bde5ada770213fbb122d041fd8b02cdb28c Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 27 Mar 2019 23:33:10 +0300 Subject: [PATCH 25/27] Regression test for parallel IPCMessageSubscriber support --- tests/unit/transport/test_ipc.py | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/unit/transport/test_ipc.py b/tests/unit/transport/test_ipc.py index 942ac8fc6c65..3492be10649b 100644 --- a/tests/unit/transport/test_ipc.py +++ b/tests/unit/transport/test_ipc.py @@ -29,6 +29,9 @@ from tests.support.paths import TMP from tests.support.unit import skipIf +import pytest +import threading + log = logging.getLogger(__name__) @@ -154,3 +157,85 @@ def test_multistream_errors(self): self.channel.send({'stop': True}) self.wait() self.assertEqual(self.payloads[:-1], [None, None, 'foo', 'foo']) + + +@skipIf(salt.utils.platform.is_windows(), 'Windows does not support Posix IPC') +class IPCMessagePubSubCase(tornado.testing.AsyncTestCase): + ''' + Test all of the clear msg stuff + ''' + def setUp(self): + super(IPCMessagePubSubCase, self).setUp() + self.opts = {'ipc_write_buffer': 0} + self.socket_path = os.path.join(TMP, 'ipc_test.ipc') + self.pub_channel = self._get_pub_channel() + self.sub_channel = self._get_sub_channel() + + def _get_pub_channel(self): + pub_channel = salt.transport.ipc.IPCMessagePublisher( + self.opts, + self.socket_path, + ) + pub_channel.start() + return pub_channel + + def _get_sub_channel(self): + sub_channel = salt.transport.ipc.IPCMessageSubscriber( + socket_path=self.socket_path, + io_loop=self.io_loop, + ) + sub_channel.connect(callback=self.stop) + self.wait() + return sub_channel + + def tearDown(self): + super(IPCMessagePubSubCase, self).tearDown() + try: + self.pub_channel.close() + except socket.error as exc: + if exc.errno != errno.EBADF: + # If its not a bad file descriptor error, raise + raise + try: + self.sub_channel.close() + except socket.error as exc: + if exc.errno != errno.EBADF: + # If its not a bad file descriptor error, raise + raise + os.unlink(self.socket_path) + del self.pub_channel + del self.sub_channel + + def test_multi_client_reading(self): + # To be completely fair let's create 2 clients. + client1 = self.sub_channel + client2 = self._get_sub_channel() + call_cnt = [] + + # Create a watchdog to be safe from hanging in sync loops (what old code did) + evt = threading.Event() + + def close_server(): + if evt.wait(1): + return + client2.close() + self.stop() + + watchdog = threading.Thread(target=close_server) + watchdog.start() + + # Runs in ioloop thread so we're safe from race conditions here + def handler(raw): + call_cnt.append(raw) + if len(call_cnt) >= 2: + evt.set() + self.stop() + + # Now let both waiting data at once + client1.read_async(handler) + client2.read_async(handler) + self.pub_channel.publish('TEST') + self.wait() + self.assertEqual(len(call_cnt), 2) + self.assertEqual(call_cnt[0], 'TEST') + self.assertEqual(call_cnt[1], 'TEST') From 37aeba314330a5cefdf9ca1d5ce069bc790e692f Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 27 Mar 2019 23:39:00 +0300 Subject: [PATCH 26/27] Minor: Fix typo in docstring --- salt/transport/ipc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index ee3c5d7c43c3..30e340c0b8b8 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -589,7 +589,7 @@ class IPCMessageSubscriberService(IPCClient): of IPCMessageSubscriber instances feeding all of them with data. It closes automatically when there are no more subscribers. - To use this rever to IPCMessageSubscriber documentation. + To use this refer to IPCMessageSubscriber documentation. ''' def __singleton_init__(self, socket_path, io_loop=None): super(IPCMessageSubscriberService, self).__singleton_init__( From 87bb513d6361150c6501e834f01f2e61706a6b3e Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Wed, 27 Mar 2019 14:58:57 -0600 Subject: [PATCH 27/27] Fix failing symlink test (#52145) --- tests/unit/modules/test_win_file.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/unit/modules/test_win_file.py b/tests/unit/modules/test_win_file.py index 3da7b45a6f40..8c7938843c30 100644 --- a/tests/unit/modules/test_win_file.py +++ b/tests/unit/modules/test_win_file.py @@ -313,14 +313,21 @@ def test_issue_52002_check_file_remove_symlink(self): ''' Make sure that directories including symlinks or symlinks can be removed ''' - base = temp.dir(prefix='base') - target = os.path.join(base, 'child 1', 'target/') + base = temp.dir(prefix='base-') + target = os.path.join(base, 'child 1', 'target\\') symlink = os.path.join(base, 'child 2', 'link') - self.assertFalse(win_file.directory_exists(target)) - self.assertFalse(win_file.directory_exists(symlink)) - self.assertTrue(win_file.makedirs_(target)) - self.assertTrue(win_file.directory_exists(symlink)) - self.assertTrue(win_file.symlink(target, symlink)) - self.assertTrue(win_file.is_link(symlink)) - self.assertTrue(win_file.remove(base)) - self.assertFalse(win_file.directory_exists(base)) + try: + # Create environment + self.assertFalse(win_file.directory_exists(target)) + self.assertFalse(win_file.directory_exists(symlink)) + self.assertTrue(win_file.makedirs_(target)) + self.assertTrue(win_file.makedirs_(symlink)) + self.assertTrue(win_file.symlink(target, symlink)) + self.assertTrue(win_file.directory_exists(symlink)) + self.assertTrue(win_file.is_link(symlink)) + # Test removal of directory containing symlink + self.assertTrue(win_file.remove(base)) + self.assertFalse(win_file.directory_exists(base)) + finally: + if os.path.exists(base): + win_file.remove(base)