[2019.2] Merge forward from 2019.2.1 to 2019.2

salt/modules/file.py

@@ -23,6 +23,7 @@
 import shutil
 import stat
 import string
+import subprocess
 import sys
 import tempfile
 import time
@@ -61,6 +62,7 @@
 import salt.utils.url
 import salt.utils.user
 import salt.utils.data
+import salt.utils.versions
 from salt.exceptions import CommandExecutionError, MinionError, SaltInvocationError, get_error_message as _get_error_message
 from salt.utils.files import HASHES, HASHES_REVMAP
 
@@ -159,6 +161,35 @@ def _splitlines_preserving_trailing_newline(str):
     return lines
 
 
+def _get_chattr_man():
+    '''
+    Get the contents of the chattr man page
+    '''
+    return subprocess.check_output(['man', 'chattr'])
+
+
+def _parse_chattr_man(man):
+    '''
+    Parse the contents of a chattr man page to find the E2fsprogs version
+    '''
+    match = re.search(
+        r'E2fsprogs version [0-9\.]+',
+        salt.utils.stringutils.to_str(man),
+    )
+    if match:
+        version = match.group().strip('E2fsprogs version ')
+    else:
+        version = None
+    return version
+
+
+def _chattr_version():
+    '''
+    Return the version of chattr installed
+    '''
+    return _parse_chattr_man(_get_chattr_man())
+
+
 def gid_to_group(gid):
     '''
     Convert the group id to the group name on this system
@@ -577,7 +608,14 @@ def lsattr(path):
     for line in result.splitlines():
         if not line.startswith('lsattr: '):
             vals = line.split(None, 1)
-            results[vals[1]] = re.findall(r"[aAcCdDeijPsStTu]", vals[0])
+            needed_version = salt.utils.versions.LooseVersion('1.41.12')
+            chattr_version = salt.utils.versions.LooseVersion(_chattr_version())
+            # The version of chattr on Centos 6 does not support extended
+            # attributes.
+            if chattr_version > needed_version:
+                results[vals[1]] = re.findall(r"[aAcCdDeijPsStTu]", vals[0])
+            else:
+                results[vals[1]] = re.findall(r"[acdijstuADST]", vals[0])
 
     return results
 

salt/modules/virt.py

@@ -3291,7 +3291,10 @@ def purge(vm_, dirs=False, removables=None, **kwargs):
             shutil.rmtree(dir_)
     if getattr(libvirt, 'VIR_DOMAIN_UNDEFINE_NVRAM', False):
         # This one is only in 1.2.8+
-        dom.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_NVRAM)
+        try:
+            dom.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_NVRAM)
+        except Exception:
+            dom.undefine()
     else:
         dom.undefine()
     conn.close()

salt/modules/x509.py

@@ -1391,10 +1391,13 @@ def create_certificate(
         for ignore in list(_STATE_INTERNAL_KEYWORDS) + \
                 ['listen_in', 'preqrequired', '__prerequired__']:
             kwargs.pop(ignore, None)
+        # TODO: Make timeout configurable in Neon
         certs = __salt__['publish.publish'](
             tgt=ca_server,
             fun='x509.sign_remote_certificate',
-            arg=salt.utils.data.decode_dict(kwargs, to_str=True))
+            arg=salt.utils.data.decode_dict(kwargs, to_str=True),
+            timeout=30
+        )
 
         if not any(certs):
             raise salt.exceptions.SaltInvocationError(

salt/utils/gitfs.py

@@ -3016,7 +3016,7 @@ def checkout(self):
                 if repo.branch == '__env__' and hasattr(repo, 'all_saltenvs'):
                     env = self.opts.get('pillarenv') \
                         or self.opts.get('saltenv') \
-                        or self.opts.get('git_pillar_base')
+                        or 'base'
                 elif repo.env:
                     env = repo.env
                 else:

tests/filename_map.yml

@@ -43,6 +43,9 @@ salt/modules/dockermod.py:
   - integration.states.test_docker_container
   - integration.states.test_docker_network
 
+salt/modules/file.py:
+  - integration.states.test_cron
+
 salt/modules/influxdb08mod.py:
   - unit.states.test_influxdb08_database
   - unit.states.test_influxdb08_user

tests/integration/files/conf/master

@@ -28,7 +28,7 @@ tcp_master_workers: 64515
 
 peer:
   '.*':
-    - 'test.*'
+    - '(x509|test).*'
 
 ext_pillar:
   - ext_pillar_opts:

tests/integration/files/engines/runtests_engine.py

@@ -56,11 +56,7 @@ def start(self):
 
     @gen.coroutine
     def _start(self):
-        if self.opts['__role'] == 'minion':
-            yield self.listen_to_minion_connected_event()
-        else:
-            self.io_loop.spawn_callback(self.fire_master_started_event)
-
+        self.io_loop.spawn_callback(self.fire_master_started_event)
         port = int(self.opts['runtests_conn_check_port'])
         log.info('Starting Pytest Engine(role=%s) on port %s', self.opts['__role'], port)
         self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -92,19 +88,6 @@ def handle_connection(self, connection, address):
                 # This is not macOS !?
                 pass
 
-    @gen.coroutine
-    def listen_to_minion_connected_event(self):
-        log.info('Listening for minion connected event...')
-        minion_start_event_match = 'salt/minion/{0}/start'.format(self.opts['id'])
-        event_bus = salt.utils.event.get_master_event(self.opts, self.opts['sock_dir'], listen=True)
-        event_bus.subscribe(minion_start_event_match)
-        while True:
-            event = event_bus.get_event(full=True, no_block=True)
-            if event is not None and event['tag'] == minion_start_event_match:
-                log.info('Got minion connected event: %s', event)
-                break
-            yield gen.sleep(0.25)
-
     @gen.coroutine
     def fire_master_started_event(self):
         log.info('Firing salt-master started event...')

tests/integration/modules/test_network.py

@@ -11,7 +11,7 @@
 import salt.utils.path
 import salt.utils.platform
 
-URL = 'repo.saltstack.com'
+URL = 'google-public-dns-a.google.com'
 
 
 class NetworkTest(ModuleCase):

tests/integration/pillar/test_git_pillar.py

@@ -488,6 +488,33 @@ def test_all_saltenvs(self):
                         'nested_dict': {'dev': True}}}
         )
 
+    def test_all_saltenvs_base(self):
+        '''
+        Test all_saltenvs parameter with base pillarenv.
+        '''
+        ret = self.get_pillar('''\
+            file_ignore_regex: []
+            file_ignore_glob: []
+            git_pillar_provider: gitpython
+            cachedir: {cachedir}
+            extension_modules: {extmods}
+            ext_pillar:
+              - git:
+                - __env__ {url_extra_repo}:
+                  - all_saltenvs: master
+                - __env__ {url}:
+                  - mountpoint: nowhere
+            ''')
+        self.assertEqual(
+            ret,
+            {'branch': 'master',
+             'motd': 'The force will be with you. Always.',
+             'mylist': ['master'],
+             'mydict': {'master': True,
+                        'nested_list': ['master'],
+                        'nested_dict': {'master': True}}}
+        )
+
 
 @destructiveTest
 @skipIf(NO_MOCK, NO_MOCK_REASON)
@@ -1742,6 +1769,103 @@ def test_all_saltenvs(self, grains):
             ''')
         self.assertEqual(ret, expected)
 
+    @requires_system_grains
+    def test_all_saltenvs_base(self, grains):
+        '''
+        Test all_saltenvs parameter.
+        '''
+        expected = {'branch': 'master',
+             'motd': 'The force will be with you. Always.',
+             'mylist': ['master'],
+             'mydict': {'master': True,
+                        'nested_list': ['master'],
+                        'nested_dict': {'master': True}
+                       }
+        }
+
+        # Test with passphraseless key and global credential options
+        ret = self.get_pillar('''\
+            file_ignore_regex: []
+            file_ignore_glob: []
+            git_pillar_provider: pygit2
+            git_pillar_pubkey: {pubkey_nopass}
+            git_pillar_privkey: {privkey_nopass}
+            cachedir: {cachedir}
+            extension_modules: {extmods}
+            ext_pillar:
+              - git:
+                - __env__ {url_extra_repo}:
+                  - all_saltenvs: master
+                - __env__ {url}:
+                  - mountpoint: nowhere
+            ''')
+        self.assertEqual(ret, expected)
+
+        # Test with passphraseless key and per-repo credential options
+        ret = self.get_pillar('''\
+            file_ignore_regex: []
+            file_ignore_glob: []
+            git_pillar_provider: pygit2
+            cachedir: {cachedir}
+            extension_modules: {extmods}
+            ext_pillar:
+              - git:
+                - __env__ {url_extra_repo}:
+                  - all_saltenvs: master
+                  - pubkey: {pubkey_nopass}
+                  - privkey: {privkey_nopass}
+                - __env__ {url}:
+                  - mountpoint: nowhere
+                  - pubkey: {pubkey_nopass}
+                  - privkey: {privkey_nopass}
+            ''')
+        self.assertEqual(ret, expected)
+
+        if grains['os_family'] == 'Debian':
+            # passphrase-protected currently does not work here
+            return
+
+        # Test with passphrase-protected key and global credential options
+        ret = self.get_pillar('''\
+            file_ignore_regex: []
+            file_ignore_glob: []
+            git_pillar_provider: pygit2
+            git_pillar_pubkey: {pubkey_withpass}
+            git_pillar_privkey: {privkey_withpass}
+            git_pillar_passphrase: {passphrase}
+            cachedir: {cachedir}
+            extension_modules: {extmods}
+            ext_pillar:
+              - git:
+                - __env__ {url_extra_repo}:
+                  - all_saltenvs: master
+                - __env__ {url}:
+                  - mountpoint: nowhere
+            ''')
+        self.assertEqual(ret, expected)
+
+        # Test with passphrase-protected key and per-repo credential options
+        ret = self.get_pillar('''\
+            file_ignore_regex: []
+            file_ignore_glob: []
+            git_pillar_provider: pygit2
+            cachedir: {cachedir}
+            extension_modules: {extmods}
+            ext_pillar:
+              - git:
+                - __env__ {url_extra_repo}:
+                  - all_saltenvs: master
+                  - pubkey: {pubkey_nopass}
+                  - privkey: {privkey_nopass}
+                  - passphrase: {passphrase}
+                - __env__ {url}:
+                  - mountpoint: nowhere
+                  - pubkey: {pubkey_nopass}
+                  - privkey: {privkey_nopass}
+                  - passphrase: {passphrase}
+            ''')
+        self.assertEqual(ret, expected)
+
 
 @skipIf(NO_MOCK, NO_MOCK_REASON)
 @skipIf(_windows_or_mac(), 'minion is windows or mac')
@@ -2119,6 +2243,33 @@ def test_all_saltenvs(self):
                         'nested_dict': {'dev': True}}}
         )
 
+    def test_all_saltenvs_base(self):
+        '''
+        Test all_saltenvs parameter with base pillarenv.
+        '''
+        ret = self.get_pillar('''\
+            file_ignore_regex: []
+            file_ignore_glob: []
+            git_pillar_provider: pygit2
+            cachedir: {cachedir}
+            extension_modules: {extmods}
+            ext_pillar:
+              - git:
+                - __env__ {url_extra_repo}:
+                  - all_saltenvs: master
+                - __env__ {url}:
+                  - mountpoint: nowhere
+            ''')
+        self.assertEqual(
+            ret,
+            {'branch': 'master',
+             'motd': 'The force will be with you. Always.',
+             'mylist': ['master'],
+             'mydict': {'master': True,
+                        'nested_list': ['master'],
+                        'nested_dict': {'master': True}}}
+        )
+
 
 @skipIf(NO_MOCK, NO_MOCK_REASON)
 @skipIf(_windows_or_mac(), 'minion is windows or mac')
@@ -2719,3 +2870,33 @@ def test_all_saltenvs(self):
                         'nested_list': ['dev'],
                         'nested_dict': {'dev': True}}}
         )
+
+    def test_all_saltenvs_base(self):
+        '''
+        Test all_saltenvs parameter with base pillarenv.
+        '''
+        ret = self.get_pillar('''\
+            file_ignore_regex: []
+            file_ignore_glob: []
+            git_pillar_provider: pygit2
+            git_pillar_user: {user}
+            git_pillar_password: {password}
+            git_pillar_insecure_auth: True
+            cachedir: {cachedir}
+            extension_modules: {extmods}
+            ext_pillar:
+              - git:
+                - __env__ {url_extra_repo}:
+                  - all_saltenvs: master
+                - __env__ {url}:
+                  - mountpoint: nowhere
+            ''')
+        self.assertEqual(
+            ret,
+            {'branch': 'master',
+             'motd': 'The force will be with you. Always.',
+             'mylist': ['master'],
+             'mydict': {'master': True,
+                        'nested_list': ['master'],
+                        'nested_dict': {'master': True}}}
+        )

tests/integration/reactor/test_reactor.py

@@ -35,4 +35,4 @@ def test_ping_reaction(self):
 
         e.fire_event({'a': 'b'}, '/test_event')
 
-        self.assertMinionEventReceived({'a': 'b'})
+        self.assertMinionEventReceived({'a': 'b'}, timeout=30)

tests/integration/states/test_x509.py

@@ -60,6 +60,9 @@ def setUp(self):
     def tearDown(self):
         os.remove(os.path.join(TMP_PILLAR_TREE, 'signing_policies.sls'))
         os.remove(os.path.join(TMP_PILLAR_TREE, 'top.sls'))
+        certs_path = os.path.join(TMP, 'pki')
+        if os.path.exists(certs_path):
+            salt.utils.files.rm_rf(certs_path)
         self.run_function('saltutil.refresh_pillar')
 
     def run_function(self, *args, **kwargs):