Make sure the file client is destroyed upon used

Signed-off-by: Pedro Algarvio <[email protected]>

salt/client/ssh/wrapper/saltcheck.py

@@ -9,6 +9,7 @@
 import tempfile
 from contextlib import closing
 
+import salt.fileclient
 import salt.utils.files
 import salt.utils.json
 import salt.utils.url
@@ -28,65 +29,66 @@ def update_master_cache(states, saltenv="base"):
     # Setup for copying states to gendir
     gendir = tempfile.mkdtemp()
     trans_tar = salt.utils.files.mkstemp()
-    if "cp.fileclient_{}".format(id(__opts__)) not in __context__:
-        __context__[
-            "cp.fileclient_{}".format(id(__opts__))
-        ] = salt.fileclient.get_file_client(__opts__)
-
-    # generate cp.list_states output and save to gendir
-    cp_output = salt.utils.json.dumps(__salt__["cp.list_states"]())
-    cp_output_file = os.path.join(gendir, "cp_output.txt")
-    with salt.utils.files.fopen(cp_output_file, "w") as fp:
-        fp.write(cp_output)
-
-    # cp state directories to gendir
-    already_processed = []
-    sls_list = salt.utils.args.split_input(states)
-    for state_name in sls_list:
-        # generate low data for each state and save to gendir
-        state_low_file = os.path.join(gendir, state_name + ".low")
-        state_low_output = salt.utils.json.dumps(
-            __salt__["state.show_low_sls"](state_name)
-        )
-        with salt.utils.files.fopen(state_low_file, "w") as fp:
-            fp.write(state_low_output)
-
-        state_name = state_name.replace(".", os.sep)
-        if state_name in already_processed:
-            log.debug("Already cached state for %s", state_name)
-        else:
-            file_copy_file = os.path.join(gendir, state_name + ".copy")
-            log.debug("copying %s to %s", state_name, gendir)
-            qualified_name = salt.utils.url.create(state_name, saltenv)
-            # Duplicate cp.get_dir to gendir
-            copy_result = __context__["cp.fileclient_{}".format(id(__opts__))].get_dir(
-                qualified_name, gendir, saltenv
+    cp_fileclient_ctx_key = "cp.fileclient_{}".format(id(__opts__))
+    if cp_fileclient_ctx_key not in __context__:
+        __context__[cp_fileclient_ctx_key] = salt.fileclient.get_file_client(__opts__)
+
+    with __context__[cp_fileclient_ctx_key] as cp_fileclient:
+
+        # generate cp.list_states output and save to gendir
+        cp_output = salt.utils.json.dumps(__salt__["cp.list_states"]())
+        cp_output_file = os.path.join(gendir, "cp_output.txt")
+        with salt.utils.files.fopen(cp_output_file, "w") as fp:
+            fp.write(cp_output)
+
+        # cp state directories to gendir
+        already_processed = []
+        sls_list = salt.utils.args.split_input(states)
+        for state_name in sls_list:
+            # generate low data for each state and save to gendir
+            state_low_file = os.path.join(gendir, state_name + ".low")
+            state_low_output = salt.utils.json.dumps(
+                __salt__["state.show_low_sls"](state_name)
             )
-            if copy_result:
-                copy_result = [dir.replace(gendir, state_cache) for dir in copy_result]
-                copy_result_output = salt.utils.json.dumps(copy_result)
-                with salt.utils.files.fopen(file_copy_file, "w") as fp:
-                    fp.write(copy_result_output)
-                already_processed.append(state_name)
+            with salt.utils.files.fopen(state_low_file, "w") as fp:
+                fp.write(state_low_output)
+
+            state_name = state_name.replace(".", os.sep)
+            if state_name in already_processed:
+                log.debug("Already cached state for %s", state_name)
             else:
-                # If files were not copied, assume state.file.sls was given and just copy state
-                state_name = os.path.dirname(state_name)
                 file_copy_file = os.path.join(gendir, state_name + ".copy")
-                if state_name in already_processed:
-                    log.debug("Already cached state for %s", state_name)
+                log.debug("copying %s to %s", state_name, gendir)
+                qualified_name = salt.utils.url.create(state_name, saltenv)
+                # Duplicate cp.get_dir to gendir
+                copy_result = cp_fileclient.get_dir(qualified_name, gendir, saltenv)
+                if copy_result:
+                    copy_result = [
+                        dir.replace(gendir, state_cache) for dir in copy_result
+                    ]
+                    copy_result_output = salt.utils.json.dumps(copy_result)
+                    with salt.utils.files.fopen(file_copy_file, "w") as fp:
+                        fp.write(copy_result_output)
+                    already_processed.append(state_name)
                 else:
-                    qualified_name = salt.utils.url.create(state_name, saltenv)
-                    copy_result = __context__[
-                        "cp.fileclient_{}".format(id(__opts__))
-                    ].get_dir(qualified_name, gendir, saltenv)
-                    if copy_result:
-                        copy_result = [
-                            dir.replace(gendir, state_cache) for dir in copy_result
-                        ]
-                        copy_result_output = salt.utils.json.dumps(copy_result)
-                        with salt.utils.files.fopen(file_copy_file, "w") as fp:
-                            fp.write(copy_result_output)
-                        already_processed.append(state_name)
+                    # If files were not copied, assume state.file.sls was given and just copy state
+                    state_name = os.path.dirname(state_name)
+                    file_copy_file = os.path.join(gendir, state_name + ".copy")
+                    if state_name in already_processed:
+                        log.debug("Already cached state for %s", state_name)
+                    else:
+                        qualified_name = salt.utils.url.create(state_name, saltenv)
+                        copy_result = cp_fileclient.get_dir(
+                            qualified_name, gendir, saltenv
+                        )
+                        if copy_result:
+                            copy_result = [
+                                dir.replace(gendir, state_cache) for dir in copy_result
+                            ]
+                            copy_result_output = salt.utils.json.dumps(copy_result)
+                            with salt.utils.files.fopen(file_copy_file, "w") as fp:
+                                fp.write(copy_result_output)
+                            already_processed.append(state_name)
 
     # turn gendir into tarball and remove gendir
     try:

salt/fileclient.py

@@ -849,7 +849,6 @@ def get_template(
             kwargs.pop("env")
 
         kwargs["saltenv"] = saltenv
-        url_data = urllib.parse.urlparse(url)
         sfn = self.cache_file(url, saltenv, cachedir=cachedir)
         if not sfn or not os.path.exists(sfn):
             return ""
@@ -1165,13 +1164,8 @@ def get_file(
 
         if not salt.utils.platform.is_windows():
             hash_server, stat_server = self.hash_and_stat_file(path, saltenv)
-            try:
-                mode_server = stat_server[0]
-            except (IndexError, TypeError):
-                mode_server = None
         else:
             hash_server = self.hash_file(path, saltenv)
-            mode_server = None
 
         # Check if file exists on server, before creating files and
         # directories
@@ -1214,13 +1208,8 @@ def get_file(
         if dest2check and os.path.isfile(dest2check):
             if not salt.utils.platform.is_windows():
                 hash_local, stat_local = self.hash_and_stat_file(dest2check, saltenv)
-                try:
-                    mode_local = stat_local[0]
-                except (IndexError, TypeError):
-                    mode_local = None
             else:
                 hash_local = self.hash_file(dest2check, saltenv)
-                mode_local = None
 
             if hash_local == hash_server:
                 return dest2check

salt/modules/dockermod.py

@@ -6644,14 +6644,6 @@ def script_retcode(
     )["retcode"]
 
 
-def _mk_fileclient():
-    """
-    Create a file client and add it to the context.
-    """
-    if "cp.fileclient" not in __context__:
-        __context__["cp.fileclient"] = salt.fileclient.get_file_client(__opts__)
-
-
 def _generate_tmp_path():
     return os.path.join("/tmp", "salt.docker.{}".format(uuid.uuid4().hex[:6]))
 
@@ -6665,11 +6657,10 @@ def _prepare_trans_tar(name, sls_opts, mods=None, pillar=None, extra_filerefs=""
     # reuse it from salt.ssh, however this function should
     # be somewhere else
     refs = salt.client.ssh.state.lowstate_file_refs(chunks, extra_filerefs)
-    _mk_fileclient()
-    trans_tar = salt.client.ssh.state.prep_trans_tar(
-        __context__["cp.fileclient"], chunks, refs, pillar, name
-    )
-    return trans_tar
+    with salt.fileclient.get_file_client(__opts__) as fileclient:
+        return salt.client.ssh.state.prep_trans_tar(
+            fileclient, chunks, refs, pillar, name
+        )
 
 
 def _compile_state(sls_opts, mods=None):

salt/pillar/__init__.py

@@ -9,7 +9,6 @@
 import os
 import sys
 import traceback
-import uuid
 
 import salt.channel.client
 import salt.ext.tornado.gen
@@ -1341,6 +1340,11 @@ def destroy(self):
         if self._closing:
             return
         self._closing = True
+        if self.client:
+            try:
+                self.client.destroy()
+            except AttributeError:
+                pass
 
     # pylint: disable=W1701
     def __del__(self):

salt/states/ansiblegate.py

@@ -32,12 +32,10 @@
           - state: installed
 
 """
-
 import logging
 import os
 import sys
 
-# Import salt modules
 import salt.fileclient
 import salt.utils.decorators.path
 from salt.utils.decorators import depends
@@ -108,13 +106,6 @@ def __virtual__():
     return __virtualname__
 
 
-def _client():
-    """
-    Get a fileclient
-    """
-    return salt.fileclient.get_file_client(__opts__)
-
-
 def _changes(plays):
     """
     Find changes in ansible return data
@@ -171,7 +162,7 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs=
     }
     if git_repo:
         if not isinstance(rundir, str) or not os.path.isdir(rundir):
-            with _client() as client:
+            with salt.fileclient.get_file_client(__opts__) as client:
                 rundir = client._extrn_path(git_repo, "base")
             log.trace("rundir set to %s", rundir)
         if not isinstance(git_kwargs, dict):

salt/utils/asynchronous.py

@@ -131,7 +131,7 @@ def _target(self, key, args, kwargs, results, io_loop):
             result = io_loop.run_sync(lambda: getattr(self.obj, key)(*args, **kwargs))
             results.append(True)
             results.append(result)
-        except Exception as exc:  # pylint: disable=broad-except
+        except Exception:  # pylint: disable=broad-except
             results.append(False)
             results.append(sys.exc_info())
 

salt/utils/jinja.py

@@ -221,6 +221,24 @@ def uptodate():
         # there is no template file within searchpaths
         raise TemplateNotFound(template)
 
+    def destroy(self):
+        for attr in ("_cached_client", "_cached_pillar_client"):
+            client = getattr(self, attr, None)
+            if client is not None:
+                try:
+                    client.destroy()
+                except AttributeError:
+                    # PillarClient and LocalClient objects do not have a destroy method
+                    pass
+                setattr(self, attr, None)
+
+    def __enter__(self):
+        self.file_client()
+        return self
+
+    def __exit__(self, *args):
+        self.destroy()
+
 
 class PrintableDict(OrderedDict):
     """

salt/utils/mako.py

@@ -97,3 +97,10 @@ def cache_file(self, fpath):
                 self.cache[fpath] = self.file_client().get_file(
                     fpath, "", True, self.saltenv
                 )
+
+        def destroy(self):
+            if self.client:
+                try:
+                    self.client.destroy()
+                except AttributeError:
+                    pass