File make-sure-the-file-client-is-destroyed-upon-use... of Package salt

Overview Repositories Revisions Requests Users Attributes Meta

File make-sure-the-file-client-is-destroyed-upon-used.patch of Package salt (Revision 22b70cb7e10cc3311a07f43fe06ed96b)

Currently displaying revision 22b70cb7e10cc3311a07f43fe06ed96b , Show latest

From a1fc5287d501a1ecdbd259e5bbdd4f7d5d06dd13 Mon Sep 17 00:00:00 2001
From: Alexander Graul <agraul@suse.com>
Date: Fri, 28 Apr 2023 09:41:28 +0200
Subject: [PATCH] Make sure the file client is destroyed upon used

Backport of https://github.com/saltstack/salt/pull/64113
---
 salt/client/ssh/wrapper/saltcheck.py          | 108 +++----
 salt/fileclient.py                            |  11 -
 salt/modules/dockermod.py                     |  17 +-
 salt/pillar/__init__.py                       |   6 +-
 salt/states/ansiblegate.py                    |  11 +-
 salt/utils/asynchronous.py                    |   2 +-
 salt/utils/jinja.py                           |  53 ++-
 salt/utils/mako.py                            |   7 +
 salt/utils/templates.py                       | 303 +++++++++---------
 .../integration/states/test_include.py        |  40 +++
 .../utils/jinja/test_salt_cache_loader.py     |  47 ++-
 11 files changed, 330 insertions(+), 275 deletions(-)
 create mode 100644 tests/pytests/integration/states/test_include.py

diff --git a/salt/client/ssh/wrapper/saltcheck.py b/salt/client/ssh/wrapper/saltcheck.py
index d47b5cf6883..b0b94593809 100644
--- a/salt/client/ssh/wrapper/saltcheck.py
+++ b/salt/client/ssh/wrapper/saltcheck.py
@@ -9,6 +9,7 @@ import tarfile
 import tempfile
 from contextlib import closing
 
+import salt.fileclient
 import salt.utils.files
 import salt.utils.json
 import salt.utils.url
@@ -28,65 +29,62 @@ def update_master_cache(states, saltenv="base"):
     # Setup for copying states to gendir
     gendir = tempfile.mkdtemp()
     trans_tar = salt.utils.files.mkstemp()
-    if "cp.fileclient_{}".format(id(__opts__)) not in __context__:
-        __context__[
-            "cp.fileclient_{}".format(id(__opts__))
-        ] = salt.fileclient.get_file_client(__opts__)
-
-    # generate cp.list_states output and save to gendir
-    cp_output = salt.utils.json.dumps(__salt__["cp.list_states"]())
-    cp_output_file = os.path.join(gendir, "cp_output.txt")
-    with salt.utils.files.fopen(cp_output_file, "w") as fp:
-        fp.write(cp_output)
-
-    # cp state directories to gendir
-    already_processed = []
-    sls_list = salt.utils.args.split_input(states)
-    for state_name in sls_list:
-        # generate low data for each state and save to gendir
-        state_low_file = os.path.join(gendir, state_name + ".low")
-        state_low_output = salt.utils.json.dumps(
-            __salt__["state.show_low_sls"](state_name)
-        )
-        with salt.utils.files.fopen(state_low_file, "w") as fp:
-            fp.write(state_low_output)
-
-        state_name = state_name.replace(".", os.sep)
-        if state_name in already_processed:
-            log.debug("Already cached state for %s", state_name)
-        else:
-            file_copy_file = os.path.join(gendir, state_name + ".copy")
-            log.debug("copying %s to %s", state_name, gendir)
-            qualified_name = salt.utils.url.create(state_name, saltenv)
-            # Duplicate cp.get_dir to gendir
-            copy_result = __context__["cp.fileclient_{}".format(id(__opts__))].get_dir(
-                qualified_name, gendir, saltenv
+    with salt.fileclient.get_file_client(__opts__) as cp_fileclient:
+
+        # generate cp.list_states output and save to gendir
+        cp_output = salt.utils.json.dumps(__salt__["cp.list_states"]())
+        cp_output_file = os.path.join(gendir, "cp_output.txt")
+        with salt.utils.files.fopen(cp_output_file, "w") as fp:
+            fp.write(cp_output)
+
+        # cp state directories to gendir
+        already_processed = []
+        sls_list = salt.utils.args.split_input(states)
+        for state_name in sls_list:
+            # generate low data for each state and save to gendir
+            state_low_file = os.path.join(gendir, state_name + ".low")
+            state_low_output = salt.utils.json.dumps(
+                __salt__["state.show_low_sls"](state_name)
             )
-            if copy_result:
-                copy_result = [dir.replace(gendir, state_cache) for dir in copy_result]
-                copy_result_output = salt.utils.json.dumps(copy_result)
-                with salt.utils.files.fopen(file_copy_file, "w") as fp:
-                    fp.write(copy_result_output)
-                already_processed.append(state_name)
+            with salt.utils.files.fopen(state_low_file, "w") as fp:
+                fp.write(state_low_output)
+
+            state_name = state_name.replace(".", os.sep)
+            if state_name in already_processed:
+                log.debug("Already cached state for %s", state_name)
             else:
-                # If files were not copied, assume state.file.sls was given and just copy state
-                state_name = os.path.dirname(state_name)
                 file_copy_file = os.path.join(gendir, state_name + ".copy")
-                if state_name in already_processed:
-                    log.debug("Already cached state for %s", state_name)
+                log.debug("copying %s to %s", state_name, gendir)
+                qualified_name = salt.utils.url.create(state_name, saltenv)
+                # Duplicate cp.get_dir to gendir
+                copy_result = cp_fileclient.get_dir(qualified_name, gendir, saltenv)
+                if copy_result:
+                    copy_result = [
+                        dir.replace(gendir, state_cache) for dir in copy_result
+                    ]
+                    copy_result_output = salt.utils.json.dumps(copy_result)
+                    with salt.utils.files.fopen(file_copy_file, "w") as fp:
+                        fp.write(copy_result_output)
+                    already_processed.append(state_name)
                 else:
-                    qualified_name = salt.utils.url.create(state_name, saltenv)
-                    copy_result = __context__[
-                        "cp.fileclient_{}".format(id(__opts__))
-                    ].get_dir(qualified_name, gendir, saltenv)
-                    if copy_result:
-                        copy_result = [
-                            dir.replace(gendir, state_cache) for dir in copy_result
-                        ]
-                        copy_result_output = salt.utils.json.dumps(copy_result)
-                        with salt.utils.files.fopen(file_copy_file, "w") as fp:
-                            fp.write(copy_result_output)
-                        already_processed.append(state_name)
+                    # If files were not copied, assume state.file.sls was given and just copy state
+                    state_name = os.path.dirname(state_name)
+                    file_copy_file = os.path.join(gendir, state_name + ".copy")
+                    if state_name in already_processed:
+                        log.debug("Already cached state for %s", state_name)
+                    else:
+                        qualified_name = salt.utils.url.create(state_name, saltenv)
+                        copy_result = cp_fileclient.get_dir(
+                            qualified_name, gendir, saltenv
+                        )
+                        if copy_result:
+                            copy_result = [
+                                dir.replace(gendir, state_cache) for dir in copy_result
+                            ]
+                            copy_result_output = salt.utils.json.dumps(copy_result)
+                            with salt.utils.files.fopen(file_copy_file, "w") as fp:
+                                fp.write(copy_result_output)
+                            already_processed.append(state_name)
 
     # turn gendir into tarball and remove gendir
     try:
diff --git a/salt/fileclient.py b/salt/fileclient.py
index fef5154a0be..f01a86dd0d4 100644
--- a/salt/fileclient.py
+++ b/salt/fileclient.py
@@ -849,7 +849,6 @@ class Client:
             kwargs.pop("env")
 
         kwargs["saltenv"] = saltenv
-        url_data = urllib.parse.urlparse(url)
         sfn = self.cache_file(url, saltenv, cachedir=cachedir)
         if not sfn or not os.path.exists(sfn):
             return ""
@@ -1165,13 +1164,8 @@ class RemoteClient(Client):
 
         if not salt.utils.platform.is_windows():
             hash_server, stat_server = self.hash_and_stat_file(path, saltenv)
-            try:
-                mode_server = stat_server[0]
-            except (IndexError, TypeError):
-                mode_server = None
         else:
             hash_server = self.hash_file(path, saltenv)
-            mode_server = None
 
         # Check if file exists on server, before creating files and
         # directories
@@ -1214,13 +1208,8 @@ class RemoteClient(Client):
         if dest2check and os.path.isfile(dest2check):
             if not salt.utils.platform.is_windows():
                 hash_local, stat_local = self.hash_and_stat_file(dest2check, saltenv)
-                try:
-                    mode_local = stat_local[0]
-                except (IndexError, TypeError):
-                    mode_local = None
             else:
                 hash_local = self.hash_file(dest2check, saltenv)
-                mode_local = None
 
             if hash_local == hash_server:
                 return dest2check
diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py
index f7344b66ac6..69b722f0c95 100644
--- a/salt/modules/dockermod.py
+++ b/salt/modules/dockermod.py
@@ -6667,14 +6667,6 @@ def script_retcode(
     )["retcode"]
 
 
-def _mk_fileclient():
-    """
-    Create a file client and add it to the context.
-    """
-    if "cp.fileclient" not in __context__:
-        __context__["cp.fileclient"] = salt.fileclient.get_file_client(__opts__)
-
-
 def _generate_tmp_path():
     return os.path.join("/tmp", "salt.docker.{}".format(uuid.uuid4().hex[:6]))
 
@@ -6688,11 +6680,10 @@ def _prepare_trans_tar(name, sls_opts, mods=None, pillar=None, extra_filerefs=""
     # reuse it from salt.ssh, however this function should
     # be somewhere else
     refs = salt.client.ssh.state.lowstate_file_refs(chunks, extra_filerefs)
-    _mk_fileclient()
-    trans_tar = salt.client.ssh.state.prep_trans_tar(
-        __context__["cp.fileclient"], chunks, refs, pillar, name
-    )
-    return trans_tar
+    with salt.fileclient.get_file_client(__opts__) as fileclient:
+        return salt.client.ssh.state.prep_trans_tar(
+            fileclient, chunks, refs, pillar, name
+        )
 
 
 def _compile_state(sls_opts, mods=None):
diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py
index 0dfab4cc579..26312b3bd53 100644
--- a/salt/pillar/__init__.py
+++ b/salt/pillar/__init__.py
@@ -9,7 +9,6 @@ import logging
 import os
 import sys
 import traceback
-import uuid
 
 import salt.channel.client
 import salt.ext.tornado.gen
@@ -1351,6 +1350,11 @@ class Pillar:
         if hasattr(self, "_closing") and self._closing:
             return
         self._closing = True
+        if self.client:
+            try:
+                self.client.destroy()
+            except AttributeError:
+                pass
 
     # pylint: disable=W1701
     def __del__(self):
diff --git a/salt/states/ansiblegate.py b/salt/states/ansiblegate.py
index 7fd4deb6c2a..9abd418c42c 100644
--- a/salt/states/ansiblegate.py
+++ b/salt/states/ansiblegate.py
@@ -32,12 +32,10 @@ state:
           - state: installed
 
 """
-
 import logging
 import os
 import sys
 
-# Import salt modules
 import salt.fileclient
 import salt.utils.decorators.path
 from salt.utils.decorators import depends
@@ -108,13 +106,6 @@ def __virtual__():
     return __virtualname__
 
 
-def _client():
-    """
-    Get a fileclient
-    """
-    return salt.fileclient.get_file_client(__opts__)
-
-
 def _changes(plays):
     """
     Find changes in ansible return data
@@ -171,7 +162,7 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs=
     }
     if git_repo:
         if not isinstance(rundir, str) or not os.path.isdir(rundir):
-            with _client() as client:
+            with salt.fileclient.get_file_client(__opts__) as client:
                 rundir = client._extrn_path(git_repo, "base")
             log.trace("rundir set to %s", rundir)
         if not isinstance(git_kwargs, dict):
diff --git a/salt/utils/asynchronous.py b/salt/utils/asynchronous.py
index 2a858feee98..0c645bbc3bb 100644
--- a/salt/utils/asynchronous.py
+++ b/salt/utils/asynchronous.py
@@ -131,7 +131,7 @@ class SyncWrapper:
             result = io_loop.run_sync(lambda: getattr(self.obj, key)(*args, **kwargs))
             results.append(True)
             results.append(result)
-        except Exception as exc:  # pylint: disable=broad-except
+        except Exception:  # pylint: disable=broad-except
             results.append(False)
             results.append(sys.exc_info())
 
diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py
index fcc5aec497e..a6a8a279605 100644
--- a/salt/utils/jinja.py
+++ b/salt/utils/jinja.py
@@ -58,19 +58,6 @@ class SaltCacheLoader(BaseLoader):
     and only loaded once per loader instance.
     """
 
-    _cached_pillar_client = None
-    _cached_client = None
-
-    @classmethod
-    def shutdown(cls):
-        for attr in ("_cached_client", "_cached_pillar_client"):
-            client = getattr(cls, attr, None)
-            if client is not None:
-                # PillarClient and LocalClient objects do not have a destroy method
-                if hasattr(client, "destroy"):
-                    client.destroy()
-                setattr(cls, attr, None)
-
     def __init__(
         self,
         opts,
@@ -93,8 +80,7 @@ class SaltCacheLoader(BaseLoader):
         log.debug("Jinja search path: %s", self.searchpath)
         self.cached = []
         self._file_client = _file_client
-        # Instantiate the fileclient
-        self.file_client()
+        self._close_file_client = _file_client is None
 
     def file_client(self):
         """
@@ -108,18 +94,10 @@ class SaltCacheLoader(BaseLoader):
             or not hasattr(self._file_client, "opts")
             or self._file_client.opts["file_roots"] != self.opts["file_roots"]
         ):
-            attr = "_cached_pillar_client" if self.pillar_rend else "_cached_client"
-            cached_client = getattr(self, attr, None)
-            if (
-                cached_client is None
-                or not hasattr(cached_client, "opts")
-                or cached_client.opts["file_roots"] != self.opts["file_roots"]
-            ):
-                cached_client = salt.fileclient.get_file_client(
-                    self.opts, self.pillar_rend
-                )
-                setattr(SaltCacheLoader, attr, cached_client)
-            self._file_client = cached_client
+            self._file_client = salt.fileclient.get_file_client(
+                self.opts, self.pillar_rend
+            )
+            self._close_file_client = True
         return self._file_client
 
     def cache_file(self, template):
@@ -221,6 +199,27 @@ class SaltCacheLoader(BaseLoader):
         # there is no template file within searchpaths
         raise TemplateNotFound(template)
 
+    def destroy(self):
+        if self._close_file_client is False:
+            return
+        if self._file_client is None:
+            return
+        file_client = self._file_client
+        self._file_client = None
+
+        try:
+            file_client.destroy()
+        except AttributeError:
+            # PillarClient and LocalClient objects do not have a destroy method
+            pass
+
+    def __enter__(self):
+        self.file_client()
+        return self
+
+    def __exit__(self, *args):
+        self.destroy()
+
 
 class PrintableDict(OrderedDict):
     """
diff --git a/salt/utils/mako.py b/salt/utils/mako.py
index 69618de9837..037d5d86deb 100644
--- a/salt/utils/mako.py
+++ b/salt/utils/mako.py
@@ -97,3 +97,10 @@ if HAS_MAKO:
                 self.cache[fpath] = self.file_client().get_file(
                     fpath, "", True, self.saltenv
                 )
+
+        def destroy(self):
+            if self.client:
+                try:
+                    self.client.destroy()
+                except AttributeError:
+                    pass
diff --git a/salt/utils/templates.py b/salt/utils/templates.py
index 4947b820a36..4a8adf2a14f 100644
--- a/salt/utils/templates.py
+++ b/salt/utils/templates.py
@@ -362,163 +362,169 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None):
     elif tmplstr.endswith("\n"):
         newline = "\n"
 
-    if not saltenv:
-        if tmplpath:
-            loader = jinja2.FileSystemLoader(os.path.dirname(tmplpath))
-    else:
-        loader = salt.utils.jinja.SaltCacheLoader(
-            opts,
-            saltenv,
-            pillar_rend=context.get("_pillar_rend", False),
-            _file_client=file_client,
-        )
+    try:
+        if not saltenv:
+            if tmplpath:
+                loader = jinja2.FileSystemLoader(os.path.dirname(tmplpath))
+        else:
+            loader = salt.utils.jinja.SaltCacheLoader(
+                opts,
+                saltenv,
+                pillar_rend=context.get("_pillar_rend", False),
+                _file_client=file_client,
+            )
 
-    env_args = {"extensions": [], "loader": loader}
-
-    if hasattr(jinja2.ext, "with_"):
-        env_args["extensions"].append("jinja2.ext.with_")
-    if hasattr(jinja2.ext, "do"):
-        env_args["extensions"].append("jinja2.ext.do")
-    if hasattr(jinja2.ext, "loopcontrols"):
-        env_args["extensions"].append("jinja2.ext.loopcontrols")
-    env_args["extensions"].append(salt.utils.jinja.SerializerExtension)
-
-    opt_jinja_env = opts.get("jinja_env", {})
-    opt_jinja_sls_env = opts.get("jinja_sls_env", {})
-
-    opt_jinja_env = opt_jinja_env if isinstance(opt_jinja_env, dict) else {}
-    opt_jinja_sls_env = opt_jinja_sls_env if isinstance(opt_jinja_sls_env, dict) else {}
-
-    # Pass through trim_blocks and lstrip_blocks Jinja parameters
-    # trim_blocks removes newlines around Jinja blocks
-    # lstrip_blocks strips tabs and spaces from the beginning of
-    # line to the start of a block.
-    if opts.get("jinja_trim_blocks", False):
-        log.debug("Jinja2 trim_blocks is enabled")
-        log.warning(
-            "jinja_trim_blocks is deprecated and will be removed in a future release,"
-            " please use jinja_env and/or jinja_sls_env instead"
-        )
-        opt_jinja_env["trim_blocks"] = True
-        opt_jinja_sls_env["trim_blocks"] = True
-    if opts.get("jinja_lstrip_blocks", False):
-        log.debug("Jinja2 lstrip_blocks is enabled")
-        log.warning(
-            "jinja_lstrip_blocks is deprecated and will be removed in a future release,"
-            " please use jinja_env and/or jinja_sls_env instead"
-        )
-        opt_jinja_env["lstrip_blocks"] = True
-        opt_jinja_sls_env["lstrip_blocks"] = True
-
-    def opt_jinja_env_helper(opts, optname):
-        for k, v in opts.items():
-            k = k.lower()
-            if hasattr(jinja2.defaults, k.upper()):
-                log.debug("Jinja2 environment %s was set to %s by %s", k, v, optname)
-                env_args[k] = v
-            else:
-                log.warning("Jinja2 environment %s is not recognized", k)
+        env_args = {"extensions": [], "loader": loader}
 
-    if "sls" in context and context["sls"] != "":
-        opt_jinja_env_helper(opt_jinja_sls_env, "jinja_sls_env")
-    else:
-        opt_jinja_env_helper(opt_jinja_env, "jinja_env")
+        if hasattr(jinja2.ext, "with_"):
+            env_args["extensions"].append("jinja2.ext.with_")
+        if hasattr(jinja2.ext, "do"):
+            env_args["extensions"].append("jinja2.ext.do")
+        if hasattr(jinja2.ext, "loopcontrols"):
+            env_args["extensions"].append("jinja2.ext.loopcontrols")
+        env_args["extensions"].append(salt.utils.jinja.SerializerExtension)
 
-    if opts.get("allow_undefined", False):
-        jinja_env = jinja2.sandbox.SandboxedEnvironment(**env_args)
-    else:
-        jinja_env = jinja2.sandbox.SandboxedEnvironment(
-            undefined=jinja2.StrictUndefined, **env_args
-        )
+        opt_jinja_env = opts.get("jinja_env", {})
+        opt_jinja_sls_env = opts.get("jinja_sls_env", {})
 
-    indent_filter = jinja_env.filters.get("indent")
-    jinja_env.tests.update(JinjaTest.salt_jinja_tests)
-    jinja_env.filters.update(JinjaFilter.salt_jinja_filters)
-    if salt.utils.jinja.JINJA_VERSION >= Version("2.11"):
-        # Use the existing indent filter on Jinja versions where it's not broken
-        jinja_env.filters["indent"] = indent_filter
-    jinja_env.globals.update(JinjaGlobal.salt_jinja_globals)
-
-    # globals
-    jinja_env.globals["odict"] = OrderedDict
-    jinja_env.globals["show_full_context"] = salt.utils.jinja.show_full_context
-
-    jinja_env.tests["list"] = salt.utils.data.is_list
-
-    decoded_context = {}
-    for key, value in context.items():
-        if not isinstance(value, str):
-            if isinstance(value, NamedLoaderContext):
-                decoded_context[key] = value.value()
-            else:
-                decoded_context[key] = value
-            continue
+        opt_jinja_env = opt_jinja_env if isinstance(opt_jinja_env, dict) else {}
+        opt_jinja_sls_env = (
+            opt_jinja_sls_env if isinstance(opt_jinja_sls_env, dict) else {}
+        )
 
-        try:
-            decoded_context[key] = salt.utils.stringutils.to_unicode(
-                value, encoding=SLS_ENCODING
+        # Pass through trim_blocks and lstrip_blocks Jinja parameters
+        # trim_blocks removes newlines around Jinja blocks
+        # lstrip_blocks strips tabs and spaces from the beginning of
+        # line to the start of a block.
+        if opts.get("jinja_trim_blocks", False):
+            log.debug("Jinja2 trim_blocks is enabled")
+            log.warning(
+                "jinja_trim_blocks is deprecated and will be removed in a future release,"
+                " please use jinja_env and/or jinja_sls_env instead"
             )
-        except UnicodeDecodeError as ex:
-            log.debug(
-                "Failed to decode using default encoding (%s), trying system encoding",
-                SLS_ENCODING,
+            opt_jinja_env["trim_blocks"] = True
+            opt_jinja_sls_env["trim_blocks"] = True
+        if opts.get("jinja_lstrip_blocks", False):
+            log.debug("Jinja2 lstrip_blocks is enabled")
+            log.warning(
+                "jinja_lstrip_blocks is deprecated and will be removed in a future release,"
+                " please use jinja_env and/or jinja_sls_env instead"
             )
-            decoded_context[key] = salt.utils.data.decode(value)
+            opt_jinja_env["lstrip_blocks"] = True
+            opt_jinja_sls_env["lstrip_blocks"] = True
+
+        def opt_jinja_env_helper(opts, optname):
+            for k, v in opts.items():
+                k = k.lower()
+                if hasattr(jinja2.defaults, k.upper()):
+                    log.debug(
+                        "Jinja2 environment %s was set to %s by %s", k, v, optname
+                    )
+                    env_args[k] = v
+                else:
+                    log.warning("Jinja2 environment %s is not recognized", k)
 
-    jinja_env.globals.update(decoded_context)
-    try:
-        template = jinja_env.from_string(tmplstr)
-        output = template.render(**decoded_context)
-    except jinja2.exceptions.UndefinedError as exc:
-        trace = traceback.extract_tb(sys.exc_info()[2])
-        line, out = _get_jinja_error(trace, context=decoded_context)
-        if not line:
-            tmplstr = ""
-        raise SaltRenderError("Jinja variable {}{}".format(exc, out), line, tmplstr)
-    except (
-        jinja2.exceptions.TemplateRuntimeError,
-        jinja2.exceptions.TemplateSyntaxError,
-        jinja2.exceptions.SecurityError,
-    ) as exc:
-        trace = traceback.extract_tb(sys.exc_info()[2])
-        line, out = _get_jinja_error(trace, context=decoded_context)
-        if not line:
-            tmplstr = ""
-        raise SaltRenderError(
-            "Jinja syntax error: {}{}".format(exc, out), line, tmplstr
-        )
-    except (SaltInvocationError, CommandExecutionError) as exc:
-        trace = traceback.extract_tb(sys.exc_info()[2])
-        line, out = _get_jinja_error(trace, context=decoded_context)
-        if not line:
-            tmplstr = ""
-        raise SaltRenderError(
-            "Problem running salt function in Jinja template: {}{}".format(exc, out),
-            line,
-            tmplstr,
-        )
-    except Exception as exc:  # pylint: disable=broad-except
-        tracestr = traceback.format_exc()
-        trace = traceback.extract_tb(sys.exc_info()[2])
-        line, out = _get_jinja_error(trace, context=decoded_context)
-        if not line:
-            tmplstr = ""
+        if "sls" in context and context["sls"] != "":
+            opt_jinja_env_helper(opt_jinja_sls_env, "jinja_sls_env")
         else:
-            tmplstr += "\n{}".format(tracestr)
-        log.debug("Jinja Error")
-        log.debug("Exception:", exc_info=True)
-        log.debug("Out: %s", out)
-        log.debug("Line: %s", line)
-        log.debug("TmplStr: %s", tmplstr)
-        log.debug("TraceStr: %s", tracestr)
+            opt_jinja_env_helper(opt_jinja_env, "jinja_env")
 
-        raise SaltRenderError(
-            "Jinja error: {}{}".format(exc, out), line, tmplstr, trace=tracestr
-        )
+        if opts.get("allow_undefined", False):
+            jinja_env = jinja2.sandbox.SandboxedEnvironment(**env_args)
+        else:
+            jinja_env = jinja2.sandbox.SandboxedEnvironment(
+                undefined=jinja2.StrictUndefined, **env_args
+            )
+
+        indent_filter = jinja_env.filters.get("indent")
+        jinja_env.tests.update(JinjaTest.salt_jinja_tests)
+        jinja_env.filters.update(JinjaFilter.salt_jinja_filters)
+        if salt.utils.jinja.JINJA_VERSION >= Version("2.11"):
+            # Use the existing indent filter on Jinja versions where it's not broken
+            jinja_env.filters["indent"] = indent_filter
+        jinja_env.globals.update(JinjaGlobal.salt_jinja_globals)
+
+        # globals
+        jinja_env.globals["odict"] = OrderedDict
+        jinja_env.globals["show_full_context"] = salt.utils.jinja.show_full_context
+
+        jinja_env.tests["list"] = salt.utils.data.is_list
+
+        decoded_context = {}
+        for key, value in context.items():
+            if not isinstance(value, str):
+                if isinstance(value, NamedLoaderContext):
+                    decoded_context[key] = value.value()
+                else:
+                    decoded_context[key] = value
+                continue
+
+            try:
+                decoded_context[key] = salt.utils.stringutils.to_unicode(
+                    value, encoding=SLS_ENCODING
+                )
+            except UnicodeDecodeError:
+                log.debug(
+                    "Failed to decode using default encoding (%s), trying system encoding",
+                    SLS_ENCODING,
+                )
+                decoded_context[key] = salt.utils.data.decode(value)
+
+        jinja_env.globals.update(decoded_context)
+        try:
+            template = jinja_env.from_string(tmplstr)
+            output = template.render(**decoded_context)
+        except jinja2.exceptions.UndefinedError as exc:
+            trace = traceback.extract_tb(sys.exc_info()[2])
+            line, out = _get_jinja_error(trace, context=decoded_context)
+            if not line:
+                tmplstr = ""
+            raise SaltRenderError("Jinja variable {}{}".format(exc, out), line, tmplstr)
+        except (
+            jinja2.exceptions.TemplateRuntimeError,
+            jinja2.exceptions.TemplateSyntaxError,
+            jinja2.exceptions.SecurityError,
+        ) as exc:
+            trace = traceback.extract_tb(sys.exc_info()[2])
+            line, out = _get_jinja_error(trace, context=decoded_context)
+            if not line:
+                tmplstr = ""
+            raise SaltRenderError(
+                "Jinja syntax error: {}{}".format(exc, out), line, tmplstr
+            )
+        except (SaltInvocationError, CommandExecutionError) as exc:
+            trace = traceback.extract_tb(sys.exc_info()[2])
+            line, out = _get_jinja_error(trace, context=decoded_context)
+            if not line:
+                tmplstr = ""
+            raise SaltRenderError(
+                "Problem running salt function in Jinja template: {}{}".format(
+                    exc, out
+                ),
+                line,
+                tmplstr,
+            )
+        except Exception as exc:  # pylint: disable=broad-except
+            tracestr = traceback.format_exc()
+            trace = traceback.extract_tb(sys.exc_info()[2])
+            line, out = _get_jinja_error(trace, context=decoded_context)
+            if not line:
+                tmplstr = ""
+            else:
+                tmplstr += "\n{}".format(tracestr)
+            log.debug("Jinja Error")
+            log.debug("Exception:", exc_info=True)
+            log.debug("Out: %s", out)
+            log.debug("Line: %s", line)
+            log.debug("TmplStr: %s", tmplstr)
+            log.debug("TraceStr: %s", tracestr)
+
+            raise SaltRenderError(
+                "Jinja error: {}{}".format(exc, out), line, tmplstr, trace=tracestr
+            )
     finally:
-        if loader and hasattr(loader, "_file_client"):
-            if hasattr(loader._file_client, "destroy"):
-                loader._file_client.destroy()
+        if loader and isinstance(loader, salt.utils.jinja.SaltCacheLoader):
+            loader.destroy()
 
     # Workaround a bug in Jinja that removes the final newline
     # (https://github.com/mitsuhiko/jinja2/issues/75)
@@ -569,9 +575,8 @@ def render_mako_tmpl(tmplstr, context, tmplpath=None):
     except Exception:  # pylint: disable=broad-except
         raise SaltRenderError(mako.exceptions.text_error_template().render())
     finally:
-        if lookup and hasattr(lookup, "_file_client"):
-            if hasattr(lookup._file_client, "destroy"):
-                lookup._file_client.destroy()
+        if lookup and isinstance(lookup, SaltMakoTemplateLookup):
+            lookup.destroy()
 
 
 def render_wempy_tmpl(tmplstr, context, tmplpath=None):
diff --git a/tests/pytests/integration/states/test_include.py b/tests/pytests/integration/states/test_include.py
new file mode 100644
index 00000000000..f814328c5e4
--- /dev/null
+++ b/tests/pytests/integration/states/test_include.py
@@ -0,0 +1,40 @@
+"""
+Integration tests for the jinja includes in states
+"""
+import logging
+
+import pytest
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.slow_test
+def test_issue_64111(salt_master, salt_minion, salt_call_cli):
+    # This needs to be an integration test. A functional test does not trigger
+    # the issue fixed.
+
+    macros_jinja = """
+    {% macro a_jinja_macro(arg) -%}
+    {{ arg }}
+    {%- endmacro %}
+    """
+
+    init_sls = """
+    include:
+      - common.file1
+    """
+
+    file1_sls = """
+    {% from 'common/macros.jinja' import a_jinja_macro with context %}
+
+    a state id:
+      cmd.run:
+        - name: echo {{ a_jinja_macro("hello world") }}
+    """
+    tf = salt_master.state_tree.base.temp_file
+
+    with tf("common/macros.jinja", macros_jinja):
+        with tf("common/init.sls", init_sls):
+            with tf("common/file1.sls", file1_sls):
+                ret = salt_call_cli.run("state.apply", "common")
+                assert ret.returncode == 0
diff --git a/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py b/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py
index 38c5ce5b724..e0f5fa158ff 100644
--- a/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py
+++ b/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py
@@ -15,7 +15,7 @@ import salt.utils.json  # pylint: disable=unused-import
 import salt.utils.stringutils  # pylint: disable=unused-import
 import salt.utils.yaml  # pylint: disable=unused-import
 from salt.utils.jinja import SaltCacheLoader
-from tests.support.mock import Mock, patch
+from tests.support.mock import Mock, call, patch
 
 
 @pytest.fixture
@@ -224,14 +224,45 @@ def test_file_client_kwarg(minion_opts, mock_file_client):
     assert loader._file_client is mock_file_client
 
 
-def test_cache_loader_shutdown(minion_opts, mock_file_client):
+def test_cache_loader_passed_file_client(minion_opts, mock_file_client):
     """
     The shudown method can be called without raising an exception when the
     file_client does not have a destroy method
     """
-    assert not hasattr(mock_file_client, "destroy")
-    mock_file_client.opts = minion_opts
-    loader = SaltCacheLoader(minion_opts, _file_client=mock_file_client)
-    assert loader._file_client is mock_file_client
-    # Shutdown method should not raise any exceptions
-    loader.shutdown()
+    # Test SaltCacheLoader creating and destroying the file client created
+    file_client = Mock()
+    with patch("salt.fileclient.get_file_client", return_value=file_client):
+        loader = SaltCacheLoader(minion_opts)
+        assert loader._file_client is None
+        with loader:
+            assert loader._file_client is file_client
+        assert loader._file_client is None
+        assert file_client.mock_calls == [call.destroy()]
+
+    # Test SaltCacheLoader reusing the file client passed
+    file_client = Mock()
+    file_client.opts = {"file_roots": minion_opts["file_roots"]}
+    with patch("salt.fileclient.get_file_client", return_value=Mock()):
+        loader = SaltCacheLoader(minion_opts, _file_client=file_client)
+        assert loader._file_client is file_client
+        with loader:
+            assert loader._file_client is file_client
+        assert loader._file_client is file_client
+        assert file_client.mock_calls == []
+
+    # Test SaltCacheLoader creating a client even though a file client was
+    # passed because the "file_roots" option is different, and, as such,
+    # the destroy method on the new file client is called, but not on the
+    # file client passed in.
+    file_client = Mock()
+    file_client.opts = {"file_roots": ""}
+    new_file_client = Mock()
+    with patch("salt.fileclient.get_file_client", return_value=new_file_client):
+        loader = SaltCacheLoader(minion_opts, _file_client=file_client)
+        assert loader._file_client is file_client
+        with loader:
+            assert loader._file_client is not file_client
+            assert loader._file_client is new_file_client
+        assert loader._file_client is None
+        assert file_client.mock_calls == []
+        assert new_file_client.mock_calls == [call.destroy()]
-- 
2.40.0

Places

File make-sure-the-file-client-is-destroyed-upon-used.patch of Package salt (Revision 22b70cb7e10cc3311a07f43fe06ed96b)

Places