File use-salt-bundle-in-dockermod.patch of Package salt

Overview Repositories Revisions Requests Users Attributes Meta

File use-salt-bundle-in-dockermod.patch of Package salt (Revision 8f61a1108956c9d5977363a990af94dd)

Currently displaying revision 8f61a1108956c9d5977363a990af94dd , Show latest

From b0891f83afa354c4b1f803af8a679ecf5a7fb63c Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 27 Jun 2022 17:59:24 +0300
Subject: [PATCH] Use Salt Bundle in dockermod

* Use Salt Bundle for salt calls in dockermod

* Add test of performing a call with the Salt Bundle
---
 salt/modules/dockermod.py                     | 197 +++++++++++++++---
 .../unit/modules/dockermod/test_module.py     |  78 ++++++-
 2 files changed, 241 insertions(+), 34 deletions(-)

diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py
index 6870c26b0e..8b6ab8058e 100644
--- a/salt/modules/dockermod.py
+++ b/salt/modules/dockermod.py
@@ -201,14 +201,19 @@ import copy
 import fnmatch
 import functools
 import gzip
+import hashlib
 import json
 import logging
 import os
+import pathlib
 import pipes
 import re
 import shutil
 import string
 import subprocess
+import sys
+import tarfile
+import tempfile
 import time
 import uuid
 
@@ -6698,6 +6703,111 @@ def _compile_state(sls_opts, mods=None):
         return st_.state.compile_high_data(high_data)
 
 
+def gen_venv_tar(cachedir, venv_dest_dir, venv_name):
+    """
+    Generate tarball with the Salt Bundle if required and return the path to it
+    """
+    exec_path = pathlib.Path(sys.executable).parts
+    venv_dir_name = "venv-salt-minion"
+    if venv_dir_name not in exec_path:
+        return None
+
+    venv_tar = os.path.join(cachedir, "venv-salt.tgz")
+    venv_hash = os.path.join(cachedir, "venv-salt.hash")
+    venv_lock = os.path.join(cachedir, ".venv-salt.lock")
+
+    venv_path = os.path.join(*exec_path[0 : exec_path.index(venv_dir_name)])
+
+    with __utils__["files.flopen"](venv_lock, "w"):
+        start_dir = os.getcwd()
+        venv_hash_file = os.path.join(venv_path, venv_dir_name, "venv-hash.txt")
+        try:
+            with __utils__["files.fopen"](venv_hash_file, "r") as fh:
+                venv_hash_src = fh.readline().strip()
+        except Exception:  # pylint: disable=broad-except
+            # It makes no sense what caused the exception
+            # Just calculate the hash different way
+            for cmd in ("rpm -qi venv-salt-minion", "dpkg -s venv-salt-minion"):
+                ret = __salt__["cmd.run_all"](
+                    cmd,
+                    python_shell=True,
+                    clean_env=True,
+                    env={"LANG": "C", "LANGUAGE": "C", "LC_ALL": "C"},
+                )
+                if ret.get("retcode") == 0 and ret.get("stdout"):
+                    venv_hash_src = hashlib.sha256(
+                        "{}\n".format(ret.get("stdout")).encode()
+                    ).hexdigest()
+                    break
+        try:
+            with __utils__["files.fopen"](venv_hash, "r") as fh:
+                venv_hash_dest = fh.readline().strip()
+        except Exception:  # pylint: disable=broad-except
+            # It makes no sense what caused the exception
+            # Set the hash to impossible value to force new tarball creation
+            venv_hash_dest = "UNKNOWN"
+        if venv_hash_src == venv_hash_dest and os.path.isfile(venv_tar):
+            return venv_tar
+        try:
+            tfd, tmp_venv_tar = tempfile.mkstemp(
+                dir=cachedir,
+                prefix=".venv-",
+                suffix=os.path.splitext(venv_tar)[1],
+            )
+            os.close(tfd)
+
+            os.chdir(venv_path)
+            tfp = tarfile.open(tmp_venv_tar, "w:gz")
+
+            for root, dirs, files in salt.utils.path.os_walk(
+                venv_dir_name, followlinks=True
+            ):
+                for name in files:
+                    if name == "python" and pathlib.Path(root).parts == (
+                        venv_dir_name,
+                        "bin",
+                    ):
+                        tfd, tmp_python_file = tempfile.mkstemp(
+                            dir=cachedir,
+                            prefix=".python-",
+                        )
+                        os.close(tfd)
+                        try:
+                            with __utils__["files.fopen"](
+                                os.path.join(root, name), "r"
+                            ) as fh_in:
+                                with __utils__["files.fopen"](
+                                    tmp_python_file, "w"
+                                ) as fh_out:
+                                    rd_lines = fh_in.readlines()
+                                    rd_lines = [
+                                        'export VIRTUAL_ENV="{}"\n'.format(
+                                            os.path.join(venv_dest_dir, venv_name)
+                                        )
+                                        if line.startswith("export VIRTUAL_ENV=")
+                                        else line
+                                        for line in rd_lines
+                                    ]
+                                    fh_out.write("".join(rd_lines))
+                            os.chmod(tmp_python_file, 0o755)
+                            tfp.add(tmp_python_file, arcname=os.path.join(root, name))
+                            continue
+                        finally:
+                            if os.path.isfile(tmp_python_file):
+                                os.remove(tmp_python_file)
+                    if not name.endswith((".pyc", ".pyo")):
+                        tfp.add(os.path.join(root, name))
+
+            tfp.close()
+            shutil.move(tmp_venv_tar, venv_tar)
+            with __utils__["files.fopen"](venv_hash, "w") as fh:
+                fh.write("{}\n".format(venv_hash_src))
+        finally:
+            os.chdir(start_dir)
+
+    return venv_tar
+
+
 def call(name, function, *args, **kwargs):
     """
     Executes a Salt function inside a running container
@@ -6733,47 +6843,68 @@ def call(name, function, *args, **kwargs):
     if function is None:
         raise CommandExecutionError("Missing function parameter")
 
-    # move salt into the container
-    thin_path = __utils__["thin.gen_thin"](
-        __opts__["cachedir"],
-        extra_mods=__salt__["config.option"]("thin_extra_mods", ""),
-        so_mods=__salt__["config.option"]("thin_so_mods", ""),
-    )
-    ret = copy_to(
-        name, thin_path, os.path.join(thin_dest_path, os.path.basename(thin_path))
-    )
+    venv_dest_path = "/var/tmp"
+    venv_name = "venv-salt-minion"
+    venv_tar = gen_venv_tar(__opts__["cachedir"], venv_dest_path, venv_name)
 
-    # figure out available python interpreter inside the container (only Python3)
-    pycmds = ("python3", "/usr/libexec/platform-python")
-    container_python_bin = None
-    for py_cmd in pycmds:
-        cmd = [py_cmd] + ["--version"]
-        ret = run_all(name, subprocess.list2cmdline(cmd))
-        if ret["retcode"] == 0:
-            container_python_bin = py_cmd
-            break
-    if not container_python_bin:
-        raise CommandExecutionError(
-            "Python interpreter cannot be found inside the container. Make sure Python is installed in the container"
+    if venv_tar is not None:
+        venv_python_bin = os.path.join(venv_dest_path, venv_name, "bin", "python")
+        dest_venv_tar = os.path.join(venv_dest_path, os.path.basename(venv_tar))
+        copy_to(name, venv_tar, dest_venv_tar, overwrite=True, makedirs=True)
+        run_all(
+            name,
+            subprocess.list2cmdline(
+                ["tar", "zxf", dest_venv_tar, "-C", venv_dest_path]
+            ),
+        )
+        run_all(name, subprocess.list2cmdline(["rm", "-f", dest_venv_tar]))
+        container_python_bin = venv_python_bin
+        thin_dest_path = os.path.join(venv_dest_path, venv_name)
+        thin_salt_call = os.path.join(thin_dest_path, "bin", "salt-call")
+    else:
+        # move salt into the container
+        thin_path = __utils__["thin.gen_thin"](
+            __opts__["cachedir"],
+            extra_mods=__salt__["config.option"]("thin_extra_mods", ""),
+            so_mods=__salt__["config.option"]("thin_so_mods", ""),
         )
 
-    # untar archive
-    untar_cmd = [
-        container_python_bin,
-        "-c",
-        'import tarfile; tarfile.open("{0}/{1}").extractall(path="{0}")'.format(
-            thin_dest_path, os.path.basename(thin_path)
-        ),
-    ]
-    ret = run_all(name, subprocess.list2cmdline(untar_cmd))
-    if ret["retcode"] != 0:
-        return {"result": False, "comment": ret["stderr"]}
+        ret = copy_to(
+            name, thin_path, os.path.join(thin_dest_path, os.path.basename(thin_path))
+        )
+
+        # figure out available python interpreter inside the container (only Python3)
+        pycmds = ("python3", "/usr/libexec/platform-python")
+        container_python_bin = None
+        for py_cmd in pycmds:
+            cmd = [py_cmd] + ["--version"]
+            ret = run_all(name, subprocess.list2cmdline(cmd))
+            if ret["retcode"] == 0:
+                container_python_bin = py_cmd
+                break
+        if not container_python_bin:
+            raise CommandExecutionError(
+                "Python interpreter cannot be found inside the container. Make sure Python is installed in the container"
+            )
+
+        # untar archive
+        untar_cmd = [
+            container_python_bin,
+            "-c",
+            'import tarfile; tarfile.open("{0}/{1}").extractall(path="{0}")'.format(
+                thin_dest_path, os.path.basename(thin_path)
+            ),
+        ]
+        ret = run_all(name, subprocess.list2cmdline(untar_cmd))
+        if ret["retcode"] != 0:
+            return {"result": False, "comment": ret["stderr"]}
+        thin_salt_call = os.path.join(thin_dest_path, "salt-call")
 
     try:
         salt_argv = (
             [
                 container_python_bin,
-                os.path.join(thin_dest_path, "salt-call"),
+                thin_salt_call,
                 "--metadata",
                 "--local",
                 "--log-file",
diff --git a/tests/pytests/unit/modules/dockermod/test_module.py b/tests/pytests/unit/modules/dockermod/test_module.py
index 8fb7806497..1ac7dff52a 100644
--- a/tests/pytests/unit/modules/dockermod/test_module.py
+++ b/tests/pytests/unit/modules/dockermod/test_module.py
@@ -3,6 +3,7 @@ Unit tests for the docker module
 """
 
 import logging
+import sys
 
 import pytest
 
@@ -26,6 +27,7 @@ def configure_loader_modules(minion_opts):
         whitelist=[
             "args",
             "docker",
+            "files",
             "json",
             "state",
             "thin",
@@ -880,13 +882,16 @@ def test_call_success():
     client = Mock()
     client.put_archive = Mock()
     get_client_mock = MagicMock(return_value=client)
+    gen_venv_tar_mock = MagicMock(return_value=None)
 
     context = {"docker.exec_driver": "docker-exec"}
     salt_dunder = {"config.option": docker_config_mock}
 
     with patch.object(docker_mod, "run_all", docker_run_all_mock), patch.object(
         docker_mod, "copy_to", docker_copy_to_mock
-    ), patch.object(docker_mod, "_get_client", get_client_mock), patch.dict(
+    ), patch.object(docker_mod, "_get_client", get_client_mock), patch.object(
+        docker_mod, "gen_venv_tar", gen_venv_tar_mock
+    ), patch.dict(
         docker_mod.__opts__, {"cachedir": "/tmp"}
     ), patch.dict(
         docker_mod.__salt__, salt_dunder
@@ -931,6 +936,11 @@ def test_call_success():
         != docker_run_all_mock.mock_calls[9][1][1]
     )
 
+    # check the parameters of gen_venv_tar call
+    assert gen_venv_tar_mock.mock_calls[0][1][0] == "/tmp"
+    assert gen_venv_tar_mock.mock_calls[0][1][1] == "/var/tmp"
+    assert gen_venv_tar_mock.mock_calls[0][1][2] == "venv-salt-minion"
+
     assert {"retcode": 0, "comment": "container cmd"} == ret
 
 
@@ -1352,3 +1362,69 @@ def test_port():
             "bar": {"6666/tcp": ports["bar"]["6666/tcp"]},
             "baz": {},
         }
+
+
+@pytest.mark.slow_test
+def test_call_with_gen_venv_tar():
+    """
+    test module calling inside containers with the Salt Bundle
+    """
+    ret = None
+    docker_run_all_mock = MagicMock(
+        return_value={
+            "retcode": 0,
+            "stdout": '{"retcode": 0, "comment": "container cmd"}',
+            "stderr": "err",
+        }
+    )
+    docker_copy_to_mock = MagicMock(return_value={"retcode": 0})
+    docker_config_mock = MagicMock(return_value="")
+    docker_cmd_run_mock = MagicMock(
+        return_value={
+            "retcode": 0,
+            "stdout": "test",
+        }
+    )
+    client = Mock()
+    client.put_archive = Mock()
+    get_client_mock = MagicMock(return_value=client)
+
+    context = {"docker.exec_driver": "docker-exec"}
+    salt_dunder = {
+        "config.option": docker_config_mock,
+        "cmd.run_all": docker_cmd_run_mock,
+    }
+
+    with patch.object(docker_mod, "run_all", docker_run_all_mock), patch.object(
+        docker_mod, "copy_to", docker_copy_to_mock
+    ), patch.object(docker_mod, "_get_client", get_client_mock), patch.object(
+        sys, "executable", "/tmp/venv-salt-minion/bin/python"
+    ), patch.dict(
+        docker_mod.__opts__, {"cachedir": "/tmp"}
+    ), patch.dict(
+        docker_mod.__salt__, salt_dunder
+    ), patch.dict(
+        docker_mod.__context__, context
+    ):
+        ret = docker_mod.call("ID", "test.arg", 1, 2, arg1="val1")
+
+    # Check that the directory is different each time
+    # [ call(name, [args]), ...
+    assert "mkdir" in docker_run_all_mock.mock_calls[0][1][1]
+
+    assert (
+        "tar zxf /var/tmp/venv-salt.tgz -C /var/tmp"
+        == docker_run_all_mock.mock_calls[1][1][1]
+    )
+
+    assert docker_run_all_mock.mock_calls[3][1][1].startswith(
+        "/var/tmp/venv-salt-minion/bin/python /var/tmp/venv-salt-minion/bin/salt-call "
+    )
+
+    # check remove the salt bundle tarball
+    assert docker_run_all_mock.mock_calls[2][1][1] == "rm -f /var/tmp/venv-salt.tgz"
+
+    # check directory cleanup
+    assert docker_run_all_mock.mock_calls[4][1][1] == "rm -rf /var/tmp/venv-salt-minion"
+
+    assert {"retcode": 0, "comment": "container cmd"} == ret
-- 
2.39.2

Places

File use-salt-bundle-in-dockermod.patch of Package salt (Revision 8f61a1108956c9d5977363a990af94dd)

Places