Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
Please login to access the resource
home:tulip
salt
fix-gitfs-__env__-and-improve-cache-cleaning-bs...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch of Package salt
From a7c98ce490833ff232946b9715909161b6ba5a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= <psuarezhernandez@suse.com> Date: Mon, 13 Nov 2023 11:24:09 +0000 Subject: [PATCH] Fix gitfs "__env__" and improve cache cleaning (bsc#1193948) (#608) * Fix __env__ and improve cache cleaning * Move gitfs locks out of cachedir/.git --------- Co-authored-by: cmcmarrow <charles.mcmarrow.4@gmail.com> --- changelog/65002.fixed.md | 1 + changelog/65086.fixed.md | 1 + salt/utils/cache.py | 32 ++ salt/utils/gitfs.py | 307 +++++++++------ .../functional/pillar/test_git_pillar.py | 262 +++++++++++++ tests/pytests/functional/utils/test_cache.py | 83 ++++ tests/pytests/functional/utils/test_gitfs.py | 275 +++++++++++++ tests/pytests/functional/utils/test_pillar.py | 365 ++++++++++++++++++ .../pytests/functional/utils/test_winrepo.py | 164 ++++++++ tests/pytests/unit/test_minion.py | 31 +- tests/pytests/unit/utils/test_gitfs.py | 18 +- tests/unit/utils/test_gitfs.py | 21 +- 12 files changed, 1393 insertions(+), 167 deletions(-) create mode 100644 changelog/65002.fixed.md create mode 100644 changelog/65086.fixed.md create mode 100644 tests/pytests/functional/pillar/test_git_pillar.py create mode 100644 tests/pytests/functional/utils/test_cache.py create mode 100644 tests/pytests/functional/utils/test_gitfs.py create mode 100644 tests/pytests/functional/utils/test_pillar.py create mode 100644 tests/pytests/functional/utils/test_winrepo.py diff --git a/changelog/65002.fixed.md b/changelog/65002.fixed.md new file mode 100644 index 0000000000..86ed2d4bcc --- /dev/null +++ b/changelog/65002.fixed.md @@ -0,0 +1 @@ +Fix __env__ and improve cache cleaning see more info at pull #65017. diff --git a/changelog/65086.fixed.md b/changelog/65086.fixed.md new file mode 100644 index 0000000000..292930f0fd --- /dev/null +++ b/changelog/65086.fixed.md @@ -0,0 +1 @@ +Moved gitfs locks to salt working dir to avoid lock wipes diff --git a/salt/utils/cache.py b/salt/utils/cache.py index a78a1f70fc..88e7fa2400 100644 --- a/salt/utils/cache.py +++ b/salt/utils/cache.py @@ -6,6 +6,7 @@ import functools import logging import os import re +import shutil import time import salt.config @@ -15,6 +16,8 @@ import salt.utils.data import salt.utils.dictupdate import salt.utils.files import salt.utils.msgpack +import salt.utils.path +import salt.version from salt.utils.zeromq import zmq log = logging.getLogger(__name__) @@ -345,3 +348,32 @@ def context_cache(func): return func(*args, **kwargs) return context_cache_wrap + + +def verify_cache_version(cache_path): + """ + Check that the cached version matches the Salt version. + If the cached version does not match the Salt version, wipe the cache. + + :return: ``True`` if cache version matches, otherwise ``False`` + """ + if not os.path.isdir(cache_path): + os.makedirs(cache_path) + with salt.utils.files.fopen( + salt.utils.path.join(cache_path, "cache_version"), "a+" + ) as file: + file.seek(0) + data = "\n".join(file.readlines()) + if data != salt.version.__version__: + log.warning(f"Cache version mismatch clearing: {repr(cache_path)}") + file.truncate(0) + file.write(salt.version.__version__) + for item in os.listdir(cache_path): + if item != "cache_version": + item_path = salt.utils.path.join(cache_path, item) + if os.path.isfile(item_path): + os.remove(item_path) + else: + shutil.rmtree(item_path) + return False + return True diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index af61aa0dda..061647edac 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -17,7 +17,6 @@ import os import shlex import shutil import stat -import string import subprocess import time import weakref @@ -26,6 +25,7 @@ from datetime import datetime import salt.ext.tornado.ioloop import salt.fileserver import salt.syspaths +import salt.utils.cache import salt.utils.configparser import salt.utils.data import salt.utils.files @@ -253,7 +253,6 @@ class GitProvider: val_cb=lambda x, y: str(y), ) self.conf = copy.deepcopy(per_remote_defaults) - # Remove the 'salt://' from the beginning of any globally-defined # per-saltenv mountpoints for saltenv, saltenv_conf in self.global_saltenv.items(): @@ -457,48 +456,38 @@ class GitProvider: self.id, ) failhard(self.role) - - hash_type = getattr(hashlib, self.opts.get("hash_type", "md5")) - # Generate full id. - # Full id helps decrease the chances of collections in the gitfs cache. - try: - target = str(self.get_checkout_target()) - except AttributeError: - target = "" - self._full_id = "-".join( - [ - getattr(self, "name", ""), - self.id, - getattr(self, "env", ""), - getattr(self, "_root", ""), - self.role, - getattr(self, "base", ""), - getattr(self, "branch", ""), - target, - ] + if hasattr(self, "name"): + self._cache_basehash = self.name + else: + hash_type = getattr(hashlib, self.opts.get("hash_type", "md5")) + # We loaded this data from yaml configuration files, so, its safe + # to use UTF-8 + self._cache_basehash = str( + base64.b64encode(hash_type(self.id.encode("utf-8")).digest()), + encoding="ascii", # base64 only outputs ascii + ).replace( + "/", "_" + ) # replace "/" with "_" to not cause trouble with file system + self._cache_hash = salt.utils.path.join(cache_root, self._cache_basehash) + self._cache_basename = "_" + if self.id.startswith("__env__"): + try: + self._cache_basename = self.get_checkout_target() + except AttributeError: + log.critical(f"__env__ cant generate basename: {self.role} {self.id}") + failhard(self.role) + self._cache_full_basename = salt.utils.path.join( + self._cache_basehash, self._cache_basename ) - # We loaded this data from yaml configuration files, so, its safe - # to use UTF-8 - base64_hash = str( - base64.b64encode(hash_type(self._full_id.encode("utf-8")).digest()), - encoding="ascii", # base64 only outputs ascii - ).replace( - "/", "_" - ) # replace "/" with "_" to not cause trouble with file system - - # limit name length to 19, so we don't eat up all the path length for windows - # this is due to pygit2 limitations - # replace any unknown char with "_" to not cause trouble with file system - name_chars = string.ascii_letters + string.digits + "-" - cache_name = "".join( - c if c in name_chars else "_" for c in getattr(self, "name", "")[:19] + self._cachedir = salt.utils.path.join(self._cache_hash, self._cache_basename) + self._salt_working_dir = salt.utils.path.join( + cache_root, "work", self._cache_full_basename ) - - self.cachedir_basename = f"{cache_name}-{base64_hash}" - self.cachedir = salt.utils.path.join(cache_root, self.cachedir_basename) - self.linkdir = salt.utils.path.join(cache_root, "links", self.cachedir_basename) - if not os.path.isdir(self.cachedir): - os.makedirs(self.cachedir) + self._linkdir = salt.utils.path.join( + cache_root, "links", self._cache_full_basename + ) + if not os.path.isdir(self._cachedir): + os.makedirs(self._cachedir) try: self.new = self.init_remote() @@ -510,12 +499,32 @@ class GitProvider: msg += " Perhaps git is not available." log.critical(msg, exc_info=True) failhard(self.role) + self.verify_auth() + self.setup_callbacks() + if not os.path.isdir(self._salt_working_dir): + os.makedirs(self._salt_working_dir) + self.fetch_request_check() + + def get_cache_basehash(self): + return self._cache_basehash + + def get_cache_hash(self): + return self._cache_hash - def full_id(self): - return self._full_id + def get_cache_basename(self): + return self._cache_basename - def get_cachedir_basename(self): - return self.cachedir_basename + def get_cache_full_basename(self): + return self._cache_full_basename + + def get_cachedir(self): + return self._cachedir + + def get_linkdir(self): + return self._linkdir + + def get_salt_working_dir(self): + return self._salt_working_dir def _get_envs_from_ref_paths(self, refs): """ @@ -557,7 +566,7 @@ class GitProvider: return ret def _get_lock_file(self, lock_type="update"): - return salt.utils.path.join(self.gitdir, lock_type + ".lk") + return salt.utils.path.join(self._salt_working_dir, lock_type + ".lk") @classmethod def add_conf_overlay(cls, name): @@ -644,7 +653,7 @@ class GitProvider: # No need to pass an environment to self.root() here since per-saltenv # configuration is a gitfs-only feature and check_root() is not used # for gitfs. - root_dir = salt.utils.path.join(self.cachedir, self.root()).rstrip(os.sep) + root_dir = salt.utils.path.join(self._cachedir, self.root()).rstrip(os.sep) if os.path.isdir(root_dir): return root_dir log.error( @@ -816,7 +825,7 @@ class GitProvider: desired_refspecs, ) if refspecs != desired_refspecs: - conf.set_multivar(remote_section, "fetch", self.refspecs) + conf.set_multivar(remote_section, "fetch", desired_refspecs) log.debug( "Refspecs for %s remote '%s' set to %s", self.role, @@ -1069,7 +1078,7 @@ class GitProvider: """ raise NotImplementedError() - def checkout(self): + def checkout(self, fetch_on_fail=True): """ This function must be overridden in a sub-class """ @@ -1192,6 +1201,21 @@ class GitProvider: else: self.url = self.id + def fetch_request_check(self): + fetch_request = salt.utils.path.join(self._salt_working_dir, "fetch_request") + if os.path.isfile(fetch_request): + log.debug(f"Fetch request: {self._salt_working_dir}") + try: + os.remove(fetch_request) + except OSError as exc: + log.error( + f"Failed to remove Fetch request: {self._salt_working_dir} {exc}", + exc_info=True, + ) + self.fetch() + return True + return False + @property def linkdir_walk(self): """ @@ -1218,14 +1242,14 @@ class GitProvider: dirs = [] self._linkdir_walk.append( ( - salt.utils.path.join(self.linkdir, *parts[: idx + 1]), + salt.utils.path.join(self._linkdir, *parts[: idx + 1]), dirs, [], ) ) try: # The linkdir itself goes at the beginning - self._linkdir_walk.insert(0, (self.linkdir, [parts[0]], [])) + self._linkdir_walk.insert(0, (self._linkdir, [parts[0]], [])) except IndexError: pass return self._linkdir_walk @@ -1275,13 +1299,17 @@ class GitPython(GitProvider): role, ) - def checkout(self): + def checkout(self, fetch_on_fail=True): """ Checkout the configured branch/tag. We catch an "Exception" class here instead of a specific exception class because the exceptions raised by GitPython when running these functions vary in different versions of GitPython. + + fetch_on_fail + If checkout fails perform a fetch then try to checkout again. """ + self.fetch_request_check() tgt_ref = self.get_checkout_target() try: head_sha = self.repo.rev_parse("HEAD").hexsha @@ -1345,6 +1373,15 @@ class GitPython(GitProvider): except Exception: # pylint: disable=broad-except continue return self.check_root() + if fetch_on_fail: + log.debug( + "Failed to checkout %s from %s remote '%s': fetch and try again", + tgt_ref, + self.role, + self.id, + ) + self.fetch() + return self.checkout(fetch_on_fail=False) log.error( "Failed to checkout %s from %s remote '%s': remote ref does not exist", tgt_ref, @@ -1360,16 +1397,16 @@ class GitPython(GitProvider): initialized by this function. """ new = False - if not os.listdir(self.cachedir): + if not os.listdir(self._cachedir): # Repo cachedir is empty, initialize a new repo there - self.repo = git.Repo.init(self.cachedir) + self.repo = git.Repo.init(self._cachedir) new = True else: # Repo cachedir exists, try to attach try: - self.repo = git.Repo(self.cachedir) + self.repo = git.Repo(self._cachedir) except git.exc.InvalidGitRepositoryError: - log.error(_INVALID_REPO, self.cachedir, self.url, self.role) + log.error(_INVALID_REPO, self._cachedir, self.url, self.role) return new self.gitdir = salt.utils.path.join(self.repo.working_dir, ".git") @@ -1603,10 +1640,14 @@ class Pygit2(GitProvider): except AttributeError: return obj.get_object() - def checkout(self): + def checkout(self, fetch_on_fail=True): """ Checkout the configured branch/tag + + fetch_on_fail + If checkout fails perform a fetch then try to checkout again. """ + self.fetch_request_check() tgt_ref = self.get_checkout_target() local_ref = "refs/heads/" + tgt_ref remote_ref = "refs/remotes/origin/" + tgt_ref @@ -1796,6 +1837,15 @@ class Pygit2(GitProvider): exc_info=True, ) return None + if fetch_on_fail: + log.debug( + "Failed to checkout %s from %s remote '%s': fetch and try again", + tgt_ref, + self.role, + self.id, + ) + self.fetch() + return self.checkout(fetch_on_fail=False) log.error( "Failed to checkout %s from %s remote '%s': remote ref does not exist", tgt_ref, @@ -1837,16 +1887,16 @@ class Pygit2(GitProvider): home = os.path.expanduser("~") pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = home new = False - if not os.listdir(self.cachedir): + if not os.listdir(self._cachedir): # Repo cachedir is empty, initialize a new repo there - self.repo = pygit2.init_repository(self.cachedir) + self.repo = pygit2.init_repository(self._cachedir) new = True else: # Repo cachedir exists, try to attach try: - self.repo = pygit2.Repository(self.cachedir) + self.repo = pygit2.Repository(self._cachedir) except KeyError: - log.error(_INVALID_REPO, self.cachedir, self.url, self.role) + log.error(_INVALID_REPO, self._cachedir, self.url, self.role) return new self.gitdir = salt.utils.path.join(self.repo.workdir, ".git") @@ -2370,6 +2420,7 @@ class GitBase: self.file_list_cachedir = salt.utils.path.join( self.opts["cachedir"], "file_lists", self.role ) + salt.utils.cache.verify_cache_version(self.cache_root) if init_remotes: self.init_remotes( remotes if remotes is not None else [], @@ -2442,8 +2493,6 @@ class GitBase: ) if hasattr(repo_obj, "repo"): # Sanity check and assign the credential parameter - repo_obj.verify_auth() - repo_obj.setup_callbacks() if self.opts["__role"] == "minion" and repo_obj.new: # Perform initial fetch on masterless minion repo_obj.fetch() @@ -2492,7 +2541,7 @@ class GitBase: # Don't allow collisions in cachedir naming cachedir_map = {} for repo in self.remotes: - cachedir_map.setdefault(repo.cachedir, []).append(repo.id) + cachedir_map.setdefault(repo.get_cachedir(), []).append(repo.id) collisions = [x for x in cachedir_map if len(cachedir_map[x]) > 1] if collisions: @@ -2509,48 +2558,42 @@ class GitBase: if any(x.new for x in self.remotes): self.write_remote_map() + def _remove_cache_dir(self, cache_dir): + try: + shutil.rmtree(cache_dir) + except OSError as exc: + log.error( + "Unable to remove old %s remote cachedir %s: %s", + self.role, + cache_dir, + exc, + ) + return False + log.debug("%s removed old cachedir %s", self.role, cache_dir) + return True + + def _iter_remote_hashes(self): + for item in os.listdir(self.cache_root): + if item in ("hash", "refs", "links", "work"): + continue + if os.path.isdir(salt.utils.path.join(self.cache_root, item)): + yield item + def clear_old_remotes(self): """ Remove cache directories for remotes no longer configured """ - try: - cachedir_ls = os.listdir(self.cache_root) - except OSError: - cachedir_ls = [] - # Remove actively-used remotes from list - for repo in self.remotes: - try: - cachedir_ls.remove(repo.cachedir_basename) - except ValueError: - pass - to_remove = [] - for item in cachedir_ls: - if item in ("hash", "refs"): - continue - path = salt.utils.path.join(self.cache_root, item) - if os.path.isdir(path): - to_remove.append(path) - failed = [] - if to_remove: - for rdir in to_remove: - try: - shutil.rmtree(rdir) - except OSError as exc: - log.error( - "Unable to remove old %s remote cachedir %s: %s", - self.role, - rdir, - exc, - ) - failed.append(rdir) - else: - log.debug("%s removed old cachedir %s", self.role, rdir) - for fdir in failed: - to_remove.remove(fdir) - ret = bool(to_remove) - if ret: + change = False + # Remove all hash dirs not part of this group + remote_set = {r.get_cache_basehash() for r in self.remotes} + for item in self._iter_remote_hashes(): + if item not in remote_set: + change = self._remove_cache_dir( + salt.utils.path.join(self.cache_root, item) or change + ) + if not change: self.write_remote_map() - return ret + return change def clear_cache(self): """ @@ -2609,6 +2652,27 @@ class GitBase: name = getattr(repo, "name", None) if not remotes or (repo.id, name) in remotes or name in remotes: try: + # Find and place fetch_request file for all the other branches for this repo + repo_work_hash = os.path.split(repo.get_salt_working_dir())[0] + for branch in os.listdir(repo_work_hash): + # Don't place fetch request in current branch being updated + if branch == repo.get_cache_basename(): + continue + branch_salt_dir = salt.utils.path.join(repo_work_hash, branch) + fetch_path = salt.utils.path.join( + branch_salt_dir, "fetch_request" + ) + if os.path.isdir(branch_salt_dir): + try: + with salt.utils.files.fopen(fetch_path, "w"): + pass + except OSError as exc: # pylint: disable=broad-except + log.error( + f"Failed to make fetch request: {fetch_path} {exc}", + exc_info=True, + ) + else: + log.error(f"Failed to make fetch request: {fetch_path}") if repo.fetch(): # We can't just use the return value from repo.fetch() # because the data could still have changed if old @@ -2863,7 +2927,7 @@ class GitBase: for repo in self.remotes: fp_.write( salt.utils.stringutils.to_str( - "{} = {}\n".format(repo.cachedir_basename, repo.id) + "{} = {}\n".format(repo.get_cache_basehash(), repo.id) ) ) except OSError: @@ -2871,15 +2935,18 @@ class GitBase: else: log.info("Wrote new %s remote map to %s", self.role, remote_map) - def do_checkout(self, repo): + def do_checkout(self, repo, fetch_on_fail=True): """ Common code for git_pillar/winrepo to handle locking and checking out of a repo. + + fetch_on_fail + If checkout fails perform a fetch then try to checkout again. """ time_start = time.time() while time.time() - time_start <= 5: try: - return repo.checkout() + return repo.checkout(fetch_on_fail=fetch_on_fail) except GitLockError as exc: if exc.errno == errno.EEXIST: time.sleep(0.1) @@ -3274,14 +3341,17 @@ class GitPillar(GitBase): role = "git_pillar" - def checkout(self): + def checkout(self, fetch_on_fail=True): """ Checkout the targeted branches/tags from the git_pillar remotes + + fetch_on_fail + If checkout fails perform a fetch then try to checkout again. """ self.pillar_dirs = OrderedDict() self.pillar_linked_dirs = [] for repo in self.remotes: - cachedir = self.do_checkout(repo) + cachedir = self.do_checkout(repo, fetch_on_fail=fetch_on_fail) if cachedir is not None: # Figure out which environment this remote should be assigned if repo.branch == "__env__" and hasattr(repo, "all_saltenvs"): @@ -3298,8 +3368,8 @@ class GitPillar(GitBase): env = "base" if tgt == repo.base else tgt if repo._mountpoint: if self.link_mountpoint(repo): - self.pillar_dirs[repo.linkdir] = env - self.pillar_linked_dirs.append(repo.linkdir) + self.pillar_dirs[repo.get_linkdir()] = env + self.pillar_linked_dirs.append(repo.get_linkdir()) else: self.pillar_dirs[cachedir] = env @@ -3308,17 +3378,19 @@ class GitPillar(GitBase): Ensure that the mountpoint is present in the correct location and points at the correct path """ - lcachelink = salt.utils.path.join(repo.linkdir, repo._mountpoint) - lcachedest = salt.utils.path.join(repo.cachedir, repo.root()).rstrip(os.sep) + lcachelink = salt.utils.path.join(repo.get_linkdir(), repo._mountpoint) + lcachedest = salt.utils.path.join(repo.get_cachedir(), repo.root()).rstrip( + os.sep + ) wipe_linkdir = False create_link = False try: with repo.gen_lock(lock_type="mountpoint", timeout=10): - walk_results = list(os.walk(repo.linkdir, followlinks=False)) + walk_results = list(os.walk(repo.get_linkdir(), followlinks=False)) if walk_results != repo.linkdir_walk: log.debug( "Results of walking %s differ from expected results", - repo.linkdir, + repo.get_linkdir(), ) log.debug("Walk results: %s", walk_results) log.debug("Expected results: %s", repo.linkdir_walk) @@ -3379,7 +3451,7 @@ class GitPillar(GitBase): # Wiping implies that we need to create the link create_link = True try: - shutil.rmtree(repo.linkdir) + shutil.rmtree(repo.get_linkdir()) except OSError: pass try: @@ -3431,6 +3503,9 @@ class GitPillar(GitBase): class WinRepo(GitBase): """ Functionality specific to the winrepo runner + + fetch_on_fail + If checkout fails perform a fetch then try to checkout again. """ role = "winrepo" @@ -3438,12 +3513,12 @@ class WinRepo(GitBase): # out the repos. winrepo_dirs = {} - def checkout(self): + def checkout(self, fetch_on_fail=True): """ Checkout the targeted branches/tags from the winrepo remotes """ self.winrepo_dirs = {} for repo in self.remotes: - cachedir = self.do_checkout(repo) + cachedir = self.do_checkout(repo, fetch_on_fail=fetch_on_fail) if cachedir is not None: self.winrepo_dirs[repo.id] = cachedir diff --git a/tests/pytests/functional/pillar/test_git_pillar.py b/tests/pytests/functional/pillar/test_git_pillar.py new file mode 100644 index 0000000000..6fd3dee431 --- /dev/null +++ b/tests/pytests/functional/pillar/test_git_pillar.py @@ -0,0 +1,262 @@ +import pytest + +from salt.pillar.git_pillar import ext_pillar +from salt.utils.immutabletypes import ImmutableDict, ImmutableList +from tests.support.mock import patch + +pytestmark = [ + pytest.mark.slow_test, +] + + +try: + import git # pylint: disable=unused-import + + HAS_GITPYTHON = True +except ImportError: + HAS_GITPYTHON = False + + +try: + import pygit2 # pylint: disable=unused-import + + HAS_PYGIT2 = True +except ImportError: + HAS_PYGIT2 = False + + +skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") +skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") + + +@pytest.fixture +def git_pillar_opts(salt_master, tmp_path): + opts = dict(salt_master.config) + opts["cachedir"] = str(tmp_path) + for key, item in opts.items(): + if isinstance(item, ImmutableDict): + opts[key] = dict(item) + elif isinstance(item, ImmutableList): + opts[key] = list(item) + return opts + + +@pytest.fixture +def gitpython_pillar_opts(git_pillar_opts): + git_pillar_opts["verified_git_pillar_provider"] = "gitpython" + return git_pillar_opts + + +@pytest.fixture +def pygit2_pillar_opts(git_pillar_opts): + git_pillar_opts["verified_git_pillar_provider"] = "pygit2" + return git_pillar_opts + + +def _get_ext_pillar(minion, pillar_opts, grains, *repos): + with patch("salt.pillar.git_pillar.__opts__", pillar_opts, create=True): + with patch("salt.pillar.git_pillar.__grains__", grains, create=True): + return ext_pillar(minion, None, *repos) + + +def _test_simple(pillar_opts, grains): + data = _get_ext_pillar( + "minion", + pillar_opts, + grains, + "https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + assert data == {"key": "value"} + + +@skipif_no_gitpython +def test_gitpython_simple(gitpython_pillar_opts, grains): + _test_simple(gitpython_pillar_opts, grains) + + +@skipif_no_pygit2 +def test_pygit2_simple(pygit2_pillar_opts, grains): + _test_simple(pygit2_pillar_opts, grains) + + +def _test_missing_env(pillar_opts, grains): + data = _get_ext_pillar( + "minion", + pillar_opts, + grains, + { + "https://github.com/saltstack/salt-test-pillar-gitfs.git": [ + {"env": "misssing"} + ] + }, + ) + assert data == {} + + +@skipif_no_gitpython +def test_gitpython_missing_env(gitpython_pillar_opts, grains): + _test_missing_env(gitpython_pillar_opts, grains) + + +@skipif_no_pygit2 +def test_pygit2_missing_env(pygit2_pillar_opts, grains): + _test_missing_env(pygit2_pillar_opts, grains) + + +def _test_env(pillar_opts, grains): + data = _get_ext_pillar( + "minion", + pillar_opts, + grains, + { + "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git": [ + {"env": "other_env"} + ] + }, + ) + assert data == {"other": "env"} + + +@skipif_no_gitpython +def test_gitpython_env(gitpython_pillar_opts, grains): + _test_env(gitpython_pillar_opts, grains) + + +@skipif_no_pygit2 +def test_pygit2_env(pygit2_pillar_opts, grains): + _test_env(pygit2_pillar_opts, grains) + + +def _test_branch(pillar_opts, grains): + data = _get_ext_pillar( + "minion", + pillar_opts, + grains, + "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + assert data == {"key": "data"} + + +@skipif_no_gitpython +def test_gitpython_branch(gitpython_pillar_opts, grains): + _test_branch(gitpython_pillar_opts, grains) + + +@skipif_no_pygit2 +def test_pygit2_branch(pygit2_pillar_opts, grains): + _test_branch(pygit2_pillar_opts, grains) + + +def _test_simple_dynamic(pillar_opts, grains): + data = _get_ext_pillar( + "minion", + pillar_opts, + grains, + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + assert data == {"key": "value"} + + +@skipif_no_gitpython +def test_gitpython_simple_dynamic(gitpython_pillar_opts, grains): + _test_simple_dynamic(gitpython_pillar_opts, grains) + + +@skipif_no_pygit2 +def test_pygit2_simple_dynamic(pygit2_pillar_opts, grains): + _test_simple_dynamic(pygit2_pillar_opts, grains) + + +def _test_missing_env_dynamic(pillar_opts, grains): + data = _get_ext_pillar( + "minion", + pillar_opts, + grains, + { + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git": [ + {"env": "misssing"} + ] + }, + ) + assert data == {} + + +@skipif_no_gitpython +def test_gitpython_missing_env_dynamic(gitpython_pillar_opts, grains): + _test_missing_env_dynamic(gitpython_pillar_opts, grains) + + +@skipif_no_pygit2 +def test_pygit2_missing_env_dynamic(pygit2_pillar_opts, grains): + _test_missing_env_dynamic(pygit2_pillar_opts, grains) + + +def _test_pillarenv_dynamic(pillar_opts, grains): + pillar_opts["pillarenv"] = "branch" + data = _get_ext_pillar( + "minion", + pillar_opts, + grains, + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + assert data == {"key": "data"} + + +@skipif_no_gitpython +def test_gitpython_pillarenv_dynamic(gitpython_pillar_opts, grains): + _test_pillarenv_dynamic(gitpython_pillar_opts, grains) + + +@skipif_no_pygit2 +def test_pygit2_pillarenv_dynamic(pygit2_pillar_opts, grains): + _test_pillarenv_dynamic(pygit2_pillar_opts, grains) + + +def _test_multiple(pillar_opts, grains): + pillar_opts["pillarenv"] = "branch" + data = _get_ext_pillar( + "minion", + pillar_opts, + grains, + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", + "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + ) + assert data == {"key": "data"} + + +@skipif_no_gitpython +def test_gitpython_multiple(gitpython_pillar_opts, grains): + _test_multiple(gitpython_pillar_opts, grains) + + +@skipif_no_pygit2 +def test_pygit2_multiple(pygit2_pillar_opts, grains): + _test_multiple(pygit2_pillar_opts, grains) + + +def _test_multiple_2(pillar_opts, grains): + data = _get_ext_pillar( + "minion", + pillar_opts, + grains, + "https://github.com/saltstack/salt-test-pillar-gitfs.git", + "https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + ) + assert data == { + "key": "value", + "key1": "value1", + "key2": "value2", + "key4": "value4", + "data1": "d", + "data2": "d2", + } + + +@skipif_no_gitpython +def test_gitpython_multiple_2(gitpython_pillar_opts, grains): + _test_multiple_2(gitpython_pillar_opts, grains) + + +@skipif_no_pygit2 +def test_pygit2_multiple_2(pygit2_pillar_opts, grains): + _test_multiple_2(pygit2_pillar_opts, grains) diff --git a/tests/pytests/functional/utils/test_cache.py b/tests/pytests/functional/utils/test_cache.py new file mode 100644 index 0000000000..d405b8246f --- /dev/null +++ b/tests/pytests/functional/utils/test_cache.py @@ -0,0 +1,83 @@ +import os + +import pytest + +import salt.utils.cache +import salt.utils.files +import salt.utils.path +import salt.version + +_DUMMY_FILES = ( + "data.txt", + "foo.t2", + "bar.t3", + "nested/test", + "nested/cache.txt", + "n/n1/n2/n3/n4/n5", +) + + +def _make_dummy_files(tmp_path): + for full_path in _DUMMY_FILES: + full_path = salt.utils.path.join(tmp_path, full_path) + path, _ = os.path.split(full_path) + if not os.path.isdir(path): + os.makedirs(path) + with salt.utils.files.fopen(full_path, "w") as file: + file.write("data") + + +def _dummy_files_exists(tmp_path): + """ + True if all files exists + False if all files are missing + None if some files exists and others are missing + """ + ret = None + for full_path in _DUMMY_FILES: + full_path = salt.utils.path.join(tmp_path, full_path) + is_file = os.path.isfile(full_path) + if ret is None: + ret = is_file + elif ret is not is_file: + return None # Some files are found and others are missing + return ret + + +def test_verify_cache_version_bad_path(): + with pytest.raises(ValueError): + # cache version should fail if given bad file python + salt.utils.cache.verify_cache_version("\0/bad/path") + + +def test_verify_cache_version(tmp_path): + # cache version should make dir if it does not exist + tmp_path = str(salt.utils.path.join(str(tmp_path), "work", "salt")) + cache_version = salt.utils.path.join(tmp_path, "cache_version") + + # check that cache clears when no cache_version is present + _make_dummy_files(tmp_path) + assert salt.utils.cache.verify_cache_version(tmp_path) is False + assert _dummy_files_exists(tmp_path) is False + + # check that cache_version has correct salt version + with salt.utils.files.fopen(cache_version, "r") as file: + assert "\n".join(file.readlines()) == salt.version.__version__ + + # check that cache does not get clear when check is called multiple times + _make_dummy_files(tmp_path) + for _ in range(3): + assert salt.utils.cache.verify_cache_version(tmp_path) is True + assert _dummy_files_exists(tmp_path) is True + + # check that cache clears when a different version is present + with salt.utils.files.fopen(cache_version, "w") as file: + file.write("-1") + assert salt.utils.cache.verify_cache_version(tmp_path) is False + assert _dummy_files_exists(tmp_path) is False + + # check that cache does not get clear when check is called multiple times + _make_dummy_files(tmp_path) + for _ in range(3): + assert salt.utils.cache.verify_cache_version(tmp_path) is True + assert _dummy_files_exists(tmp_path) is True diff --git a/tests/pytests/functional/utils/test_gitfs.py b/tests/pytests/functional/utils/test_gitfs.py new file mode 100644 index 0000000000..30a5f147fa --- /dev/null +++ b/tests/pytests/functional/utils/test_gitfs.py @@ -0,0 +1,275 @@ +import os.path + +import pytest + +from salt.fileserver.gitfs import PER_REMOTE_ONLY, PER_REMOTE_OVERRIDES +from salt.utils.gitfs import GitFS, GitPython, Pygit2 +from salt.utils.immutabletypes import ImmutableDict, ImmutableList + +pytestmark = [ + pytest.mark.slow_test, +] + + +try: + import git # pylint: disable=unused-import + + HAS_GITPYTHON = True +except ImportError: + HAS_GITPYTHON = False + + +try: + import pygit2 # pylint: disable=unused-import + + HAS_PYGIT2 = True +except ImportError: + HAS_PYGIT2 = False + + +skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") +skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") + + +@pytest.fixture +def gitfs_opts(salt_factories, tmp_path): + config_defaults = {"cachedir": str(tmp_path)} + factory = salt_factories.salt_master_daemon( + "gitfs-functional-master", defaults=config_defaults + ) + config_defaults = dict(factory.config) + for key, item in config_defaults.items(): + if isinstance(item, ImmutableDict): + config_defaults[key] = dict(item) + elif isinstance(item, ImmutableList): + config_defaults[key] = list(item) + return config_defaults + + +@pytest.fixture +def gitpython_gitfs_opts(gitfs_opts): + gitfs_opts["verified_gitfs_provider"] = "gitpython" + GitFS.instance_map.clear() # wipe instance_map object map for clean run + return gitfs_opts + + +@pytest.fixture +def pygit2_gitfs_opts(gitfs_opts): + gitfs_opts["verified_gitfs_provider"] = "pygit2" + GitFS.instance_map.clear() # wipe instance_map object map for clean run + return gitfs_opts + + +def _get_gitfs(opts, *remotes): + return GitFS( + opts, + remotes, + per_remote_overrides=PER_REMOTE_OVERRIDES, + per_remote_only=PER_REMOTE_ONLY, + ) + + +def _test_gitfs_simple(gitfs_opts): + g = _get_gitfs( + gitfs_opts, + {"https://github.com/saltstack/salt-test-pillar-gitfs.git": [{"name": "bob"}]}, + ) + g.fetch_remotes() + assert len(g.remotes) == 1 + assert set(g.file_list({"saltenv": "main"})) == {".gitignore", "README.md"} + + +@skipif_no_gitpython +def test_gitpython_gitfs_simple(gitpython_gitfs_opts): + _test_gitfs_simple(gitpython_gitfs_opts) + + +@skipif_no_pygit2 +def test_pygit2_gitfs_simple(pygit2_gitfs_opts): + _test_gitfs_simple(pygit2_gitfs_opts) + + +def _test_gitfs_simple_base(gitfs_opts): + g = _get_gitfs( + gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + g.fetch_remotes() + assert len(g.remotes) == 1 + assert set(g.file_list({"saltenv": "base"})) == { + ".gitignore", + "README.md", + "file.sls", + "top.sls", + } + + +@skipif_no_gitpython +def test_gitpython_gitfs_simple_base(gitpython_gitfs_opts): + _test_gitfs_simple_base(gitpython_gitfs_opts) + + +@skipif_no_pygit2 +def test_pygit2_gitfs_simple_base(pygit2_gitfs_opts): + _test_gitfs_simple_base(pygit2_gitfs_opts) + + +@skipif_no_gitpython +def test_gitpython_gitfs_provider(gitpython_gitfs_opts): + g = _get_gitfs( + gitpython_gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + assert len(g.remotes) == 1 + assert g.provider == "gitpython" + assert isinstance(g.remotes[0], GitPython) + + +@skipif_no_pygit2 +def test_pygit2_gitfs_provider(pygit2_gitfs_opts): + g = _get_gitfs( + pygit2_gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + assert len(g.remotes) == 1 + assert g.provider == "pygit2" + assert isinstance(g.remotes[0], Pygit2) + + +def _test_gitfs_minion(gitfs_opts): + gitfs_opts["__role"] = "minion" + g = _get_gitfs( + gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + g.fetch_remotes() + assert len(g.remotes) == 1 + assert set(g.file_list({"saltenv": "base"})) == { + ".gitignore", + "README.md", + "file.sls", + "top.sls", + } + assert set(g.file_list({"saltenv": "main"})) == {".gitignore", "README.md"} + + +@skipif_no_gitpython +def test_gitpython_gitfs_minion(gitpython_gitfs_opts): + _test_gitfs_minion(gitpython_gitfs_opts) + + +@skipif_no_pygit2 +def test_pygit2_gitfs_minion(pygit2_gitfs_opts): + _test_gitfs_minion(pygit2_gitfs_opts) + + +def _test_fetch_request_with_mountpoint(opts): + mpoint = [{"mountpoint": "salt/m"}] + p = _get_gitfs( + opts, + {"https://github.com/saltstack/salt-test-pillar-gitfs.git": mpoint}, + ) + p.fetch_remotes() + assert len(p.remotes) == 1 + repo = p.remotes[0] + assert repo.mountpoint("testmount") == "salt/m" + assert set(p.file_list({"saltenv": "testmount"})) == { + "salt/m/test_dir1/testfile3", + "salt/m/test_dir1/test_dir2/testfile2", + "salt/m/.gitignore", + "salt/m/README.md", + "salt/m/test_dir1/test_dir2/testfile1", + } + + +@skipif_no_gitpython +def test_gitpython_fetch_request_with_mountpoint(gitpython_gitfs_opts): + _test_fetch_request_with_mountpoint(gitpython_gitfs_opts) + + +@skipif_no_pygit2 +def test_pygit2_fetch_request_with_mountpoint(pygit2_gitfs_opts): + _test_fetch_request_with_mountpoint(pygit2_gitfs_opts) + + +def _test_name(opts): + p = _get_gitfs( + opts, + { + "https://github.com/saltstack/salt-test-pillar-gitfs.git": [ + {"name": "name1"} + ] + }, + { + "https://github.com/saltstack/salt-test-pillar-gitfs.git": [ + {"name": "name2"} + ] + }, + ) + p.fetch_remotes() + assert len(p.remotes) == 2 + repo = p.remotes[0] + repo2 = p.remotes[1] + assert repo.get_cache_basehash() == "name1" + assert repo2.get_cache_basehash() == "name2" + + +@skipif_no_gitpython +def test_gitpython_name(gitpython_gitfs_opts): + _test_name(gitpython_gitfs_opts) + + +@skipif_no_pygit2 +def test_pygit2_name(pygit2_gitfs_opts): + _test_name(pygit2_gitfs_opts) + + +def _test_remote_map(opts): + p = _get_gitfs( + opts, + "https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + p.fetch_remotes() + assert len(p.remotes) == 1 + assert os.path.isfile(os.path.join(opts["cachedir"], "gitfs", "remote_map.txt")) + + +@skipif_no_gitpython +def test_gitpython_remote_map(gitpython_gitfs_opts): + _test_remote_map(gitpython_gitfs_opts) + + +@skipif_no_pygit2 +def test_pygit2_remote_map(pygit2_gitfs_opts): + _test_remote_map(pygit2_gitfs_opts) + + +def _test_lock(opts): + g = _get_gitfs( + opts, + "https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + g.fetch_remotes() + assert len(g.remotes) == 1 + repo = g.remotes[0] + assert repo.get_salt_working_dir() in repo._get_lock_file() + assert repo.lock() == ( + [ + "Set update lock for gitfs remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" + ], + [], + ) + assert os.path.isfile(repo._get_lock_file()) + assert repo.clear_lock() == ( + [ + "Removed update lock for gitfs remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" + ], + [], + ) + assert not os.path.isfile(repo._get_lock_file()) + + +@skipif_no_gitpython +def test_gitpython_lock(gitpython_gitfs_opts): + _test_lock(gitpython_gitfs_opts) + + +@skipif_no_pygit2 +def test_pygit2_lock(pygit2_gitfs_opts): + _test_lock(pygit2_gitfs_opts) diff --git a/tests/pytests/functional/utils/test_pillar.py b/tests/pytests/functional/utils/test_pillar.py new file mode 100644 index 0000000000..143edbf6ff --- /dev/null +++ b/tests/pytests/functional/utils/test_pillar.py @@ -0,0 +1,365 @@ +import os + +import pytest + +from salt.pillar.git_pillar import GLOBAL_ONLY, PER_REMOTE_ONLY, PER_REMOTE_OVERRIDES +from salt.utils.gitfs import GitPillar, GitPython, Pygit2 +from salt.utils.immutabletypes import ImmutableDict, ImmutableList + +pytestmark = [ + pytest.mark.slow_test, +] + + +try: + import git # pylint: disable=unused-import + + HAS_GITPYTHON = True +except ImportError: + HAS_GITPYTHON = False + + +try: + import pygit2 # pylint: disable=unused-import + + HAS_PYGIT2 = True +except ImportError: + HAS_PYGIT2 = False + + +skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") +skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") + + +@pytest.fixture +def pillar_opts(salt_factories, tmp_path): + config_defaults = {"cachedir": str(tmp_path)} + factory = salt_factories.salt_master_daemon( + "pillar-functional-master", defaults=config_defaults + ) + config_defaults = dict(factory.config) + for key, item in config_defaults.items(): + if isinstance(item, ImmutableDict): + config_defaults[key] = dict(item) + elif isinstance(item, ImmutableList): + config_defaults[key] = list(item) + return config_defaults + + +@pytest.fixture +def gitpython_pillar_opts(pillar_opts): + pillar_opts["verified_git_pillar_provider"] = "gitpython" + return pillar_opts + + +@pytest.fixture +def pygit2_pillar_opts(pillar_opts): + pillar_opts["verified_git_pillar_provider"] = "pygit2" + return pillar_opts + + +def _get_pillar(opts, *remotes): + return GitPillar( + opts, + remotes, + per_remote_overrides=PER_REMOTE_OVERRIDES, + per_remote_only=PER_REMOTE_ONLY, + global_only=GLOBAL_ONLY, + ) + + +@skipif_no_gitpython +def test_gitpython_pillar_provider(gitpython_pillar_opts): + p = _get_pillar( + gitpython_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + assert len(p.remotes) == 1 + assert p.provider == "gitpython" + assert isinstance(p.remotes[0], GitPython) + + +@skipif_no_pygit2 +def test_pygit2_pillar_provider(pygit2_pillar_opts): + p = _get_pillar( + pygit2_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + assert len(p.remotes) == 1 + assert p.provider == "pygit2" + assert isinstance(p.remotes[0], Pygit2) + + +def _test_env(opts): + p = _get_pillar( + opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + assert len(p.remotes) == 1 + p.checkout() + repo = p.remotes[0] + # test that two different pillarenvs can exist at the same time + files = set(os.listdir(repo.get_cachedir())) + for f in (".gitignore", "README.md", "file.sls", "top.sls"): + assert f in files + opts["pillarenv"] = "main" + p2 = _get_pillar( + opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + assert len(p.remotes) == 1 + p2.checkout() + repo2 = p2.remotes[0] + files = set(os.listdir(repo2.get_cachedir())) + for f in (".gitignore", "README.md"): + assert f in files + for f in ("file.sls", "top.sls", "back.sls", "rooms.sls"): + assert f not in files + assert repo.get_cachedir() != repo2.get_cachedir() + files = set(os.listdir(repo.get_cachedir())) + for f in (".gitignore", "README.md", "file.sls", "top.sls"): + assert f in files + + # double check cache paths + assert ( + repo.get_cache_hash() == repo2.get_cache_hash() + ) # __env__ repos share same hash + assert repo.get_cache_basename() != repo2.get_cache_basename() + assert repo.get_linkdir() != repo2.get_linkdir() + assert repo.get_salt_working_dir() != repo2.get_salt_working_dir() + assert repo.get_cache_basename() == "master" + assert repo2.get_cache_basename() == "main" + + assert repo.get_cache_basename() in repo.get_cachedir() + assert ( + os.path.join(repo.get_cache_basehash(), repo.get_cache_basename()) + == repo.get_cache_full_basename() + ) + assert repo.get_linkdir() not in repo.get_cachedir() + assert repo.get_salt_working_dir() not in repo.get_cachedir() + + +@skipif_no_gitpython +def test_gitpython_env(gitpython_pillar_opts): + _test_env(gitpython_pillar_opts) + + +@skipif_no_pygit2 +def test_pygit2_env(pygit2_pillar_opts): + _test_env(pygit2_pillar_opts) + + +def _test_checkout_fetch_on_fail(opts): + p = _get_pillar(opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git") + p.checkout(fetch_on_fail=False) # TODO write me + + +@skipif_no_gitpython +def test_gitpython_checkout_fetch_on_fail(gitpython_pillar_opts): + _test_checkout_fetch_on_fail(gitpython_pillar_opts) + + +@skipif_no_pygit2 +def test_pygit2_checkout_fetch_on_fail(pygit2_pillar_opts): + _test_checkout_fetch_on_fail(pygit2_pillar_opts) + + +def _test_multiple_repos(opts): + p = _get_pillar( + opts, + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", + "main https://github.com/saltstack/salt-test-pillar-gitfs.git", + "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + ) + p.checkout() + assert len(p.remotes) == 5 + # make sure all repos dont share cache and working dir + assert len({r.get_cachedir() for r in p.remotes}) == 5 + assert len({r.get_salt_working_dir() for r in p.remotes}) == 5 + + p2 = _get_pillar( + opts, + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", + "main https://github.com/saltstack/salt-test-pillar-gitfs.git", + "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + ) + p2.checkout() + assert len(p2.remotes) == 5 + # make sure that repos are given same cache dir + for repo, repo2 in zip(p.remotes, p2.remotes): + assert repo.get_cachedir() == repo2.get_cachedir() + assert repo.get_salt_working_dir() == repo2.get_salt_working_dir() + opts["pillarenv"] = "main" + p3 = _get_pillar( + opts, + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", + "main https://github.com/saltstack/salt-test-pillar-gitfs.git", + "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + ) + p3.checkout() + # check that __env__ has different cache with different pillarenv + assert p.remotes[0].get_cachedir() != p3.remotes[0].get_cachedir() + assert p.remotes[1].get_cachedir() == p3.remotes[1].get_cachedir() + assert p.remotes[2].get_cachedir() == p3.remotes[2].get_cachedir() + assert p.remotes[3].get_cachedir() != p3.remotes[3].get_cachedir() + assert p.remotes[4].get_cachedir() == p3.remotes[4].get_cachedir() + + # check that other branch data is in cache + files = set(os.listdir(p.remotes[4].get_cachedir())) + for f in (".gitignore", "README.md", "file.sls", "top.sls", "other_env.sls"): + assert f in files + + +@skipif_no_gitpython +def test_gitpython_multiple_repos(gitpython_pillar_opts): + _test_multiple_repos(gitpython_pillar_opts) + + +@skipif_no_pygit2 +def test_pygit2_multiple_repos(pygit2_pillar_opts): + _test_multiple_repos(pygit2_pillar_opts) + + +def _test_fetch_request(opts): + p = _get_pillar( + opts, + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", + "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + ) + frequest = os.path.join(p.remotes[0].get_salt_working_dir(), "fetch_request") + frequest_other = os.path.join(p.remotes[1].get_salt_working_dir(), "fetch_request") + opts["pillarenv"] = "main" + p2 = _get_pillar( + opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + frequest2 = os.path.join(p2.remotes[0].get_salt_working_dir(), "fetch_request") + assert frequest != frequest2 + assert os.path.isfile(frequest) is False + assert os.path.isfile(frequest2) is False + assert os.path.isfile(frequest_other) is False + p.fetch_remotes() + assert os.path.isfile(frequest) is False + # fetch request was placed + assert os.path.isfile(frequest2) is True + p2.checkout() + # fetch request was found + assert os.path.isfile(frequest2) is False + p2.fetch_remotes() + assert os.path.isfile(frequest) is True + assert os.path.isfile(frequest2) is False + assert os.path.isfile(frequest_other) is False + for _ in range(3): + p2.fetch_remotes() + assert os.path.isfile(frequest) is True + assert os.path.isfile(frequest2) is False + assert os.path.isfile(frequest_other) is False + # fetch request should still be processed even on fetch_on_fail=False + p.checkout(fetch_on_fail=False) + assert os.path.isfile(frequest) is False + assert os.path.isfile(frequest2) is False + assert os.path.isfile(frequest_other) is False + + +@skipif_no_gitpython +def test_gitpython_fetch_request(gitpython_pillar_opts): + _test_fetch_request(gitpython_pillar_opts) + + +@skipif_no_pygit2 +def test_pygit2_fetch_request(pygit2_pillar_opts): + _test_fetch_request(pygit2_pillar_opts) + + +def _test_clear_old_remotes(opts): + p = _get_pillar( + opts, + "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", + "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", + ) + repo = p.remotes[0] + repo2 = p.remotes[1] + opts["pillarenv"] = "main" + p2 = _get_pillar( + opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + repo3 = p2.remotes[0] + assert os.path.isdir(repo.get_cachedir()) is True + assert os.path.isdir(repo2.get_cachedir()) is True + assert os.path.isdir(repo3.get_cachedir()) is True + p.clear_old_remotes() + assert os.path.isdir(repo.get_cachedir()) is True + assert os.path.isdir(repo2.get_cachedir()) is True + assert os.path.isdir(repo3.get_cachedir()) is True + p2.clear_old_remotes() + assert os.path.isdir(repo.get_cachedir()) is True + assert os.path.isdir(repo2.get_cachedir()) is False + assert os.path.isdir(repo3.get_cachedir()) is True + + +@skipif_no_gitpython +def test_gitpython_clear_old_remotes(gitpython_pillar_opts): + _test_clear_old_remotes(gitpython_pillar_opts) + + +@skipif_no_pygit2 +def test_pygit2_clear_old_remotes(pygit2_pillar_opts): + _test_clear_old_remotes(pygit2_pillar_opts) + + +def _test_remote_map(opts): + p = _get_pillar( + opts, + "https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + p.fetch_remotes() + assert len(p.remotes) == 1 + assert os.path.isfile( + os.path.join(opts["cachedir"], "git_pillar", "remote_map.txt") + ) + + +@skipif_no_gitpython +def test_gitpython_remote_map(gitpython_pillar_opts): + _test_remote_map(gitpython_pillar_opts) + + +@skipif_no_pygit2 +def test_pygit2_remote_map(pygit2_pillar_opts): + _test_remote_map(pygit2_pillar_opts) + + +def _test_lock(opts): + p = _get_pillar( + opts, + "https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + p.fetch_remotes() + assert len(p.remotes) == 1 + repo = p.remotes[0] + assert repo.get_salt_working_dir() in repo._get_lock_file() + assert repo.lock() == ( + [ + "Set update lock for git_pillar remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" + ], + [], + ) + assert os.path.isfile(repo._get_lock_file()) + assert repo.clear_lock() == ( + [ + "Removed update lock for git_pillar remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" + ], + [], + ) + assert not os.path.isfile(repo._get_lock_file()) + + +@skipif_no_gitpython +def test_gitpython_lock(gitpython_pillar_opts): + _test_lock(gitpython_pillar_opts) + + +@skipif_no_pygit2 +def test_pygit2_lock(pygit2_pillar_opts): + _test_lock(pygit2_pillar_opts) diff --git a/tests/pytests/functional/utils/test_winrepo.py b/tests/pytests/functional/utils/test_winrepo.py new file mode 100644 index 0000000000..117d995bba --- /dev/null +++ b/tests/pytests/functional/utils/test_winrepo.py @@ -0,0 +1,164 @@ +import os + +import pytest + +from salt.runners.winrepo import GLOBAL_ONLY, PER_REMOTE_ONLY, PER_REMOTE_OVERRIDES +from salt.utils.gitfs import GitPython, Pygit2, WinRepo +from salt.utils.immutabletypes import ImmutableDict, ImmutableList + +pytestmark = [ + pytest.mark.slow_test, +] + + +try: + import git # pylint: disable=unused-import + + HAS_GITPYTHON = True +except ImportError: + HAS_GITPYTHON = False + + +try: + import pygit2 # pylint: disable=unused-import + + HAS_PYGIT2 = True +except ImportError: + HAS_PYGIT2 = False + + +skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") +skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") + + +@pytest.fixture +def winrepo_opts(salt_factories, tmp_path): + config_defaults = {"cachedir": str(tmp_path)} + factory = salt_factories.salt_master_daemon( + "winrepo-functional-master", defaults=config_defaults + ) + config_defaults = dict(factory.config) + for key, item in config_defaults.items(): + if isinstance(item, ImmutableDict): + config_defaults[key] = dict(item) + elif isinstance(item, ImmutableList): + config_defaults[key] = list(item) + return config_defaults + + +@pytest.fixture +def gitpython_winrepo_opts(winrepo_opts): + winrepo_opts["verified_winrepo_provider"] = "gitpython" + return winrepo_opts + + +@pytest.fixture +def pygit2_winrepo_opts(winrepo_opts): + winrepo_opts["verified_winrepo_provider"] = "pygit2" + return winrepo_opts + + +def _get_winrepo(opts, *remotes): + return WinRepo( + opts, + remotes, + per_remote_overrides=PER_REMOTE_OVERRIDES, + per_remote_only=PER_REMOTE_ONLY, + global_only=GLOBAL_ONLY, + ) + + +@skipif_no_gitpython +def test_gitpython_winrepo_provider(gitpython_winrepo_opts): + w = _get_winrepo( + gitpython_winrepo_opts, + "https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + assert len(w.remotes) == 1 + assert w.provider == "gitpython" + assert isinstance(w.remotes[0], GitPython) + + +@skipif_no_pygit2 +def test_pygit2_winrepo_provider(pygit2_winrepo_opts): + w = _get_winrepo( + pygit2_winrepo_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" + ) + assert len(w.remotes) == 1 + assert w.provider == "pygit2" + assert isinstance(w.remotes[0], Pygit2) + + +def _test_winrepo_simple(opts): + w = _get_winrepo(opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git") + assert len(w.remotes) == 1 + w.checkout() + repo = w.remotes[0] + files = set(os.listdir(repo.get_cachedir())) + for f in (".gitignore", "README.md", "file.sls", "top.sls"): + assert f in files + + +@skipif_no_gitpython +def test_gitpython_winrepo_simple(gitpython_winrepo_opts): + _test_winrepo_simple(gitpython_winrepo_opts) + + +@skipif_no_pygit2 +def test_pygit2_winrepo_simple(pygit2_winrepo_opts): + _test_winrepo_simple(pygit2_winrepo_opts) + + +def _test_remote_map(opts): + p = _get_winrepo( + opts, + "https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + p.fetch_remotes() + assert len(p.remotes) == 1 + assert os.path.isfile(os.path.join(opts["cachedir"], "winrepo", "remote_map.txt")) + + +@skipif_no_gitpython +def test_gitpython_remote_map(gitpython_winrepo_opts): + _test_remote_map(gitpython_winrepo_opts) + + +@skipif_no_pygit2 +def test_pygit2_remote_map(pygit2_winrepo_opts): + _test_remote_map(pygit2_winrepo_opts) + + +def _test_lock(opts): + w = _get_winrepo( + opts, + "https://github.com/saltstack/salt-test-pillar-gitfs.git", + ) + w.fetch_remotes() + assert len(w.remotes) == 1 + repo = w.remotes[0] + assert repo.get_salt_working_dir() in repo._get_lock_file() + assert repo.lock() == ( + [ + "Set update lock for winrepo remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" + ], + [], + ) + assert os.path.isfile(repo._get_lock_file()) + assert repo.clear_lock() == ( + [ + "Removed update lock for winrepo remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" + ], + [], + ) + assert not os.path.isfile(repo._get_lock_file()) + + +@skipif_no_gitpython +def test_gitpython_lock(gitpython_winrepo_opts): + _test_lock(gitpython_winrepo_opts) + + +@skipif_no_pygit2 +def test_pygit2_lock(pygit2_winrepo_opts): + _test_lock(pygit2_winrepo_opts) diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py index 4508eaee95..740743194e 100644 --- a/tests/pytests/unit/test_minion.py +++ b/tests/pytests/unit/test_minion.py @@ -21,35 +21,33 @@ from tests.support.mock import MagicMock, patch log = logging.getLogger(__name__) -def test_minion_load_grains_false(): +def test_minion_load_grains_false(minion_opts): """ Minion does not generate grains when load_grains is False """ - opts = {"random_startup_delay": 0, "grains": {"foo": "bar"}} + minion_opts["grains"] = {"foo": "bar"} with patch("salt.loader.grains") as grainsfunc: - minion = salt.minion.Minion(opts, load_grains=False) - assert minion.opts["grains"] == opts["grains"] + minion = salt.minion.Minion(minion_opts, load_grains=False) + assert minion.opts["grains"] == minion_opts["grains"] grainsfunc.assert_not_called() -def test_minion_load_grains_true(): +def test_minion_load_grains_true(minion_opts): """ Minion generates grains when load_grains is True """ - opts = {"random_startup_delay": 0, "grains": {}} with patch("salt.loader.grains") as grainsfunc: - minion = salt.minion.Minion(opts, load_grains=True) + minion = salt.minion.Minion(minion_opts, load_grains=True) assert minion.opts["grains"] != {} grainsfunc.assert_called() -def test_minion_load_grains_default(): +def test_minion_load_grains_default(minion_opts): """ Minion load_grains defaults to True """ - opts = {"random_startup_delay": 0, "grains": {}} with patch("salt.loader.grains") as grainsfunc: - minion = salt.minion.Minion(opts) + minion = salt.minion.Minion(minion_opts) assert minion.opts["grains"] != {} grainsfunc.assert_called() @@ -91,24 +89,17 @@ def test_send_req_tries(req_channel, minion_opts): assert rtn == 30 - -@patch("salt.channel.client.ReqChannel.factory") -def test_mine_send_tries(req_channel_factory): +def test_mine_send_tries(minion_opts): channel_enter = MagicMock() channel_enter.send.side_effect = lambda load, timeout, tries: tries channel = MagicMock() channel.__enter__.return_value = channel_enter - opts = { - "random_startup_delay": 0, - "grains": {}, - "return_retry_tries": 20, - "minion_sign_messages": False, - } + minion_opts["return_retry_tries"] = 20 with patch("salt.channel.client.ReqChannel.factory", return_value=channel), patch( "salt.loader.grains" ): - minion = salt.minion.Minion(opts) + minion = salt.minion.Minion(minion_opts) minion.tok = "token" data = {} diff --git a/tests/pytests/unit/utils/test_gitfs.py b/tests/pytests/unit/utils/test_gitfs.py index e9915de412..2bf627049f 100644 --- a/tests/pytests/unit/utils/test_gitfs.py +++ b/tests/pytests/unit/utils/test_gitfs.py @@ -1,5 +1,4 @@ import os -import string import time import pytest @@ -214,11 +213,11 @@ def test_checkout_pygit2(_prepare_provider): provider.init_remote() provider.fetch() provider.branch = "master" - assert provider.cachedir in provider.checkout() + assert provider.get_cachedir() in provider.checkout() provider.branch = "simple_tag" - assert provider.cachedir in provider.checkout() + assert provider.get_cachedir() in provider.checkout() provider.branch = "annotated_tag" - assert provider.cachedir in provider.checkout() + assert provider.get_cachedir() in provider.checkout() provider.branch = "does_not_exist" assert provider.checkout() is None @@ -238,18 +237,9 @@ def test_checkout_pygit2_with_home_env_unset(_prepare_provider): assert "HOME" in os.environ -def test_full_id_pygit2(_prepare_provider): - assert _prepare_provider.full_id().startswith("-") - assert _prepare_provider.full_id().endswith("/pygit2-repo---gitfs-master--") - - @pytest.mark.skipif(not HAS_PYGIT2, reason="This host lacks proper pygit2 support") @pytest.mark.skip_on_windows( reason="Skip Pygit2 on windows, due to pygit2 access error on windows" ) def test_get_cachedir_basename_pygit2(_prepare_provider): - basename = _prepare_provider.get_cachedir_basename() - assert len(basename) == 45 - assert basename[0] == "-" - # check that a valid base64 is given '/' -> '_' - assert all(c in string.ascii_letters + string.digits + "+_=" for c in basename[1:]) + assert "_" == _prepare_provider.get_cache_basename() diff --git a/tests/unit/utils/test_gitfs.py b/tests/unit/utils/test_gitfs.py index 6d8e97a239..259ea056fc 100644 --- a/tests/unit/utils/test_gitfs.py +++ b/tests/unit/utils/test_gitfs.py @@ -114,27 +114,14 @@ class TestGitBase(TestCase, AdaptedConfigurationTestCaseMixin): self.assertTrue(self.main_class.remotes[0].fetched) self.assertFalse(self.main_class.remotes[1].fetched) - def test_full_id(self): - self.assertEqual( - self.main_class.remotes[0].full_id(), "-file://repo1.git---gitfs-master--" - ) - - def test_full_id_with_name(self): - self.assertEqual( - self.main_class.remotes[1].full_id(), - "repo2-file://repo2.git---gitfs-master--", - ) - def test_get_cachedir_basename(self): self.assertEqual( - self.main_class.remotes[0].get_cachedir_basename(), - "-jXhnbGDemchtZwTwaD2s6VOaVvs98a7w+AtiYlmOVb0=", + self.main_class.remotes[0].get_cache_basename(), + "_", ) - - def test_get_cachedir_base_with_name(self): self.assertEqual( - self.main_class.remotes[1].get_cachedir_basename(), - "repo2-nuezpiDtjQRFC0ZJDByvi+F6Vb8ZhfoH41n_KFxTGsU=", + self.main_class.remotes[1].get_cache_basename(), + "_", ) def test_git_provider_mp_lock(self): -- 2.42.0
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
