saltstack_issue_39420_salt_3004

saltstack_issue_39420_salt_3004.patch

diff --git a/salt/config/__init__.py b/salt/config/__init__.py
index 97a7fce2f0..1c1b82471e 100644
--- a/salt/config/__init__.py
+++ b/salt/config/__init__.py
@@ -532,6 +532,8 @@ VALID_OPTS = immutabletypes.freeze(
         # could conceivably be loaded as non-string types, which is OK because
         # git_pillar will normalize them to strings. But rather than include all the
         # possible types they could be, we'll just skip type-checking.
+        "git_pillar_saltenv_whitelist": list,
+        "git_pillar_saltenv_blacklist": list,
         "git_pillar_ssl_verify": bool,
         "git_pillar_global_lock": bool,
         "git_pillar_user": str,
@@ -540,7 +542,9 @@ VALID_OPTS = immutabletypes.freeze(
         "git_pillar_privkey": str,
         "git_pillar_pubkey": str,
         "git_pillar_passphrase": str,
+        "git_pillar_ref_types": list,
         "git_pillar_refspecs": list,
+        "git_pillar_disable_saltenv_mapping": bool,
         "git_pillar_includes": bool,
         "git_pillar_verify_config": bool,
         # NOTE: gitfs_base, gitfs_fallback, gitfs_mountpoint, and gitfs_root omitted
@@ -1071,6 +1075,8 @@ DEFAULT_MINION_OPTS = immutabletypes.freeze(
         "git_pillar_env": "",
         "git_pillar_fallback": "",
         "git_pillar_root": "",
+        "git_pillar_saltenv_whitelist": [],
+        "git_pillar_saltenv_blacklist": [],
         "git_pillar_ssl_verify": True,
         "git_pillar_global_lock": True,
         "git_pillar_user": "",
@@ -1079,7 +1085,9 @@ DEFAULT_MINION_OPTS = immutabletypes.freeze(
         "git_pillar_privkey": "",
         "git_pillar_pubkey": "",
         "git_pillar_passphrase": "",
+        "git_pillar_ref_types": ["branch", "tag", "sha"],
         "git_pillar_refspecs": _DFLT_REFSPECS,
+        "git_pillar_disable_saltenv_mapping": False,
         "git_pillar_includes": True,
         "gitfs_remotes": [],
         "gitfs_mountpoint": "",
@@ -1320,6 +1328,8 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze(
         "git_pillar_env": "",
         "git_pillar_fallback": "",
         "git_pillar_root": "",
+        "git_pillar_saltenv_whitelist": [],
+        "git_pillar_saltenv_blacklist": [],
         "git_pillar_ssl_verify": True,
         "git_pillar_global_lock": True,
         "git_pillar_user": "",
@@ -1328,7 +1338,9 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze(
         "git_pillar_privkey": "",
         "git_pillar_pubkey": "",
         "git_pillar_passphrase": "",
+        "git_pillar_ref_types": ["branch", "tag", "sha"],
         "git_pillar_refspecs": _DFLT_REFSPECS,
+        "git_pillar_disable_saltenv_mapping": False,
         "git_pillar_includes": True,
         "git_pillar_verify_config": True,
         "gitfs_remotes": [],
diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py
index 22f5c3a0a9..6336e34b6a 100644
--- a/salt/pillar/__init__.py
+++ b/salt/pillar/__init__.py
@@ -500,6 +500,7 @@ class Pillar:
         pillar_override=None,
         pillarenv=None,
         extra_minion_data=None,
+        client=None,
     ):
         self.minion_id = minion_id
         self.ext = ext
@@ -510,6 +511,10 @@ class Pillar:
         self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv)
         self.saltenv = saltenv
         self.client = salt.fileclient.get_file_client(self.opts, True)
+        if client is None:
+            self.client = salt.fileclient.get_file_client(self.opts, True)
+        else:
+            self.client = client
         self.avail = self.__gather_avail()
 
         if opts.get("file_client", "") == "local" and not opts.get(
@@ -637,10 +642,11 @@ class Pillar:
         """
         Pull the file server environments out of the master options
         """
-        envs = ["base"]
-        if "pillar_roots" in self.opts:
-            envs.extend([x for x in list(self.opts["pillar_roots"]) if x not in envs])
-        return envs
+        envs = {"base"}
+        envs.update(self.client.envs())
+        if self.opts.get("__git_pillar", False) and self.opts["pillarenv"]:
+            envs.update([self.opts["pillarenv"]])
+        return sorted(envs)
 
     def get_tops(self):
         """
@@ -653,20 +659,21 @@ class Pillar:
         # Gather initial top files
         try:
             saltenvs = set()
+            fileserverenvs = self._get_envs()
             if self.opts["pillarenv"]:
                 # If the specified pillarenv is not present in the available
                 # pillar environments, do not cache the pillar top file.
-                if self.opts["pillarenv"] not in self.opts["pillar_roots"]:
+                if self.opts["pillarenv"] not in fileserverenvs:
                     log.debug(
                         "pillarenv '%s' not found in the configured pillar "
                         "environments (%s)",
                         self.opts["pillarenv"],
-                        ", ".join(self.opts["pillar_roots"]),
+                        ", ".join(fileserverenvs),
                     )
                 else:
                     saltenvs.add(self.opts["pillarenv"])
             else:
-                saltenvs = self._get_envs()
+                saltenvs = fileserverenvs
                 if self.opts.get("pillar_source_merging_strategy", None) == "none":
                     saltenvs &= {self.saltenv or "base"}
 
@@ -860,7 +867,7 @@ class Pillar:
                 msg = "Specified SLS '{}' in environment '{}' was not found. ".format(
                     sls, saltenv
                 )
-                if self.opts.get("__git_pillar", False) is True:
+                if self.opts.get("__git_pillar", False):
                     msg += (
                         "This is likely caused by a git_pillar top file "
                         "containing an environment other than the one for the "
@@ -875,11 +882,13 @@ class Pillar:
                         "could also be due to environment '{1}' not being "
                         "defined in 'pillar_roots'.".format(sls, saltenv)
                     )
-                log.debug(msg)
+                log.error(msg)
+                errors.append(msg)
                 # return state, mods, errors
                 return None, mods, errors
         state = None
         try:
+            defaults["file_client"] = self.client
             state = compile_template(
                 fn_,
                 self.rend,
diff --git a/salt/pillar/git_pillar.py b/salt/pillar/git_pillar.py
index e8ece28a81..fa353e10b1 100644
--- a/salt/pillar/git_pillar.py
+++ b/salt/pillar/git_pillar.py
@@ -398,9 +398,21 @@ import salt.utils.versions
 from salt.exceptions import FileserverConfigError
 from salt.pillar import Pillar
 
-PER_REMOTE_OVERRIDES = ("base", "env", "root", "ssl_verify", "refspecs", "fallback")
-PER_REMOTE_ONLY = ("name", "mountpoint", "all_saltenvs")
-GLOBAL_ONLY = ("branch",)
+PER_REMOTE_OVERRIDES = (
+    "base",
+    "fallback",
+    "root",
+    "ssl_verify",
+    "saltenv_whitelist",
+    "saltenv_blacklist",
+    "refspecs",
+    "disable_saltenv_mapping",
+    "ref_types",
+    # "update_interval",
+    "env",  # != gitfs
+)
+PER_REMOTE_ONLY = ("name", "mountpoint", "all_saltenvs")  # "saltenv"
+GLOBAL_ONLY = ("branch",)  # != gitfs
 
 # Set up logging
 log = logging.getLogger(__name__)
@@ -435,7 +447,7 @@ def ext_pillar(minion_id, pillar, *repos):  # pylint: disable=unused-argument
     opts = copy.deepcopy(__opts__)
     opts["pillar_roots"] = {}
     opts["__git_pillar"] = True
-    git_pillar = salt.utils.gitfs.GitPillar(
+    git_pillar_client = salt.utils.gitfs.GitPillarClient(
         opts,
         repos,
         per_remote_overrides=PER_REMOTE_OVERRIDES,
@@ -445,67 +457,49 @@ def ext_pillar(minion_id, pillar, *repos):  # pylint: disable=unused-argument
     if __opts__.get("__role") == "minion":
         # If masterless, fetch the remotes. We'll need to remove this once
         # we make the minion daemon able to run standalone.
-        git_pillar.fetch_remotes()
-    git_pillar.checkout()
+        git_pillar_client.fetch_remotes()
     ret = {}
     merge_strategy = __opts__.get("pillar_source_merging_strategy", "smart")
     merge_lists = __opts__.get("pillar_merge_lists", False)
-    for pillar_dir, env in git_pillar.pillar_dirs.items():
-        # Map env if env == '__env__' before checking the env value
-        if env == "__env__":
-            env = (
-                opts.get("pillarenv")
-                or opts.get("saltenv")
-                or opts.get("git_pillar_base")
-            )
-            log.debug("__env__ maps to %s", env)
-
-        # If pillarenv is set, only grab pillars with that match pillarenv
-        if opts["pillarenv"] and env != opts["pillarenv"]:
-            log.debug(
-                "env '%s' for pillar dir '%s' does not match pillarenv '%s', skipping",
-                env,
-                pillar_dir,
-                opts["pillarenv"],
-            )
-            continue
-        if pillar_dir in git_pillar.pillar_linked_dirs:
-            log.debug(
-                "git_pillar is skipping processing on %s as it is a mounted repo",
-                pillar_dir,
-            )
-            continue
-        else:
-            log.debug(
-                "git_pillar is processing pillar SLS from %s for pillar env '%s'",
-                pillar_dir,
-                env,
-            )
 
-        pillar_roots = [pillar_dir]
 
+    def remote_env(remote):
+        if remote.branch == "__env__" and hasattr(remote, "all_saltenvs"):
+            return opts.get("pillarenv") or opts.get("saltenv") or "base"
+        elif remote.env:
+            return remote.env
+        else:
+            if remote.branch == remote.base:
+                return "base"
+            else:
+                tgt = remote.get_checkout_target()
+                return "base" if tgt == remote.base else tgt
+
+    for remote_idx, remote in enumerate(git_pillar_client.remotes):
+        env = remote_env(remote)
+        remotes = [repos[remote_idx]]
         if __opts__["git_pillar_includes"]:
-            # Add the rest of the pillar_dirs in this environment to the
-            # list, excluding the current pillar_dir being processed. This
-            # is because it was already specified above as the first in the
-            # list, so that its top file is sourced from the correct
-            # location and not from another git_pillar remote.
-            pillar_roots.extend(
-                [
-                    d
-                    for (d, e) in git_pillar.pillar_dirs.items()
-                    if env == e and d != pillar_dir
-                ]
-            )
-
-        opts["pillar_roots"] = {env: pillar_roots}
-
-        local_pillar = Pillar(opts, __grains__, minion_id, env)
+            for remote_idx2, remote2 in enumerate(git_pillar_client.remotes):
+                if remote_idx2 == remote_idx:
+                    continue
+                env2 = remote_env(remote2)
+                if env2 == env:
+                    remotes.append(repos[remote_idx2])
+
+        repo_git_pillar_client = salt.utils.gitfs.GitPillarClient(
+            opts,
+            remotes,
+            per_remote_overrides=PER_REMOTE_OVERRIDES,
+            per_remote_only=PER_REMOTE_ONLY,
+            global_only=GLOBAL_ONLY,
+        )
+        local_pillar = Pillar(
+            opts, __grains__, minion_id, env, client=repo_git_pillar_client
+        )
+        compiled = local_pillar.compile_pillar(ext=False)
         ret = salt.utils.dictupdate.merge(
             ret,
-            local_pillar.compile_pillar(ext=False),
-            strategy=merge_strategy,
-            merge_lists=merge_lists,
+            compiled, strategy=merge_strategy, merge_lists=merge_lists,
         )
     return ret
 
diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py
index 7fddac7b2b..dbf4ea2b7f 100644
--- a/salt/utils/gitfs.py
+++ b/salt/utils/gitfs.py
@@ -21,6 +21,7 @@ import weakref
 from datetime import datetime
 
 import salt.ext.tornado.ioloop
+import salt.fileclient
 import salt.fileserver
 import salt.utils.configparser
 import salt.utils.data
@@ -524,8 +525,6 @@ class GitProvider:
             strip_sep = (
                 lambda x: x.rstrip(os.sep) if name in ("root", "mountpoint") else x
             )
-            if self.role != "gitfs":
-                return strip_sep(getattr(self, "_" + name))
             # Get saltenv-specific configuration
             saltenv_conf = self.saltenv.get(tgt_env, {})
             if name == "ref":
@@ -541,6 +540,43 @@ class GitProvider:
                     else:
                         return None
 
+                def _get_git_pillar_env():
+                    if self.branch == "__env__":
+                        if hasattr(self, "all_saltenvs"):
+                            pillarenv = (
+                                self.opts.get("pillarenv")
+                                or self.opts.get("saltenv")
+                                or "base"
+                            )
+                            if tgt_env == pillarenv:
+                                return self.all_saltenvs
+                            else:
+                                return None
+                        else:
+                            if tgt_env == "base":
+                                return self.base
+                            else:
+                                return tgt_env
+                    elif self.env:
+                        if tgt_env == self.env:
+                            return self.branch
+                        else:
+                            return None
+                    else:
+                        if tgt_env == "base":
+                            if self.branch == self.base:
+                                return self.branch
+                            else:
+                                return None
+                        else:
+                            if tgt_env == self.branch:
+                                return self.branch
+                            else:
+                                return None
+
+                if self.opts.get("__git_pillar", False):
+                    return _get_git_pillar_env()
+
                 # Return the all_saltenvs branch/tag if it is configured
                 per_saltenv_ref = _get_per_saltenv(tgt_env)
                 try:
@@ -1014,6 +1050,17 @@ class GitProvider:
         Check if an environment is exposed by comparing it against a whitelist
         and blacklist.
         """
+        if self.opts.get("__git_pillar", False):
+            if self.branch != "__env__":
+                if self.env:
+                    if tgt_env != self.env:
+                        return False
+                elif tgt_env == "base":
+                    if self.base != self.branch:
+                        return False
+                else:
+                    if tgt_env != self.branch:
+                        return False
         return salt.utils.stringutils.check_whitelist_blacklist(
             tgt_env,
             whitelist=self.saltenv_whitelist,
@@ -2846,6 +2893,7 @@ class GitFS(GitBase):
         remotes=None,
         per_remote_overrides=(),
         per_remote_only=PER_REMOTE_ONLY,
+        global_only=GLOBAL_ONLY,
         git_providers=None,
         cache_root=None,
         init_remotes=True,
@@ -2870,6 +2918,7 @@ class GitFS(GitBase):
                 remotes if remotes is not None else [],
                 per_remote_overrides=per_remote_overrides,
                 per_remote_only=per_remote_only,
+                global_only=global_only,
                 git_providers=git_providers
                 if git_providers is not None
                 else GIT_PROVIDERS,
@@ -2893,6 +2942,7 @@ class GitFS(GitBase):
         remotes,
         per_remote_overrides=(),
         per_remote_only=PER_REMOTE_ONLY,
+        global_only=GLOBAL_ONLY,
         git_providers=None,
         cache_root=None,
         init_remotes=True,
@@ -2932,15 +2982,29 @@ class GitFS(GitBase):
         fnd = {"path": "", "rel": ""}
         if os.path.isabs(path):
             return fnd
-
-        dest = salt.utils.path.join(self.cache_root, "refs", tgt_env, path)
-        hashes_glob = salt.utils.path.join(
-            self.hash_cachedir, tgt_env, "{}.hash.*".format(path)
-        )
-        blobshadest = salt.utils.path.join(
-            self.hash_cachedir, tgt_env, "{}.hash.blob_sha1".format(path)
-        )
-        lk_fn = salt.utils.path.join(self.hash_cachedir, tgt_env, "{}.lk".format(path))
+        if self.opts.get("__git_pillar", False):
+            name = getattr(self.remotes[0], "name", self.remotes[0].hash)
+            dest = salt.utils.path.join(self.cache_root, "refs", name, tgt_env, path)
+            hashes_glob = salt.utils.path.join(
+                self.hash_cachedir, name, tgt_env, "{}.hash.*".format(path)
+            )
+            blobshadest = salt.utils.path.join(
+                self.hash_cachedir, name, tgt_env, "{}.hash.blob_sha1".format(path)
+            )
+            lk_fn = salt.utils.path.join(
+                self.hash_cachedir, name, tgt_env, "{}.lk".format(path)
+            )
+        else:
+            dest = salt.utils.path.join(self.cache_root, "refs", tgt_env, path)
+            hashes_glob = salt.utils.path.join(
+                self.hash_cachedir, tgt_env, "{}.hash.*".format(path)
+            )
+            blobshadest = salt.utils.path.join(
+                self.hash_cachedir, tgt_env, "{}.hash.blob_sha1".format(path)
+            )
+            lk_fn = salt.utils.path.join(
+                self.hash_cachedir, tgt_env, "{}.lk".format(path)
+            )
         destdir = os.path.dirname(dest)
         hashdir = os.path.dirname(blobshadest)
         if not os.path.isdir(destdir):
@@ -3353,6 +3417,81 @@ class GitPillar(GitBase):
         return True
 
 
+class GitPillarClient(GitBase, salt.fileclient.Client):
+    """
+    Functionality specific to the git external pillar implementing the salt.fileclient.Client interface
+    """
+
+    role = "git_pillar"
+
+    def __init__(
+        self,
+        opts,
+        remotes,
+        per_remote_overrides=(),
+        per_remote_only=PER_REMOTE_ONLY,
+        global_only=GLOBAL_ONLY,
+        git_providers=None,
+        cache_root=None,
+        init_remotes=True,
+    ):
+        self._envs = None
+        GitBase.__init__(
+            self,
+            opts,
+            remotes,
+            per_remote_overrides=per_remote_overrides,
+            per_remote_only=per_remote_only,
+            global_only=global_only,
+            git_providers=git_providers,
+            cache_root=cache_root,
+            init_remotes=init_remotes,
+        )
+        salt.fileclient.Client.__init__(self, opts)
+
+    def get_file(
+        self, path, dest="", makedirs=False, saltenv="base", gzip=None, cachedir=None
+    ):
+        """
+        Get a single file from the salt-master
+        path must be a salt server location, aka, salt://path/to/file, if
+        dest is omitted, then the downloaded file will be placed in the minion
+        cache
+        """
+        path = self._check_proto(path)
+        ret = GitFS.find_file(self, path, tgt_env=saltenv)
+        return ret["path"]
+
+    def file_list(self, saltenv="base", prefix=""):
+        """
+        List the files on the master
+        """
+        load = {"saltenv": saltenv, "prefix": prefix, "cmd": "_file_list"}
+        return GitFS._file_lists(self, load, "files")
+
+    def file_list_emptydirs(self, saltenv="base", prefix=""):
+        """
+        List the empty dirs on the master
+        """
+        # Cannot have empty dirs in git
+        return []
+
+    def dir_list(self, saltenv="base", prefix=""):
+        """
+        List the dirs on the master
+        """
+        load = {"saltenv": saltenv, "prefix": prefix, "cmd": "_dir_list"}
+        return GitFS._file_lists(self, load, "dirs")
+
+    def envs(self):
+        """
+        Return a list of available environments
+        """
+        if self._envs is None:
+            self._envs = GitFS.envs(self)
+        return self._envs
+
+
 class WinRepo(GitBase):
     """
     Functionality specific to the winrepo runner
diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py
index 0cb70bf64a..07559a4058 100644
--- a/salt/utils/jinja.py
+++ b/salt/utils/jinja.py
@@ -87,7 +87,9 @@ class SaltCacheLoader(BaseLoader):
         self.cached = []
         self._file_client = _file_client
         # Instantiate the fileclient
-        self.file_client()
+        if _file_client is None:
+            log.debug("SaltCacheLoader::__init__ - file_client is None. Instantiate")
+            self.file_client()
 
     def file_client(self):
         """
@@ -112,7 +114,19 @@ class SaltCacheLoader(BaseLoader):
         Cache a file from the salt master
         """
         saltpath = salt.utils.url.create(template)
-        self.file_client().get_file(saltpath, "", True, self.saltenv)
+        ret_path = self.file_client().get_file(saltpath, "", True, self.saltenv)
+        # Only continue if the file is from salt
+        if ret_path is not False:
+            # remove 'template' string in returned full path and add cache directory into searchpath
+            m = ret_path.find(template, 0, len(ret_path))
+            if m >= 0:
+                _rpath = ret_path[:m]
+                if _rpath not in self.searchpath:
+                    self.searchpath.append(_rpath)
+                log.debug(
+                    "SaltCacheLoader::cache_file Added directory %s into searchpath",
+                    _rpath,
+                )
 
     def check_cache(self, template):
         """