Merge pull request madduck#71 from salt-formulas/andrewp-yaml-git

usable yaml_git and mixed storage types

README-extensions.rst

@@ -577,3 +577,117 @@ This adds the subfolder to the node name and the structure above can then be use
       ...
 
 If the subfolder path starts with the underscore character ``_``, then the subfolder path is NOT added to the node name.
+
+
+Git storage type
+----------------
+
+Reclass node and class yaml files can be read from a remote git repository with the yaml_git storage type. Use nodes_uri and
+classes_uri to define the git repos to use for nodes and classes. These can be the same repo.
+
+For salt masters using ssh connections the private and public keys must be readable by the salt daemon, which requires the
+private key NOT be password protected. For stand alone reclass using ssh connections if the privkey and pubkey options
+are not defined then any in memory key (from ssh-add) will be used.
+
+Salt master reclass config example:
+
+.. code-block:: yaml
+
+  storage_type:yaml_git
+  nodes_uri:
+    # branch to use
+    branch: master
+
+    # cache directory (default: ~/.reclass/git/cache)
+    cache_dir: /var/cache/reclass/git
+
+    # lock directory (default: ~/.reclass/git/lock)
+    lock_dir: /var/cache/reclass/lock
+
+    # private key for ssh connections (no default, but will used keys stored
+    # by ssh-add in memory if privkey and pubkey are not set)
+    privkey: /root/salt_rsa
+    # public key for ssh connections
+    pubkey: /root/salt_rsa.pub
+
+    repo: git+ssh://[email protected]:salt/nodes.git
+
+  classes_uri:
+    # branch to use or __env__ to use the branch matching the node
+    # environment name
+    branch: __env__
+
+    # cache directory (default: ~/.reclass/git/cache)
+    cache_dir: /var/cache/reclass/git
+
+    # lock directory (default: ~/.reclass/git/lock)
+    lock_dir: /var/cache/reclass/lock
+
+    # private key for ssh connections (no default, but will used keys stored
+    # by ssh-add in memory if privkey and pubkey are not set)
+    privkey: /root/salt_rsa
+    # public key for ssh connections
+    pubkey: /root/salt_rsa.pub
+
+    # branch/env overrides for specific branches
+    env_overrides:
+    # prod env uses master branch
+    - prod:
+        branch: master
+    # use master branch for nodes with no environment defined
+    - none:
+        branch: master
+
+    repo: git+ssh://[email protected]:salt/site.git
+
+    # root directory of the class hierarcy in git repo
+    # defaults to root directory of git repo if not given
+    root: classes
+
+
+Mixed storage type
+------------------
+
+Use a mixture of storage types.
+
+Salt master reclass config example, which by default uses yaml_git storage but overrides the location for
+classes for the pre-prod environment to use a directory on the local disc:
+
+.. code-block:: yaml
+
+  storage_type: mixed
+  nodes_uri:
+    # storage type to use
+    storage_type: yaml_git
+
+    # yaml_git storage options
+    branch: master
+    cache_dir: /var/cache/reclass/git
+    lock_dir: /var/cache/reclass/lock
+    privkey: /root/salt_rsa
+    pubkey: /root/salt_rsa.pub
+    repo: git+ssh://[email protected]:salt/nodes.git
+
+  classes_uri:
+    # storage type to use
+    storage_type: yaml_git
+
+    # yaml_git storage options
+    branch: __env__
+    cache_dir: /var/cache/reclass/git
+    lock_dir: /var/cache/reclass/lock
+    privkey: /root/salt_rsa
+    pubkey: /root/salt_rsa.pub
+    repo: git+ssh://[email protected]:salt/site.git
+    root: classes
+
+    env_overrides:
+    - prod:
+        branch: master
+    - none:
+        branch: master
+    - pre-prod:
+        # override storage type for this environment
+        storage_type: yaml_fs
+        # options for yaml_fs storage type
+        uri: /srv/salt/env/pre-prod/classes

reclass/core.py

@@ -189,7 +189,7 @@ def _get_inventory(self, all_envs, environment, queries):
                             node.interpolate_single_export(q)
                         except InterpolationError as e:
                             e.nodename = nodename
-                            raise InvQueryError(q.contents(), e, context=p, uri=q.uri)
+                            raise InvQueryError(q.contents, e, context=p, uri=q.uri)
                 inventory[nodename] = node.exports.as_dict()
         return inventory
 

reclass/storage/yaml_git/__init__.py

@@ -9,8 +9,11 @@
 
 import collections
 import distutils.version
+import errno
+import fcntl
 import fnmatch
 import os
+import time
 
 # Squelch warning on centos7 due to upgrading cffi
 # see https://github.com/saltstack/salt/pull/39871
@@ -50,6 +53,7 @@ def __init__(self, dictionary):
         self.branch = None
         self.root = None
         self.cache_dir = None
+        self.lock_dir = None
         self.pubkey = None
         self.privkey = None
         self.password = None
@@ -59,6 +63,7 @@ def update(self, dictionary):
         if 'repo' in dictionary: self.repo = dictionary['repo']
         if 'branch' in dictionary: self.branch = dictionary['branch']
         if 'cache_dir' in dictionary: self.cache_dir = dictionary['cache_dir']
+        if 'lock_dir' in dictionary: self.lock_dir = dictionary['lock_dir']
         if 'pubkey' in dictionary: self.pubkey = dictionary['pubkey']
         if 'privkey' in dictionary: self.privkey = dictionary['privkey']
         if 'password' in dictionary: self.password = dictionary['password']
@@ -72,8 +77,31 @@ def __repr__(self):
         return '<{0}: {1} {2} {3}>'.format(self.__class__.__name__, self.repo, self.branch, self.root)
 
 
-class GitRepo(object):
+class LockFile():
+    def __init__(self, file):
+        self._file = file
+
+    def __enter__(self):
+        self._fd = open(self._file, 'w+')
+        start = time.time()
+        while True:
+            if (time.time() - start) > 120:
+                raise IOError('Timeout waiting to lock file: {0}'.format(self._file))
+            try:
+                fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except IOError as e:
+                # raise on unrelated IOErrors
+                if e.errno != errno.EAGAIN:
+                    raise
+                else:
+                    time.sleep(0.1)
+
+    def __exit__(self, type, value, traceback):
+        self._fd.close()
 
+
+class GitRepo(object):
     def __init__(self, uri, node_name_mangler, class_name_mangler):
         if pygit2 is None:
             raise errors.MissingModuleError('pygit2')
@@ -85,11 +113,18 @@ def __init__(self, uri, node_name_mangler, class_name_mangler):
             self.cache_dir = '{0}/{1}/{2}'.format(os.path.expanduser("~"), '.reclass/cache/git', self.name)
         else:
             self.cache_dir = '{0}/{1}'.format(uri.cache_dir, self.name)
-
+        if uri.lock_dir is None:
+            self.lock_file = '{0}/{1}/{2}'.format(os.path.expanduser("~"), '.reclass/cache/lock', self.name)
+        else:
+            self.lock_file = '{0}/{1}'.format(uri.lock_dir, self.name)
+        lock_dir = os.path.dirname(self.lock_file)
+        if not os.path.exists(lock_dir):
+            os.makedirs(lock_dir)
         self._node_name_mangler = node_name_mangler
         self._class_name_mangler = class_name_mangler
-        self._init_repo(uri)
-        self._fetch()
+        with LockFile(self.lock_file):
+            self._init_repo(uri)
+            self._fetch()
         self.branches = self.repo.listall_branches()
         self.files = self.files_in_repo()
 
@@ -99,10 +134,7 @@ def _init_repo(self, uri):
         else:
             os.makedirs(self.cache_dir)
             self.repo = pygit2.init_repository(self.cache_dir, bare=True)
-
-        if not self.repo.remotes:
             self.repo.create_remote('origin', self.url)
-
         if 'ssh' in self.transport:
             if '@' in self.url:
                 user, _, _ = self.url.partition('@')
@@ -130,7 +162,6 @@ def _fetch(self):
         if self.credentials is not None:
             origin.credentials = self.credentials
         fetch_results = origin.fetch(**fetch_kwargs)
-
         remote_branches = self.repo.listall_branches(pygit2.GIT_BRANCH_REMOTE)
         local_branches = self.repo.listall_branches()
         for remote_branch_name in remote_branches:
@@ -208,8 +239,8 @@ def nodes(self, branch, subdir):
                     ret[node_name] = file
         return ret
 
-class ExternalNodeStorage(ExternalNodeStorageBase):
 
+class ExternalNodeStorage(ExternalNodeStorageBase):
     def __init__(self, nodes_uri, classes_uri, compose_node_name):
         super(ExternalNodeStorage, self).__init__(STORAGE_NAME, compose_node_name)
         self._repos = dict()