Don't barf on Yaml with tags

Yaml can contain custom tags in the form of '!vault' and such. Before
this commit, this would cause Yaml to be unable to read the entire yaml
file.

This commit introduces a custom yaml loader that implements the
SafeLoader class and overrides what happens when tags are encountered.
For `vault` tags, it returns an 'ENCRYPTED CONTENTS REDACTED' string.
For all other (unknown) tags, it returns the tag value as a string.

This should also fix "Skipping encrypted file'-like warnings when
generating CMDBs.

Author: fboender

example/host_vars/eek.electricmonk.nl

@@ -4,3 +4,11 @@ fact_with_int_keys: {
     1: "hoi",
     2: 1
 }
+password: !vault |
+    $ANSIBLE_VAULT;1.1;AES256
+    35366338653734663863626465336664363333383433326233343339356231363064346366656232
+    6163653539393537653532306464313738633330363561390a363530636531346338343338383962
+    62626663376161393639393962653862363931356431376635613034616534363266633665363436
+    6364616662346362370a373337316636396438623438383965636164633138386435626333643566
+    6439
+

src/ansiblecmdb/__init__.py

@@ -1,2 +1,3 @@
+from . import ihateyaml
 from .parser import HostsParser, DynInvParser
 from .ansible import Ansible

src/ansiblecmdb/ansible.py

@@ -4,12 +4,7 @@
 import subprocess
 import codecs
 import logging
-try:
-    import yaml
-except ImportError:
-    import yaml3 as yaml
-
-
+from . import ihateyaml
 import ansiblecmdb.util as util
 import ansiblecmdb.parser as parser
 
@@ -182,18 +177,25 @@ def _parse_hostvar_file(self, hostname, path):
         try:
             self.log.debug("Reading host vars from {}".format(path))
             f = codecs.open(path, 'r', encoding='utf8')
-            invars = yaml.safe_load(f)
+            invars = ihateyaml.safe_load(f)
             f.close()
-
-            if invars is not None:
-                if hostname == "all":
-                    # Hostname 'all' is special and applies to all hosts
-                    for hostname in self.hosts_all():
-                        self.update_host(hostname, {'hostvars': invars}, overwrite=False)
-                else:
-                    self.update_host(hostname, {'hostvars': invars}, overwrite=True)
         except Exception as err:
+            # Just catch everything because yaml...
             self.log.warning("Yaml couldn't load '{0}'. Skipping. Error was: {1}".format(path, err))
+            return
+
+        if invars is None:
+            # Empty file or whatever. This is *probably* a side-effect of our
+            # own yaml.SafeLoader implementation ('ihateyaml'), because this
+            # problem didn't exist before.
+            return
+
+        if hostname == "all":
+            # Hostname 'all' is special and applies to all hosts
+            for hostname in self.hosts_all():
+                self.update_host(hostname, {'hostvars': invars}, overwrite=False)
+        else:
+            self.update_host(hostname, {'hostvars': invars}, overwrite=True)
 
     def _parse_groupvar_dir(self, inventory_path):
         """
@@ -218,19 +220,13 @@ def _parse_groupvar_dir(self, inventory_path):
                 # filename is the group name
                 groupname = strip_exts(filename, ('.yml', '.yaml', '.json'))
 
-                # Check for ansible-vault files, because they're valid yaml for
-                # some reason... (psst, the reason is that yaml sucks)
-                first_line = open(full_path, 'r').readline()
-                if first_line.startswith('$ANSIBLE_VAULT'):
-                    self.log.warning("Skipping encrypted vault file {0}".format(full_path))
-                    continue
-
                 try:
                     self.log.debug("Reading group vars from {}".format(full_path))
                     f = codecs.open(full_path, 'r', encoding='utf8')
-                    invars = yaml.safe_load(f)
+                    invars = ihateyaml.safe_load(f)
                     f.close()
                 except Exception as err:
+                    # Just catch everything because yaml...
                     self.log.warning("Yaml couldn't load '{0}' because '{1}'. Skipping".format(full_path, err))
                     continue  # Go to next file
 

src/ansiblecmdb/ihateyaml.py

@@ -0,0 +1,45 @@
+"""
+Custom Yaml library wrapper, because Yaml is a bag of shit.
+"""
+
+try:
+    import yaml
+except ImportError:
+    import yaml3 as yaml
+
+class StupidYAMLShit(yaml.SafeLoader):
+    """
+    FUCK PYYAML. This class overrides some insanely deep shit which took at
+    least two hours to get working. This class overrides SafeLoader and handles
+    tags (e.g. '!bullshit') nonsense in PyYAML, because obviously there is no
+    ignore_tags option or a simple callback that actually works. That would be
+    user-friendly, and user-friendliness is insanity, amirite?!
+
+    Also, there is no single entry point to hook into this, so we need to
+    specifically inherit from *SafeLoader* and not from *Loader*. Thanks for
+    wasting some more of my fucking time PyYAML, you turd.
+
+    On a pyyaml-rage-induced side note: How many apps are vulnerable because
+    all the PyYaml docs mention 'load' and not 'safe_load'? Was this diseased
+    pile of gunk written by a PHP programmer?!
+
+    """
+    def handle_tag(self, node_name, node):
+        # I just *know* there are gonna be problems with simply returning a
+        # Scalar, but I don't give a fuck at this point.
+        if node_name == "vault":
+            new_node = yaml.ScalarNode(node_name, 'ENCRYPTED CONTENTS REDACTED')
+        else:
+            new_node = yaml.ScalarNode(node_name, node.value)
+
+        return self.construct_scalar(new_node)
+
+
+# Fugly!
+StupidYAMLShit.add_multi_constructor(
+    u'!',
+    StupidYAMLShit.handle_tag)
+
+
+def safe_load(contents):
+    return yaml.load(contents, Loader=StupidYAMLShit)

src/ansiblecmdb/parser.py

@@ -8,17 +8,7 @@
     import ushlex as shlex
 else:
     import shlex
-try:
-    import yaml
-except ImportError as err:
-    import yaml3 as yaml
-
-
-def default_ctor(loader, tag_suffix, node):
-    return tag_suffix + ' ' + node.value
-
-
-yaml.add_multi_constructor('', default_ctor)
+from . import ihateyaml
 
 
 class HostsParser(object):
@@ -189,7 +179,7 @@ def _parse_line_vars(self, line):
         k, v = line.strip().split('=', 1)
         if v.startswith('['):
             try:
-                list_res = yaml.safe_load(v)
+                list_res = ihateyaml.safe_load(v)
                 if isinstance(list_res[0], dict):
                     key_values = list_res[0]
                     return key_values

test/test.py

@@ -3,8 +3,8 @@
 import imp
 import os
 
-sys.path.insert(0, '../lib')
-sys.path.insert(0, '../src')
+sys.path.insert(0, os.path.realpath('../lib'))
+sys.path.insert(0, os.path.realpath('../src'))
 import ansiblecmdb