Merge pull request #348 from matburt/filtered_events

Allow limiting the ansible event datastructure

Reviewed-by: https://github.com/apps/ansible-zuul

ansible_runner/__main__.py

@@ -307,6 +307,22 @@ def main(sys_args=None):
              "Ansible output (default=False)"
     )
 
+    runner_group.add_argument(
+        "--omit-event-data",
+        action="store_true",
+        help="Omits including extra event data in the callback payloads "
+             "or the Runner payload data files "
+             "(status and stdout still included)"
+    )
+
+    runner_group.add_argument(
+        "--only-failed-event-data",
+        action="store_true",
+        help="Only adds extra event data for failed tasks in the callback "
+             "payloads or the Runner payload data files "
+             "(status and stdout still included for other events)"
+    )
+
     runner_group.add_argument(
         "-q", "--quiet",
         action="store_true",
@@ -515,6 +531,8 @@ def main(sys_args=None):
                                    rotate_artifacts=args.rotate_artifacts,
                                    ignore_logging=False,
                                    json_mode=args.json,
+                                   omit_event_data=args.omit_event_data,
+                                   only_failed_event_data=args.only_failed_event_data,
                                    inventory=args.inventory,
                                    forks=args.forks,
                                    project_dir=args.project_dir,

ansible_runner/display_callback/events.py

@@ -134,40 +134,45 @@ def get(self):
         return ctx
 
     def get_begin_dict(self):
+        omit_event_data = os.getenv("RUNNER_OMIT_EVENTS", "False").lower() == "true"
+        include_only_failed_event_data = os.getenv("RUNNER_ONLY_FAILED_EVENTS", "False").lower() == "true"
         event_data = self.get()
-        # TODO: Need to rework these values
+        event = event_data.pop('event', None)
+        if not event:
+            event = 'verbose'
+            for key in ('debug', 'verbose', 'deprecated', 'warning', 'system_warning', 'error'):
+                if event_data.get(key, False):
+                    event = key
+                    break
+        event_dict = dict(event=event)
+        should_process_event_data = (include_only_failed_event_data and event in ('runner_on_failed', 'runner_on_async_failed', 'runner_on_item_failed')) \
+            or not include_only_failed_event_data
         if os.getenv('JOB_ID', ''):
-            event_data['job_id'] = int(os.getenv('JOB_ID', '0'))
+            event_dict['job_id'] = int(os.getenv('JOB_ID', '0'))
         if os.getenv('AD_HOC_COMMAND_ID', ''):
-            event_data['ad_hoc_command_id'] = int(os.getenv('AD_HOC_COMMAND_ID', '0'))
+            event_dict['ad_hoc_command_id'] = int(os.getenv('AD_HOC_COMMAND_ID', '0'))
         if os.getenv('PROJECT_UPDATE_ID', ''):
-            event_data['project_update_id'] = int(os.getenv('PROJECT_UPDATE_ID', '0'))
-        event_data.setdefault('pid', os.getpid())
-        event_data.setdefault('uuid', str(uuid.uuid4()))
-        event_data.setdefault('created', datetime.datetime.utcnow().isoformat())
+            event_dict['project_update_id'] = int(os.getenv('PROJECT_UPDATE_ID', '0'))
+        event_dict['pid'] = event_data.get('pid', os.getpid())
+        event_dict['uuid'] = event_data.get('uuid', str(uuid.uuid4()))
+        event_dict['created'] = event_data.get('created', datetime.datetime.utcnow().isoformat())
         if not event_data.get('parent_uuid', None):
             for key in ('task_uuid', 'play_uuid', 'playbook_uuid'):
                 parent_uuid = event_data.get(key, None)
                 if parent_uuid and parent_uuid != event_data.get('uuid', None):
-                    event_data['parent_uuid'] = parent_uuid
-                    break
-
-        event = event_data.pop('event', None)
-        if not event:
-            event = 'verbose'
-            for key in ('debug', 'verbose', 'deprecated', 'warning', 'system_warning', 'error'):
-                if event_data.get(key, False):
-                    event = key
+                    event_dict['parent_uuid'] = parent_uuid
                     break
-        max_res = int(os.getenv("MAX_EVENT_RES", 700000))
-        if event not in ('playbook_on_stats',) and "res" in event_data and len(str(event_data['res'])) > max_res:
-            event_data['res'] = {}
-        event_dict = dict(event=event, event_data=event_data)
-        for key in list(event_data.keys()):
-            if key in ('job_id', 'ad_hoc_command_id', 'project_update_id', 'uuid', 'parent_uuid', 'created',):
-                event_dict[key] = event_data.pop(key)
-            elif key in ('verbosity', 'pid'):
-                event_dict[key] = event_data[key]
+        else:
+            event_dict['parent_uuid'] = event_data.get('parent_uuid', None)
+        if "verbosity" in event_data.keys():
+            event_dict["verbosity"] = event_data.pop("verbosity")
+        if not omit_event_data and should_process_event_data:
+            max_res = int(os.getenv("MAX_EVENT_RES", 700000))
+            if event not in ('playbook_on_stats',) and "res" in event_data and len(str(event_data['res'])) > max_res:
+                event_data['res'] = {}
+        else:
+            event_data = dict()
+        event_dict['event_data'] = event_data
         return event_dict
 
     def get_end_dict(self):

ansible_runner/interface.py

@@ -124,6 +124,8 @@ def run(**kwargs):
     :param fact_cache: A string that will be used as the name for the subdirectory of the fact cache in artifacts directory.
                        This is only used for 'jsonfile' type fact caches.
     :param fact_cache_type: A string of the type of fact cache to use.  Defaults to 'jsonfile'.
+    :param omit_event_data: Omits extra ansible event data from event payload (stdout and event still included)
+    :param only_failed_event_data: Omits extra ansible event data unless it's a failed event (stdout and event still included)
     :type private_data_dir: str
     :type ident: str
     :type json_mode: bool
@@ -155,6 +157,8 @@ def run(**kwargs):
     :type directory_isolation_base_path: str
     :type fact_cache: str
     :type fact_cache_type: str
+    :type omit_event_data: bool
+    :type only_failed_event_data: bool
 
     :returns: A :py:class:`ansible_runner.runner.Runner` object
     '''

ansible_runner/runner_config.py

@@ -79,7 +79,8 @@ def __init__(self,
                  process_isolation=False, process_isolation_executable=None, process_isolation_path=None,
                  process_isolation_hide_paths=None, process_isolation_show_paths=None, process_isolation_ro_paths=None,
                  tags=None, skip_tags=None, fact_cache_type='jsonfile', fact_cache=None, project_dir=None,
-                 directory_isolation_base_path=None, envvars=None, forks=None, cmdline=None):
+                 directory_isolation_base_path=None, envvars=None, forks=None, cmdline=None, omit_event_data=False,
+                 only_failed_event_data=False):
         self.private_data_dir = os.path.abspath(private_data_dir)
         self.ident = str(ident)
         self.json_mode = json_mode
@@ -127,6 +128,8 @@ def __init__(self,
         self.envvars = envvars
         self.forks = forks
         self.cmdline_args = cmdline
+        self.omit_event_data = omit_event_data
+        self.only_failed_event_data = only_failed_event_data
 
     def prepare(self):
         """
@@ -200,6 +203,9 @@ def prepare(self):
             self.env['ANSIBLE_CACHE_PLUGIN'] = 'jsonfile'
             self.env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] = self.fact_cache
 
+        self.env["RUNNER_OMIT_EVENTS"] = str(self.omit_event_data)
+        self.env["RUNNER_ONLY_FAILED_EVENTS"] = str(self.only_failed_event_data)
+
     def prepare_inventory(self):
         """
         Prepares the inventory default under ``private_data_dir`` if it's not overridden by the constructor.

test/integration/test_events.py

@@ -37,7 +37,10 @@ def test_basic_events(is_run_async=False,g_facts=False):
 
     okay_event = okay_events[0]
     assert "uuid" in okay_event and len(okay_event['uuid']) == 36
+    assert "parent_uuid" in okay_event and len(okay_event['parent_uuid']) == 36
     assert "stdout" in okay_event and len(okay_event['stdout']) > 0
+    assert "start_line" in okay_event and int(okay_event['start_line']) > 0
+    assert "end_line" in okay_event and int(okay_event['end_line']) > 0
     assert "event_data" in okay_event and len(okay_event['event_data']) > 0
 
 
@@ -54,6 +57,24 @@ def test_basic_serializeable():
     json.dumps(events)
 
 
+def test_event_omission():
+    tdir = tempfile.mkdtemp()
+    r = run(private_data_dir=tdir,
+            inventory="localhost ansible_connection=local",
+            omit_event_data=True,
+            playbook=[{'hosts': 'all', 'gather_facts': False, 'tasks': [{'debug': {'msg': "test"}}]}])
+    assert not any([x['event_data'] for x in r.events])
+
+
+def test_event_omission_except_failed():
+    tdir = tempfile.mkdtemp()
+    r = run(private_data_dir=tdir,
+            inventory="localhost ansible_connection=local",
+            only_failed_event_data=True,
+            playbook=[{'hosts': 'all', 'gather_facts': False, 'tasks': [{'fail': {'msg': "test"}}]}])
+    all_event_datas = [x['event_data'] for x in r.events if x['event_data']]
+    assert len(all_event_datas) == 1
+
 @pytest.mark.skipif(LooseVersion(pkg_resources.get_distribution('ansible').version) < LooseVersion('2.8'),
                     reason="Valid only on Ansible 2.8+")
 def test_runner_on_start(rc):