From 7b98321df14dc6982c1b77bc931538a4d80187a1 Mon Sep 17 00:00:00 2001 From: Jason Yip Date: Sat, 11 Nov 2023 13:44:04 -0600 Subject: [PATCH] Support multiple file includes and include cycles in include section Previously, include sections had to be lists of size 1 and never properly addressed other sizes. Include cycles were possible to cause infinite loops. This commit supports including multiple compose files and can detect cyclic include behavior. This commit also parses each included file only once as an optimization. Signed-off-by: Jason Yip --- podman_compose.py | 96 ++++++++++++++----- tests/include_w_cycle_fail/README.md | 19 ++++ .../docker-compose.base-1.yaml | 9 ++ .../docker-compose.base-2.yaml | 9 ++ .../include_w_cycle_fail/docker-compose.yaml | 4 + .../docker-compose.base-1.yaml | 9 ++ .../docker-compose.base-2.yaml | 6 ++ .../docker-compose.yaml | 5 + 8 files changed, 132 insertions(+), 25 deletions(-) create mode 100644 tests/include_w_cycle_fail/README.md create mode 100644 tests/include_w_cycle_fail/docker-compose.base-1.yaml create mode 100644 tests/include_w_cycle_fail/docker-compose.base-2.yaml create mode 100644 tests/include_w_cycle_fail/docker-compose.yaml create mode 100644 tests/include_w_multi_indegree/docker-compose.base-1.yaml create mode 100644 tests/include_w_multi_indegree/docker-compose.base-2.yaml create mode 100644 tests/include_w_multi_indegree/docker-compose.yaml diff --git a/podman_compose.py b/podman_compose.py index 20011455..326840f8 100755 --- a/podman_compose.py +++ b/podman_compose.py @@ -14,6 +14,7 @@ import os import getpass import argparse +import copy import itertools import subprocess import time @@ -1590,39 +1591,84 @@ def _parse_compose_file(self): "COMPOSE_PATH_SEPARATOR": pathsep, } ) - compose = {} - # Iterate over files primitively to allow appending to files in-loop - files_iter = iter(files) - while True: - try: - filename = next(files_iter) - except StopIteration: - break + # read each compose file and their includes, but detect cycles using iterative DFS + # files_stack is a stack of 2-tuples of filename and current include chain length + files_stack = files[::-1] + include_chain = [None] + include_chain_set = set() + # Cache mapping filename to compose data to avoid multiple file reads + files_content_merged = {None: {}} + files_content = {} + + while files_stack: + filename = files_stack[-1] + if filename in files_content: + del files_stack[-1] + # if filename is not in include_chain_set, then filename has already been processed + # and confirmed to NOT have a cyclic include chain, so there is no need + # to modify its entry in files_content_merged + # if filename is in include_chain_set, then we are now confirming that it does NOT + # have cyclic include chain and must finalize its entry in files_content_merged + if filename in include_chain_set: + del include_chain[-1] + include_chain_set.remove(filename) + # all included files of current file have been merged at this point, + # so merge the main content + if filename in files_content_merged: + rec_merge_one( + files_content_merged[filename], files_content[filename] + ) + else: + files_content_merged[filename] = files_content[filename] + + # merge final content of current file into parent file + if include_chain[-1] in files_content_merged: + rec_merge_one( + files_content_merged[include_chain[-1]], + files_content_merged[filename], + ) + else: + files_content_merged[include_chain[-1]] = copy.deepcopy( + files_content_merged[filename] + ) + continue with open(filename, "r", encoding="utf-8") as f: content = yaml.safe_load(f) - # log(filename, json.dumps(content, indent = 2)) - if not isinstance(content, dict): + + # log(filename, json.dumps(content, indent = 2)) + if not is_dict(content): + sys.stderr.write( + "Compose file does not contain a top level object: %s\n" % filename + ) + sys.exit(1) + content = normalize(content) + # log(filename, json.dumps(content, indent = 2)) + content = rec_subs(content, self.environ) + + # must add filename to file_visited before iterating includes as + # the inclusion of itself should be detected + include_chain.append(filename) + include_chain_set.add(filename) + + # include keyword shouldn't be necessary in the content data + # include list should be reversed as the stack is LIFO and the original + # include order should be respected + for include in reversed(content.pop("include", [])): + if include in include_chain_set: sys.stderr.write( - "Compose file does not contain a top level object: %s\n" + "Compose file contains a cyclic chain of file includes: %s\n" % filename ) sys.exit(1) - content = normalize(content) - # log(filename, json.dumps(content, indent = 2)) - content = rec_subs(content, self.environ) - rec_merge(compose, content) - # If `include` is used, append included files to files - include = compose.get("include", None) - if include: - files.append(*include) - # As compose obj is updated and tested with every loop, not deleting `include` - # from it, results in it being tested again and again, original values for - # `include` be appended to `files`, and, included files be processed for ever. - # Solution is to remove 'include' key from compose obj. This doesn't break - # having `include` present and correctly processed in included files - del compose["include"] + + files_stack.append(include) + + files_content[filename] = content + + compose = files_content_merged[None] + resolved_services = self._resolve_profiles( compose.get("services", {}), set(args.profile) ) diff --git a/tests/include_w_cycle_fail/README.md b/tests/include_w_cycle_fail/README.md new file mode 100644 index 00000000..9a4752c9 --- /dev/null +++ b/tests/include_w_cycle_fail/README.md @@ -0,0 +1,19 @@ +# Test podman-compose with include cycle (fail scenario) + +```shell +podman-compose up || echo $? +``` + +expected output would be something like + +``` +Compose file contains a cyclic chain of file includes: docker-compose.base-2.yaml + +exit code: 1 +``` + +Expected `podman-compose` exit code: +```shell +echo $? +1 +``` diff --git a/tests/include_w_cycle_fail/docker-compose.base-1.yaml b/tests/include_w_cycle_fail/docker-compose.base-1.yaml new file mode 100644 index 00000000..d3347ccc --- /dev/null +++ b/tests/include_w_cycle_fail/docker-compose.base-1.yaml @@ -0,0 +1,9 @@ +version: '3.6' + +include: + - docker-compose.base-2.yaml + +services: + web: + image: busybox + diff --git a/tests/include_w_cycle_fail/docker-compose.base-2.yaml b/tests/include_w_cycle_fail/docker-compose.base-2.yaml new file mode 100644 index 00000000..9b757a01 --- /dev/null +++ b/tests/include_w_cycle_fail/docker-compose.base-2.yaml @@ -0,0 +1,9 @@ +version: '3.6' + +include: + - docker-compose.base-2.yaml + +services: + web: + command: ["/bin/busybox", "httpd", "-f", "-h", ".", "-p", "8003"] + diff --git a/tests/include_w_cycle_fail/docker-compose.yaml b/tests/include_w_cycle_fail/docker-compose.yaml new file mode 100644 index 00000000..3e28c4db --- /dev/null +++ b/tests/include_w_cycle_fail/docker-compose.yaml @@ -0,0 +1,4 @@ +version: '3.6' + +include: + - docker-compose.base-1.yaml diff --git a/tests/include_w_multi_indegree/docker-compose.base-1.yaml b/tests/include_w_multi_indegree/docker-compose.base-1.yaml new file mode 100644 index 00000000..d3347ccc --- /dev/null +++ b/tests/include_w_multi_indegree/docker-compose.base-1.yaml @@ -0,0 +1,9 @@ +version: '3.6' + +include: + - docker-compose.base-2.yaml + +services: + web: + image: busybox + diff --git a/tests/include_w_multi_indegree/docker-compose.base-2.yaml b/tests/include_w_multi_indegree/docker-compose.base-2.yaml new file mode 100644 index 00000000..b49d1d3a --- /dev/null +++ b/tests/include_w_multi_indegree/docker-compose.base-2.yaml @@ -0,0 +1,6 @@ +version: '3.6' + +services: + web: + command: ["/bin/busybox", "httpd", "-f", "-h", ".", "-p", "8003"] + diff --git a/tests/include_w_multi_indegree/docker-compose.yaml b/tests/include_w_multi_indegree/docker-compose.yaml new file mode 100644 index 00000000..4b93c6e1 --- /dev/null +++ b/tests/include_w_multi_indegree/docker-compose.yaml @@ -0,0 +1,5 @@ +version: '3.6' + +include: + - docker-compose.base-1.yaml + - docker-compose.base-2.yaml