From a97c376349b68c3077e391528dd24a4bc1e745a0 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:59:14 -0400 Subject: [PATCH 1/8] Add a way to mutate JSON within manifests --- tools/build_project.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/build_project.py b/tools/build_project.py index ec8c865f..7da54683 100755 --- a/tools/build_project.py +++ b/tools/build_project.py @@ -470,6 +470,20 @@ def main(): LOGGER.error("Defines were unused, aborting: %s", unused_defines) sys.exit(1) + # JSON config + for json_config in manifest.get("json_config", []): + json_path = args.build_dir / json_config["file"] + + subprocess.run( + [ + "jq", + json_config["jq"], + json_path, + json_path, + ], + check=True, + ) + # Fix Gecko SDK bugs sl_rail_util_pti_config_h = args.build_dir / "config/sl_rail_util_pti_config.h" From ce99bc987e957370f57a4bd47b0587c3d60b572e Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:16:04 -0400 Subject: [PATCH 2/8] Apply patch from @tl-sl --- tools/build_project.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tools/build_project.py b/tools/build_project.py index 7da54683..29efad21 100755 --- a/tools/build_project.py +++ b/tools/build_project.py @@ -370,6 +370,23 @@ def main(): with (args.build_dir / "gbl_metadata.yaml").open("w") as f: yaml.dump(manifest["gbl"], f) + # JSON config + for json_config in manifest.get("json_config", []): + json_path = build_template_path / json_config["file"] + + result = subprocess.run( + [ + "jq", + json_config["jq"], + json_path, + ], + capture_output=True, + check=True, + ) + + with open(json_path, "wb") as f: + f.write(result.stdout) + # Next, generate a chip-specific project from the modified base project print(f"Generating project for {manifest['device']}") @@ -470,20 +487,6 @@ def main(): LOGGER.error("Defines were unused, aborting: %s", unused_defines) sys.exit(1) - # JSON config - for json_config in manifest.get("json_config", []): - json_path = args.build_dir / json_config["file"] - - subprocess.run( - [ - "jq", - json_config["jq"], - json_path, - json_path, - ], - check=True, - ) - # Fix Gecko SDK bugs sl_rail_util_pti_config_h = args.build_dir / "config/sl_rail_util_pti_config.h" From a56214986f85ae12e08899b4bb63fb025fc4361f Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:24:35 -0400 Subject: [PATCH 3/8] Use the `jq` Python package --- Dockerfile | 7 +++++-- README.md | 2 +- tools/build_project.py | 15 +++------------ 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index c8cedaa4..11c94cd2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM debian:bookworm ARG DEBIAN_FRONTEND=noninteractive +ENV PIP_ROOT_USER_ACTION=ignore + RUN \ apt-get update \ && apt-get install -y --no-install-recommends \ @@ -16,9 +18,10 @@ RUN \ default-jre-headless \ patch \ python3 \ - python3-ruamel.yaml \ + python3-pip \ unzip \ - xz-utils + xz-utils \ + && pip install jq ruamel.yaml # Install Simplicity Commander (unfortunately no stable URL available, this # is known to be working with Commander_linux_x86_64_1v15p0b1306.tar.bz). diff --git a/README.md b/README.md index e3aaf22a..fb96fd9a 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ tool will automatically determine which SDK and toolchain to use. > automatically found so these flags can be omitted. ```bash -pip install ruamel.yaml # Only dependency +pip install jq ruamel.yaml python tools/build_project.py \ # The following SDK and toolchain flags can be omitted on macOS diff --git a/tools/build_project.py b/tools/build_project.py index 29efad21..5e355d74 100755 --- a/tools/build_project.py +++ b/tools/build_project.py @@ -19,6 +19,7 @@ import multiprocessing from datetime import datetime, timezone +import jq from ruamel.yaml import YAML @@ -374,18 +375,8 @@ def main(): for json_config in manifest.get("json_config", []): json_path = build_template_path / json_config["file"] - result = subprocess.run( - [ - "jq", - json_config["jq"], - json_path, - ], - capture_output=True, - check=True, - ) - - with open(json_path, "wb") as f: - f.write(result.stdout) + result = jq.compile(json_config["jq"]).input_text(json_path.read_text()).first() + json_path.write_text(json.dumps(result, indent=2)) # Next, generate a chip-specific project from the modified base project print(f"Generating project for {manifest['device']}") From 710c94564b0d09ae0243a8940071531d5b2b23fa Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:34:04 -0400 Subject: [PATCH 4/8] Use a virtual environment... --- Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 11c94cd2..bbc453f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,6 @@ FROM debian:bookworm ARG DEBIAN_FRONTEND=noninteractive -ENV PIP_ROOT_USER_ACTION=ignore - RUN \ apt-get update \ && apt-get install -y --no-install-recommends \ @@ -19,9 +17,15 @@ RUN \ patch \ python3 \ python3-pip \ + python3-virtualenv \ unzip \ - xz-utils \ - && pip install jq ruamel.yaml + xz-utils + +RUN \ + virtualenv /opt/venv \ + /opt/venv/pip install jq ruamel.yaml + +ENV PATH="/opt/venv/bin:$PATH" # Install Simplicity Commander (unfortunately no stable URL available, this # is known to be working with Commander_linux_x86_64_1v15p0b1306.tar.bz). From 2cacb7798f78d914d3c7ae4d7ea1792e583f4cda Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:37:42 -0400 Subject: [PATCH 5/8] Include mistakenly omitted `&&` in Docker command --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bbc453f2..55a00b6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN \ RUN \ virtualenv /opt/venv \ - /opt/venv/pip install jq ruamel.yaml + && /opt/venv/pip install jq ruamel.yaml ENV PATH="/opt/venv/bin:$PATH" From 3db28b8fef1e1347a96817f33564a4f50f3c9dce Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:44:56 -0400 Subject: [PATCH 6/8] Actually test using `docker build .` --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 55a00b6d..16883225 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN \ RUN \ virtualenv /opt/venv \ - && /opt/venv/pip install jq ruamel.yaml + && /opt/venv/bin/pip install jq ruamel.yaml ENV PATH="/opt/venv/bin:$PATH" From 6e5be68c54537d6714f80fb41d1c0eb5d43efd6b Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:00:28 -0400 Subject: [PATCH 7/8] Use the absolute path to the venv Python instead of modifying `PATH` `slc-cli` does not play well in a venv, it seems --- .github/workflows/build.yaml | 2 +- Dockerfile | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index edfcfd74..8cbb06a4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -159,7 +159,7 @@ jobs: done # Build it - python3 tools/build_project.py \ + /opt/venv/bin/python3 tools/build_project.py \ $sdk_args \ $toolchain_args \ --manifest "${{ matrix.manifest }}" \ diff --git a/Dockerfile b/Dockerfile index 16883225..6ceaa561 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,8 +25,6 @@ RUN \ virtualenv /opt/venv \ && /opt/venv/bin/pip install jq ruamel.yaml -ENV PATH="/opt/venv/bin:$PATH" - # Install Simplicity Commander (unfortunately no stable URL available, this # is known to be working with Commander_linux_x86_64_1v15p0b1306.tar.bz). RUN \ From 343d7d0ecc64699883fbe490ca8448ce06c7a985 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:26:43 -0400 Subject: [PATCH 8/8] Allow `template:` in JSON config --- tools/build_project.py | 81 +++++++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/tools/build_project.py b/tools/build_project.py index 5e355d74..73bbcc24 100755 --- a/tools/build_project.py +++ b/tools/build_project.py @@ -23,9 +23,27 @@ from ruamel.yaml import YAML -SLC = ["slc", "--daemon", "--daemon-timeout", "1"] - LOGGER = logging.getLogger(__name__) +SLC = ["slc", "--daemon", "--daemon-timeout", "1"] +DEFAULT_JSON_CONFIG = [ + # Fix a few paths by default + { + "file": "config/zcl/zcl_config.zap", + "jq": '(.package[] | select(.type == "zcl-properties")).path = $zcl_zap', + "args": { + "zcl_zap": "template:{sdk}/app/zcl/zcl-zap.json", + }, + "skip_if_missing": True, + }, + { + "file": "config/zcl/zcl_config.zap", + "jq": '(.package[] | select(.type == "gen-templates-json")).path = $gen_templates', + "args": { + "gen_templates": "template:{sdk}/protocol/zigbee/app/framework/gen-template/gen-templates.json", + }, + "skip_if_missing": True, + }, +] yaml = YAML(typ="safe") @@ -39,6 +57,14 @@ def evaulate_f_string(f_string: str, variables: dict[str, typing.Any]) -> str: return eval("f" + repr(f_string), variables) +def expand_template(value: typing.Any, env: dict[str, typing.Any]) -> typing.Any: + """Expand a template string.""" + if isinstance(value, str) and value.startswith("template:"): + return evaulate_f_string(value.replace("template:", "", 1), env) + else: + return value + + def ensure_folder(path: str | pathlib.Path) -> pathlib.Path: """Ensure that the path exists and is a folder.""" path = pathlib.Path(path) @@ -275,10 +301,10 @@ def main(): # Ensure we can load the correct SDK and toolchain sdks = load_sdks(args.sdks) - sdk, sdk_version = next( + sdk, sdk_name_version = next( (path, version) for path, version in sdks.items() if version == manifest["sdk"] ) - sdk_name = sdk_version.split(":", 1)[0] + sdk_name, _, sdk_version = sdk_name_version.partition(":") toolchains = load_toolchains(args.toolchains) toolchain = next( @@ -371,11 +397,33 @@ def main(): with (args.build_dir / "gbl_metadata.yaml").open("w") as f: yaml.dump(manifest["gbl"], f) - # JSON config - for json_config in manifest.get("json_config", []): + # Template variables + value_template_env = { + "git_repo_hash": get_git_commit_id(repo=pathlib.Path(__file__).parent.parent), + "manifest_name": args.manifest.stem, + "now": datetime.now(timezone.utc), + "sdk": sdk, + "sdk_version": sdk_version, + } + + for json_config in DEFAULT_JSON_CONFIG + manifest.get("json_config", []): json_path = build_template_path / json_config["file"] - result = jq.compile(json_config["jq"]).input_text(json_path.read_text()).first() + if json_config.get("skip_if_missing", False) and not json_path.exists(): + continue + + jq_args = { + k: expand_template(v, value_template_env) + for k, v in json_config.get("args", {}).items() + } + + LOGGER.debug("Substituting JQ args for %s: %s", json_path, jq_args) + + result = ( + jq.compile(json_config["jq"], args=jq_args) + .input_text(json_path.read_text()) + .first() + ) json_path.write_text(json.dumps(result, indent=2)) # Next, generate a chip-specific project from the modified base project @@ -407,13 +455,6 @@ def main(): LOGGER.error("Referenced extension not present in SDK: %s", expected_dir) sys.exit(1) - # Template variables for C defines - value_template_env = { - "git_repo_hash": get_git_commit_id(repo=pathlib.Path(__file__).parent.parent), - "manifest_name": args.manifest.stem, - "now": datetime.now(timezone.utc), - } - # Actually search for C defines within config unused_defines = set(manifest.get("c_defines", {}).keys()) @@ -424,7 +465,7 @@ def main(): new_config_h_lines = [] for index, line in enumerate(config_h_lines): - for define, value_template in manifest.get("c_defines", {}).items(): + for define, value in manifest.get("c_defines", {}).items(): if f"#define {define} " not in line: continue @@ -449,15 +490,7 @@ def main(): assert re.match(r'#warning ".*? not configured"', prev_line) new_config_h_lines.pop(index - 1) - value_template = str(value_template) - - if value_template.startswith("template:"): - value = value_template.replace("template:", "", 1).format( - **value_template_env - ) - else: - value = value_template - + value = expand_template(value, value_template_env) new_config_h_lines.append(f"#define {define}{alignment}{value}") written_config[define] = value