diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index edfcfd74..8cbb06a4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -159,7 +159,7 @@ jobs: done # Build it - python3 tools/build_project.py \ + /opt/venv/bin/python3 tools/build_project.py \ $sdk_args \ $toolchain_args \ --manifest "${{ matrix.manifest }}" \ diff --git a/Dockerfile b/Dockerfile index c8cedaa4..6ceaa561 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,10 +16,15 @@ RUN \ default-jre-headless \ patch \ python3 \ - python3-ruamel.yaml \ + python3-pip \ + python3-virtualenv \ unzip \ xz-utils +RUN \ + virtualenv /opt/venv \ + && /opt/venv/bin/pip install jq ruamel.yaml + # Install Simplicity Commander (unfortunately no stable URL available, this # is known to be working with Commander_linux_x86_64_1v15p0b1306.tar.bz). RUN \ diff --git a/README.md b/README.md index e3aaf22a..fb96fd9a 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ tool will automatically determine which SDK and toolchain to use. > automatically found so these flags can be omitted. ```bash -pip install ruamel.yaml # Only dependency +pip install jq ruamel.yaml python tools/build_project.py \ # The following SDK and toolchain flags can be omitted on macOS diff --git a/tools/build_project.py b/tools/build_project.py index ec8c865f..73bbcc24 100755 --- a/tools/build_project.py +++ b/tools/build_project.py @@ -19,12 +19,31 @@ import multiprocessing from datetime import datetime, timezone +import jq from ruamel.yaml import YAML -SLC = ["slc", "--daemon", "--daemon-timeout", "1"] - LOGGER = logging.getLogger(__name__) +SLC = ["slc", "--daemon", "--daemon-timeout", "1"] +DEFAULT_JSON_CONFIG = [ + # Fix a few paths by default + { + "file": "config/zcl/zcl_config.zap", + "jq": '(.package[] | select(.type == "zcl-properties")).path = $zcl_zap', + "args": { + "zcl_zap": "template:{sdk}/app/zcl/zcl-zap.json", + }, + "skip_if_missing": True, + }, + { + "file": "config/zcl/zcl_config.zap", + "jq": '(.package[] | select(.type == "gen-templates-json")).path = $gen_templates', + "args": { + "gen_templates": "template:{sdk}/protocol/zigbee/app/framework/gen-template/gen-templates.json", + }, + "skip_if_missing": True, + }, +] yaml = YAML(typ="safe") @@ -38,6 +57,14 @@ def evaulate_f_string(f_string: str, variables: dict[str, typing.Any]) -> str: return eval("f" + repr(f_string), variables) +def expand_template(value: typing.Any, env: dict[str, typing.Any]) -> typing.Any: + """Expand a template string.""" + if isinstance(value, str) and value.startswith("template:"): + return evaulate_f_string(value.replace("template:", "", 1), env) + else: + return value + + def ensure_folder(path: str | pathlib.Path) -> pathlib.Path: """Ensure that the path exists and is a folder.""" path = pathlib.Path(path) @@ -274,10 +301,10 @@ def main(): # Ensure we can load the correct SDK and toolchain sdks = load_sdks(args.sdks) - sdk, sdk_version = next( + sdk, sdk_name_version = next( (path, version) for path, version in sdks.items() if version == manifest["sdk"] ) - sdk_name = sdk_version.split(":", 1)[0] + sdk_name, _, sdk_version = sdk_name_version.partition(":") toolchains = load_toolchains(args.toolchains) toolchain = next( @@ -370,6 +397,35 @@ def main(): with (args.build_dir / "gbl_metadata.yaml").open("w") as f: yaml.dump(manifest["gbl"], f) + # Template variables + value_template_env = { + "git_repo_hash": get_git_commit_id(repo=pathlib.Path(__file__).parent.parent), + "manifest_name": args.manifest.stem, + "now": datetime.now(timezone.utc), + "sdk": sdk, + "sdk_version": sdk_version, + } + + for json_config in DEFAULT_JSON_CONFIG + manifest.get("json_config", []): + json_path = build_template_path / json_config["file"] + + if json_config.get("skip_if_missing", False) and not json_path.exists(): + continue + + jq_args = { + k: expand_template(v, value_template_env) + for k, v in json_config.get("args", {}).items() + } + + LOGGER.debug("Substituting JQ args for %s: %s", json_path, jq_args) + + result = ( + jq.compile(json_config["jq"], args=jq_args) + .input_text(json_path.read_text()) + .first() + ) + json_path.write_text(json.dumps(result, indent=2)) + # Next, generate a chip-specific project from the modified base project print(f"Generating project for {manifest['device']}") @@ -399,13 +455,6 @@ def main(): LOGGER.error("Referenced extension not present in SDK: %s", expected_dir) sys.exit(1) - # Template variables for C defines - value_template_env = { - "git_repo_hash": get_git_commit_id(repo=pathlib.Path(__file__).parent.parent), - "manifest_name": args.manifest.stem, - "now": datetime.now(timezone.utc), - } - # Actually search for C defines within config unused_defines = set(manifest.get("c_defines", {}).keys()) @@ -416,7 +465,7 @@ def main(): new_config_h_lines = [] for index, line in enumerate(config_h_lines): - for define, value_template in manifest.get("c_defines", {}).items(): + for define, value in manifest.get("c_defines", {}).items(): if f"#define {define} " not in line: continue @@ -441,15 +490,7 @@ def main(): assert re.match(r'#warning ".*? not configured"', prev_line) new_config_h_lines.pop(index - 1) - value_template = str(value_template) - - if value_template.startswith("template:"): - value = value_template.replace("template:", "", 1).format( - **value_template_env - ) - else: - value = value_template - + value = expand_template(value, value_template_env) new_config_h_lines.append(f"#define {define}{alignment}{value}") written_config[define] = value