From cae1077821e8244a932bd4f31d8b476f81ab5cd9 Mon Sep 17 00:00:00 2001 From: Andrzej Aksenczuuk <96312831+aaksenczuk@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:52:24 +0200 Subject: [PATCH 01/28] remove azure deps (#368) * remove azure deps * fix config --------- Co-authored-by: Andrzej --- scargo/file_generators/docker_gen.py | 2 +- .../templates/docker/Dockerfile-atsam.j2 | 3 --- .../templates/docker/Dockerfile-custom.j2 | 13 +++++++++++++ .../templates/docker/Dockerfile-esp32.j2 | 12 ------------ .../templates/docker/Dockerfile-stm32.j2 | 5 +---- .../file_generators/templates/docker/Dockerfile.j2 | 3 --- 6 files changed, 15 insertions(+), 23 deletions(-) diff --git a/scargo/file_generators/docker_gen.py b/scargo/file_generators/docker_gen.py index ca639dce..5c49dcb4 100644 --- a/scargo/file_generators/docker_gen.py +++ b/scargo/file_generators/docker_gen.py @@ -27,7 +27,7 @@ def generate_docker_env(self) -> None: self._create_file_from_template( "docker/Dockerfile-custom.j2", "Dockerfile-custom", - template_params={}, + template_params={"project": self._config.project}, overwrite=False, ) self._create_file_from_template( diff --git a/scargo/file_generators/templates/docker/Dockerfile-atsam.j2 b/scargo/file_generators/templates/docker/Dockerfile-atsam.j2 index daa73428..919d7ff4 100644 --- a/scargo/file_generators/templates/docker/Dockerfile-atsam.j2 +++ b/scargo/file_generators/templates/docker/Dockerfile-atsam.j2 @@ -13,7 +13,4 @@ RUN wget https://developer.arm.com/-/media/Files/downloads/gnu-rm/10.3-2021.10/g # Set up the compiler path ENV PATH $PATH:/opt/gcc-arm-none-eabi/bin -RUN curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash && \ - rm -rf /var/lib/apt/lists/* - ENV LC_ALL=C diff --git a/scargo/file_generators/templates/docker/Dockerfile-custom.j2 b/scargo/file_generators/templates/docker/Dockerfile-custom.j2 index eb03cbb8..a131ad2f 100644 --- a/scargo/file_generators/templates/docker/Dockerfile-custom.j2 +++ b/scargo/file_generators/templates/docker/Dockerfile-custom.j2 @@ -1,3 +1,16 @@ # user can add his docker code below which will be integrated with # main docker file and not overwritten by scargo update +{% if project.target.family == "esp32" %} +# Install QEMU +ARG QEMU_VER=esp-develop-20220919 +ARG QEMU_DIST=qemu-${QEMU_VER}.tar.bz2 +ARG QEMU_SHA256=f6565d3f0d1e463a63a7f81aec94cce62df662bd42fc7606de4b4418ed55f870 +RUN : \ + && wget --no-verbose https://github.com/espressif/qemu/releases/download/${QEMU_VER}/${QEMU_DIST} \ + && echo "${QEMU_SHA256} *${QEMU_DIST}" | sha256sum --check --strict - \ + && tar -xf ${QEMU_DIST} -C /opt \ + && rm ${QEMU_DIST} \ + && : +ENV PATH=/opt/qemu/bin:${PATH} +{% endif %} diff --git a/scargo/file_generators/templates/docker/Dockerfile-esp32.j2 b/scargo/file_generators/templates/docker/Dockerfile-esp32.j2 index 813d81a4..29173743 100644 --- a/scargo/file_generators/templates/docker/Dockerfile-esp32.j2 +++ b/scargo/file_generators/templates/docker/Dockerfile-esp32.j2 @@ -35,15 +35,3 @@ exec "$@"' > /opt/esp/entrypoint.sh && chmod 775 /opt/esp/entrypoint.sh RUN umask 0002 && /opt/esp-idf/python_env/idf4.4_py3.8_env/bin/pip install pyclang RUN umask 0002 && curl -o /usr/bin/run-clang-tidy.py ${RUN_CLANG_TIDY_SCRIPT_URL} && \ chmod 775 /usr/bin/run-clang-tidy.py - -# Install QEMU -ARG QEMU_VER=esp-develop-20220919 -ARG QEMU_DIST=qemu-${QEMU_VER}.tar.bz2 -ARG QEMU_SHA256=f6565d3f0d1e463a63a7f81aec94cce62df662bd42fc7606de4b4418ed55f870 -RUN : \ - && wget --no-verbose https://github.com/espressif/qemu/releases/download/${QEMU_VER}/${QEMU_DIST} \ - && echo "${QEMU_SHA256} *${QEMU_DIST}" | sha256sum --check --strict - \ - && tar -xf ${QEMU_DIST} -C /opt \ - && rm ${QEMU_DIST} \ - && : -ENV PATH=/opt/qemu/bin:${PATH} diff --git a/scargo/file_generators/templates/docker/Dockerfile-stm32.j2 b/scargo/file_generators/templates/docker/Dockerfile-stm32.j2 index aa76a7a4..b3df24d8 100644 --- a/scargo/file_generators/templates/docker/Dockerfile-stm32.j2 +++ b/scargo/file_generators/templates/docker/Dockerfile-stm32.j2 @@ -13,7 +13,4 @@ RUN wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-rm/ # Set up the compiler path ENV PATH $PATH:/opt/gcc-arm-none-eabi/bin -RUN curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash && \ - rm -rf /var/lib/apt/lists/* - -ENV LC_ALL=C \ No newline at end of file +ENV LC_ALL=C diff --git a/scargo/file_generators/templates/docker/Dockerfile.j2 b/scargo/file_generators/templates/docker/Dockerfile.j2 index 43a8b070..baf2f596 100644 --- a/scargo/file_generators/templates/docker/Dockerfile.j2 +++ b/scargo/file_generators/templates/docker/Dockerfile.j2 @@ -65,9 +65,6 @@ RUN pip3 install {{ scargo_package_version }} RUN usermod -a -G dialout ${USER_NAME} USER ${USER_NAME} -{% if project.target.family != "x86" %} -RUN az extension add --name azure-iot -{% endif %} {% if project.target.family == "esp32" %} RUN /bin/bash -c "echo 'source $IDF_PATH/export.sh'" >> ~/.bashrc CMD ["/bin/bash"] From 793b0480876fb4a4899f953d185509693fa4f23d Mon Sep 17 00:00:00 2001 From: Pawel Date: Thu, 12 Oct 2023 10:40:47 +0200 Subject: [PATCH 02/28] support for vscode debug --- scargo/commands/debug.py | 13 +--- scargo/commands/flash.py | 27 ++++---- scargo/commands/update.py | 4 ++ scargo/file_generators/docker_gen.py | 4 ++ .../atsam-gdb.script.j2} | 0 .../atsam-openocd.cfg.j2} | 0 .../templates/docker/devcontainer.json.j2 | 3 +- .../templates/vscode/launch.json.j2 | 20 ++++++ .../templates/vscode/tasks.json.j2 | 40 ++++++++++++ scargo/file_generators/vscode_gen.py | 61 +++++++++++++++++++ scargo/target_helpers/atsam_helper.py | 14 +++-- tests/ut/ut_scargo_update.py | 1 + 12 files changed, 154 insertions(+), 33 deletions(-) rename scargo/file_generators/templates/{atsam/gdb-flash.script.j2 => docker/atsam-gdb.script.j2} (100%) rename scargo/file_generators/templates/{atsam/openocd_script.cfg.j2 => docker/atsam-openocd.cfg.j2} (100%) create mode 100644 scargo/file_generators/templates/vscode/launch.json.j2 create mode 100644 scargo/file_generators/templates/vscode/tasks.json.j2 create mode 100644 scargo/file_generators/vscode_gen.py diff --git a/scargo/commands/debug.py b/scargo/commands/debug.py index 135b8a51..0c0d6464 100644 --- a/scargo/commands/debug.py +++ b/scargo/commands/debug.py @@ -8,7 +8,6 @@ import subprocess import sys from pathlib import Path -from tempfile import TemporaryDirectory from time import sleep from typing import List, Optional @@ -17,7 +16,6 @@ from scargo.docker_utils import run_scargo_again_in_docker from scargo.logger import get_logger from scargo.path_utils import find_program_path -from scargo.target_helpers.atsam_helper import AtsamScrips, generate_openocd_script logger = get_logger() @@ -129,16 +127,7 @@ def _debug_esp32(self) -> None: self._debug_embedded(openocd_args, "xtensa-esp32-elf-gdb") def _debug_atsam(self) -> None: - config = prepare_config() - - temp_script_dir = TemporaryDirectory() - temp_script_dir_path = Path(temp_script_dir.name) - generate_openocd_script(temp_script_dir_path, config) - - openocd_args = [ - "-f", - str(temp_script_dir_path / AtsamScrips.openocd_cfg), - ] + openocd_args = ["-f", ".devcontainer/openocd-script.cfg"] self._debug_embedded(openocd_args, "gdb-multiarch") def _get_bin_path(self, bin_name: str) -> Path: diff --git a/scargo/commands/flash.py b/scargo/commands/flash.py index 4d62acb1..4e4cc7f4 100644 --- a/scargo/commands/flash.py +++ b/scargo/commands/flash.py @@ -7,18 +7,14 @@ import subprocess import sys from pathlib import Path -from tempfile import TemporaryDirectory from typing import Optional from scargo.config import Config from scargo.config_utils import prepare_config +from scargo.file_generators.vscode_gen import generate_launch_json from scargo.logger import get_logger from scargo.path_utils import find_program_path -from scargo.target_helpers.atsam_helper import ( - AtsamScrips, - generate_gdb_script, - generate_openocd_script, -) +from scargo.target_helpers.atsam_helper import AtsamScrips, generate_gdb_script if platform.system() == "Windows": from subprocess import DETACHED_PROCESS # type: ignore[attr-defined] @@ -157,19 +153,18 @@ def flash_atsam( project_path = config.project_root bin_name = f"{config.project.name.lower()}.bin" + elf_name = f"{config.project.name.lower()}" build_path = Path(project_path, "build", flash_profile, "bin") bin_path = build_path / bin_name - exec_path = build_path / config.project.name.lower() + elf_path = build_path / elf_name if not bin_path.exists(): logger.error("%s does not exist", bin_path) logger.info("Did you run scargo build --profile %s", flash_profile) sys.exit(1) - temp_script_dir = TemporaryDirectory() - temp_script_dir_path = Path(temp_script_dir.name) - generate_openocd_script(temp_script_dir_path, config) - generate_gdb_script(temp_script_dir_path, config, bin_path) + generate_gdb_script(Path(".devcontainer"), config, bin_path) + generate_launch_json(Path(".vscode"), config, elf_path) openocd_process = None try: @@ -178,7 +173,9 @@ def flash_atsam( [ openocd_path, "-f", - str(temp_script_dir_path / AtsamScrips.openocd_cfg), + str( + Path(f"{project_path}/.devcontainer") / AtsamScrips.openocd_cfg + ), ], creationflags=DETACHED_PROCESS, ) @@ -188,7 +185,7 @@ def flash_atsam( "sudo", openocd_path, "-f", - str(temp_script_dir_path / AtsamScrips.openocd_cfg), + ".devcontainer/openocd-script.cfg", ], stdin=PIPE, stdout=PIPE, @@ -197,8 +194,8 @@ def flash_atsam( gdb_command = [ str(gdb_multiarch_path), - f"{exec_path}", - f"--command={temp_script_dir_path / AtsamScrips.gdb_flash}", + f"{elf_path}", + "--command=.devcontainer/atsam-gdb.script", "--batch", ] subprocess.check_call(gdb_command) diff --git a/scargo/commands/update.py b/scargo/commands/update.py index cbf73bb5..80c71a53 100644 --- a/scargo/commands/update.py +++ b/scargo/commands/update.py @@ -17,6 +17,7 @@ from scargo.file_generators.env_gen import generate_env from scargo.file_generators.readme_gen import generate_readme from scargo.file_generators.tests_gen import generate_tests +from scargo.file_generators.vscode_gen import generate_vscode from scargo.global_values import SCARGO_DOCKER_ENV, SCARGO_LOCK_FILE, SCARGO_PKG_PATH from scargo.logger import get_logger @@ -44,6 +45,7 @@ def scargo_update(config_file_path: Path) -> None: """ project_path = config_file_path.parent docker_path = Path(project_path, ".devcontainer") + vscode_path = Path(project_path, ".vscode") config = get_scargo_config_or_exit(config_file_path) if not config.project: logger.error("File `%s`: Section `project` not found.", config_file_path) @@ -70,6 +72,8 @@ def scargo_update(config_file_path: Path) -> None: generate_docker_compose(docker_path, config) generate_env(docker_path, config) + generate_vscode(vscode_path, config) + generate_cmake(config) generate_conanfile(config) generate_conanprofile(config) diff --git a/scargo/file_generators/docker_gen.py b/scargo/file_generators/docker_gen.py index ca639dce..809d12e3 100644 --- a/scargo/file_generators/docker_gen.py +++ b/scargo/file_generators/docker_gen.py @@ -11,6 +11,7 @@ from scargo.config import Config from scargo.file_generators.base_gen import create_file_from_template from scargo.global_values import SCARGO_PKG_PATH +from scargo.target_helpers.atsam_helper import generate_openocd_script class _DockerComposeTemplate: @@ -47,6 +48,9 @@ def generate_docker_env(self) -> None: template_params={}, ) + if self._config.project.target.family == "atsam": + generate_openocd_script(self.docker_path, self._config) + custom_docker = self._get_dockerfile_custom_content() scargo_package_version = self._set_up_package_version() diff --git a/scargo/file_generators/templates/atsam/gdb-flash.script.j2 b/scargo/file_generators/templates/docker/atsam-gdb.script.j2 similarity index 100% rename from scargo/file_generators/templates/atsam/gdb-flash.script.j2 rename to scargo/file_generators/templates/docker/atsam-gdb.script.j2 diff --git a/scargo/file_generators/templates/atsam/openocd_script.cfg.j2 b/scargo/file_generators/templates/docker/atsam-openocd.cfg.j2 similarity index 100% rename from scargo/file_generators/templates/atsam/openocd_script.cfg.j2 rename to scargo/file_generators/templates/docker/atsam-openocd.cfg.j2 diff --git a/scargo/file_generators/templates/docker/devcontainer.json.j2 b/scargo/file_generators/templates/docker/devcontainer.json.j2 index 038d5e9f..d7a6b51e 100644 --- a/scargo/file_generators/templates/docker/devcontainer.json.j2 +++ b/scargo/file_generators/templates/docker/devcontainer.json.j2 @@ -28,7 +28,8 @@ "xaver.clang-format", "ms-vscode.cpptools-extension-pack", "streetsidesoftware.code-spell-checker", - "ciarant.vscode-structurizr" + "ciarant.vscode-structurizr", + "coolchyni.beyond-debug" ] } } diff --git a/scargo/file_generators/templates/vscode/launch.json.j2 b/scargo/file_generators/templates/vscode/launch.json.j2 new file mode 100644 index 00000000..296ecb22 --- /dev/null +++ b/scargo/file_generators/templates/vscode/launch.json.j2 @@ -0,0 +1,20 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "GDB", + "preLaunchTask": "Start-OpenOCD", // The name of the task defined above + "postDebugTask": "Kill-OpenOCD", + "type": "by-gdb", + "request": "launch", + "debuggerPath": "gdb-multiarch", + "program": "{{bin_path}}", + "remote": { + "enabled": true, + "address": ":3333", + "mode": "remote", + "execfile": "{{bin_path}}" + } + } + ] +} diff --git a/scargo/file_generators/templates/vscode/tasks.json.j2 b/scargo/file_generators/templates/vscode/tasks.json.j2 new file mode 100644 index 00000000..aed5abcd --- /dev/null +++ b/scargo/file_generators/templates/vscode/tasks.json.j2 @@ -0,0 +1,40 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "shell", + "label": "Start-OpenOCD", + "command": "sudo", + "args": [ + "openocd", + "-f", + ".devcontainer/openocd-script.cfg" + ], + "problemMatcher": { + "pattern": { + "regexp": "^(Info |Warn |Error):(.*)$", + "severity": 1, + "message": 2 + }, + "background": { + "activeOnStart": true, + "beginsPattern": "^.*SWD.*", + "endsPattern": ".*watchpoints.*" + } + }, + "presentation": { + "reveal": "always" + }, + "isBackground": true + }, + { + "label": "Kill-OpenOCD", + "command": "sudo", + "args": [ + "pkill", + "openocd" + ], + "type": "shell" + } + ] +} diff --git a/scargo/file_generators/vscode_gen.py b/scargo/file_generators/vscode_gen.py new file mode 100644 index 00000000..3753100c --- /dev/null +++ b/scargo/file_generators/vscode_gen.py @@ -0,0 +1,61 @@ +# # +# @copyright Copyright (C) 2023 SpyroSoft Solutions S.A. All rights reserved. +# # + + +from pathlib import Path +from typing import Any, Dict + +from scargo.config import Config +from scargo.file_generators.base_gen import create_file_from_template + + +class _VSCodeTemplate: + """ + This class is a container for docker compose yaml files creation with multilayer approach + """ + + def __init__(self, config: Config, vscode_path: Path): + self.vscode_path = vscode_path + self._config = config + + def generate_vscode_env(self) -> None: + """Generate dirs and files""" + + self._create_file_from_template( + "vscode/tasks.json.j2", + "tasks.json", + template_params={"project": self._config.project}, + ) + + def generate_launch_json(self, elf_path: Path) -> None: + self._create_file_from_template( + "vscode/launch.json.j2", + "launch.json", + template_params={"bin_path": elf_path}, + ) + + def _create_file_from_template( + self, + template_path: str, + output_filename: str, + template_params: Dict[str, Any], + overwrite: bool = True, + ) -> None: + create_file_from_template( + template_path, + self.vscode_path / output_filename, + template_params=template_params, + config=self._config, + overwrite=overwrite, + ) + + +def generate_vscode(vscode_path: Path, config: Config) -> None: + vscode_template = _VSCodeTemplate(config, vscode_path) + vscode_template.generate_vscode_env() + + +def generate_launch_json(vscode_path: Path, config: Config, elf_path: Path) -> None: + vscode_template = _VSCodeTemplate(config, vscode_path) + vscode_template.generate_launch_json(elf_path) diff --git a/scargo/target_helpers/atsam_helper.py b/scargo/target_helpers/atsam_helper.py index d9f06b2f..95f1ee3a 100644 --- a/scargo/target_helpers/atsam_helper.py +++ b/scargo/target_helpers/atsam_helper.py @@ -14,8 +14,8 @@ @dataclass class AtsamScrips: - openocd_cfg = "config/openocd_script.cfg" - gdb_flash = "config/gdb-flash.script" + openocd_cfg = "openocd-script.cfg" + gdb_flash = "atsam-gdb.script" def get_atsam_cpu(chip_label: str) -> Optional[str]: @@ -68,8 +68,12 @@ def generate_openocd_script(outdir: Path, config: Config) -> None: if openocd_script_name and openocd_script_name: write_template( outdir / AtsamScrips.openocd_cfg, - "atsam/openocd_script.cfg.j2", - {"flash_driver": flash_driver, "script_name": openocd_script_name}, + "docker/atsam-openocd.cfg.j2", + { + "config": config, + "flash_driver": flash_driver, + "script_name": openocd_script_name, + }, ) @@ -78,6 +82,6 @@ def generate_gdb_script(outdir: Path, config: Config, bin_path: Path) -> None: if flash_driver: write_template( outdir / AtsamScrips.gdb_flash, - "atsam/gdb-flash.script.j2", + "docker/atsam-gdb.script.j2", {"flash_driver": flash_driver, "bin_path": bin_path}, ) diff --git a/tests/ut/ut_scargo_update.py b/tests/ut/ut_scargo_update.py index af8d9b8f..afd61eb1 100644 --- a/tests/ut/ut_scargo_update.py +++ b/tests/ut/ut_scargo_update.py @@ -11,6 +11,7 @@ EXPECTED_FILES_AND_DIRS = [ ".clang-format", ".clang-tidy", + ".vscode", ".devcontainer", ".gitignore", ".gitlab-ci.yml", From a73f9ac39c7b62addfca7a3ce302bf14880a51d1 Mon Sep 17 00:00:00 2001 From: Pawel Date: Fri, 13 Oct 2023 14:03:08 +0200 Subject: [PATCH 03/28] vscode native debug support for stm32 --- scargo/commands/debug.py | 9 +----- scargo/commands/flash.py | 27 ++++++++--------- scargo/file_generators/docker_gen.py | 12 +++----- .../templates/docker/stm32-openocd.cfg.j2 | 4 +++ .../templates/docker/stm32.cfg.j2 | 1 - scargo/target_helpers/stm32_helper.py | 29 +++++++++++++++++++ 6 files changed, 52 insertions(+), 30 deletions(-) create mode 100644 scargo/file_generators/templates/docker/stm32-openocd.cfg.j2 delete mode 100644 scargo/file_generators/templates/docker/stm32.cfg.j2 create mode 100644 scargo/target_helpers/stm32_helper.py diff --git a/scargo/commands/debug.py b/scargo/commands/debug.py index 0c0d6464..6abed7c8 100644 --- a/scargo/commands/debug.py +++ b/scargo/commands/debug.py @@ -107,14 +107,7 @@ def _debug_stm32(self) -> None: chip_script = f"target/{self._chip[:7].lower()}x.cfg" if not Path("/usr/share/openocd/scripts", chip_script).exists(): chip_script = f"target/{self._chip[:7].lower()}.cfg" - openocd_args = [ - "-f", - "interface/stlink-v2-1.cfg", - "-f", - chip_script, - "-f", - ".devcontainer/stm32.cfg", - ] + openocd_args = ["-f", ".devcontainer/openocd-script.cfg"] self._debug_embedded(openocd_args, "gdb-multiarch") def _debug_esp32(self) -> None: diff --git a/scargo/commands/flash.py b/scargo/commands/flash.py index 4e4cc7f4..26ba8869 100644 --- a/scargo/commands/flash.py +++ b/scargo/commands/flash.py @@ -9,12 +9,13 @@ from pathlib import Path from typing import Optional +import scargo.target_helpers.atsam_helper as atsam_helper +import scargo.target_helpers.stm32_helper as stm32_helper from scargo.config import Config from scargo.config_utils import prepare_config from scargo.file_generators.vscode_gen import generate_launch_json from scargo.logger import get_logger from scargo.path_utils import find_program_path -from scargo.target_helpers.atsam_helper import AtsamScrips, generate_gdb_script if platform.system() == "Windows": from subprocess import DETACHED_PROCESS # type: ignore[attr-defined] @@ -103,13 +104,12 @@ def flash_stm32( port: Optional[str] = None, ) -> None: project_path = config.project_root - bin_path = ( - project_path - / "build" - / flash_profile - / "bin" - / f"{config.project.name.lower()}.bin" - ) + + bin_name = f"{config.project.name.lower()}.bin" + elf_name = f"{config.project.name.lower()}.elf" + build_path = Path(project_path, "build", flash_profile, "bin") + bin_path = build_path / bin_name + elf_path = build_path / elf_name flash_start = hex(config.get_stm32_config().flash_start) @@ -133,6 +133,9 @@ def flash_stm32( command.extend(["write", str(bin_path), flash_start]) subprocess.check_call(command) + stm32_helper.generate_openocd_script(Path(".devcontainer"), config) + generate_launch_json(Path(".vscode"), config, elf_path) + def flash_atsam( config: Config, @@ -163,7 +166,7 @@ def flash_atsam( logger.info("Did you run scargo build --profile %s", flash_profile) sys.exit(1) - generate_gdb_script(Path(".devcontainer"), config, bin_path) + atsam_helper.generate_gdb_script(Path(".devcontainer"), config, bin_path) generate_launch_json(Path(".vscode"), config, elf_path) openocd_process = None @@ -173,9 +176,7 @@ def flash_atsam( [ openocd_path, "-f", - str( - Path(f"{project_path}/.devcontainer") / AtsamScrips.openocd_cfg - ), + str(Path(f"{project_path}/.devcontainer/openocd-script.cfg")), ], creationflags=DETACHED_PROCESS, ) @@ -199,7 +200,7 @@ def flash_atsam( "--batch", ] subprocess.check_call(gdb_command) - except Exception as e: + except subprocess.CalledProcessError as e: print(e) finally: # temp_script_dir.cleanup() diff --git a/scargo/file_generators/docker_gen.py b/scargo/file_generators/docker_gen.py index 31095143..392ab946 100644 --- a/scargo/file_generators/docker_gen.py +++ b/scargo/file_generators/docker_gen.py @@ -7,11 +7,12 @@ from pathlib import Path from typing import Any, Dict +import scargo.target_helpers.atsam_helper as atsam_helper +import scargo.target_helpers.stm32_helper as stm32_helper from scargo import __version__ from scargo.config import Config from scargo.file_generators.base_gen import create_file_from_template from scargo.global_values import SCARGO_PKG_PATH -from scargo.target_helpers.atsam_helper import generate_openocd_script class _DockerComposeTemplate: @@ -42,14 +43,9 @@ def generate_docker_env(self) -> None: template_params={"project": self._config.project}, ) if self._config.project.target.family == "stm32": - self._create_file_from_template( - "docker/stm32.cfg.j2", - "stm32.cfg", - template_params={}, - ) - + stm32_helper.generate_openocd_script(self.docker_path, self._config) if self._config.project.target.family == "atsam": - generate_openocd_script(self.docker_path, self._config) + atsam_helper.generate_openocd_script(self.docker_path, self._config) custom_docker = self._get_dockerfile_custom_content() scargo_package_version = self._set_up_package_version() diff --git a/scargo/file_generators/templates/docker/stm32-openocd.cfg.j2 b/scargo/file_generators/templates/docker/stm32-openocd.cfg.j2 new file mode 100644 index 00000000..1e8b9786 --- /dev/null +++ b/scargo/file_generators/templates/docker/stm32-openocd.cfg.j2 @@ -0,0 +1,4 @@ +source [find interface/stlink-v2-1.cfg] +source [find {{chip_script}}] + +gdb_memory_map disable \ No newline at end of file diff --git a/scargo/file_generators/templates/docker/stm32.cfg.j2 b/scargo/file_generators/templates/docker/stm32.cfg.j2 deleted file mode 100644 index 80a1c0cf..00000000 --- a/scargo/file_generators/templates/docker/stm32.cfg.j2 +++ /dev/null @@ -1 +0,0 @@ -gdb_memory_map disable \ No newline at end of file diff --git a/scargo/target_helpers/stm32_helper.py b/scargo/target_helpers/stm32_helper.py new file mode 100644 index 00000000..2b7f3a2e --- /dev/null +++ b/scargo/target_helpers/stm32_helper.py @@ -0,0 +1,29 @@ +from dataclasses import dataclass +from pathlib import Path + +from scargo.config import Config +from scargo.file_generators.base_gen import write_template + +script_dir = Path(__file__).parent +atmel_arxml_path = script_dir / "atmel.xml" + +ADDITIONAL_CPU_DATA = {"cortex-m23": ["atsaml10e16a"]} + + +@dataclass +class STM32Scrips: + openocd_cfg = "openocd-script.cfg" + + +def generate_openocd_script(outdir: Path, config: Config) -> None: + chip = config.get_stm32_config().chip + + chip_script = f"target/{chip[:7].lower()}x.cfg" + if not Path("/usr/share/openocd/scripts", chip_script).exists(): + chip_script = f"target/{chip[:7].lower()}.cfg" + + write_template( + outdir / STM32Scrips.openocd_cfg, + "docker/stm32-openocd.cfg.j2", + {"chip_script": chip_script}, + ) From bfbee0d91d8e454dc432495b1ff8c06a82ab0fee Mon Sep 17 00:00:00 2001 From: Andrzej Aksenczuuk <96312831+aaksenczuk@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:04:04 +0200 Subject: [PATCH 04/28] fix new generating pipeline file outside the folder (#371) * fix new generating pipeline file outside the folder * fix after rev * correct file path --------- Co-authored-by: Andrzej --- scargo/file_generators/cicd_gen.py | 34 ++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/scargo/file_generators/cicd_gen.py b/scargo/file_generators/cicd_gen.py index dc1c8a73..dee9e2ea 100644 --- a/scargo/file_generators/cicd_gen.py +++ b/scargo/file_generators/cicd_gen.py @@ -46,16 +46,15 @@ def merge_dictionaries( dict1[key] = dict2[key] return dict1 - def _get_cicd_content(self, path: str, file_name: str) -> Dict[str, Any]: - ci_folder = Path(path) - file_path = ci_folder / file_name + def _get_cicd_content(self, cicd_dir_path: Path, file_name: str) -> Dict[str, Any]: + file_path = cicd_dir_path / file_name # Check if the ci folder exists. If not, create it. - if not ci_folder.exists(): - ci_folder.mkdir(parents=True) + if not cicd_dir_path.is_dir(): + cicd_dir_path.mkdir(parents=True) # Check if file exists. If not, create it. - if not file_path.exists(): + if not file_path.is_file(): file_path.touch() # Read the file @@ -66,6 +65,16 @@ def _get_cicd_content(self, path: str, file_name: str) -> Dict[str, Any]: return {} return data + def _generate_custom_cicd(self, custom_cicd_dict: Dict[str, Any]) -> None: + """Generate custom cicd file""" + # Merge custom_cicd with base_cicd + base_cicd = self._get_cicd_content(self._config.project_root, ".gitlab-ci.yml") + merged_dict = self.merge_dictionaries(base_cicd, custom_cicd_dict) + with open( + self._config.project_root / ".gitlab-ci.yml", "w", encoding="utf-8" + ) as f: + yaml.dump(merged_dict, f, sort_keys=False) + def generate_cicd_env(self) -> None: """Generate dirs and files""" create_file_from_template( @@ -84,18 +93,17 @@ def generate_cicd_env(self) -> None: }, config=self._config, ) - custom_cicd = self._get_cicd_content(".devcontainer", ".gitlab-ci-custom.yml") + project_path = self._config.project_root + output_path = Path(project_path, ".devcontainer") + + custom_cicd = self._get_cicd_content(output_path, ".gitlab-ci-custom.yml") # Check if custom_cicd is empty and if so, do not merge - if custom_cicd is None: + if not custom_cicd: logger.info("Custom file is empty or currupted. Skipping cicd file merge.") return - # Merge custom_cicd with base_cicd - base_cicd = self._get_cicd_content(".", ".gitlab-ci.yml") - merged_dict = self.merge_dictionaries(base_cicd, custom_cicd) - with open(".gitlab-ci.yml", "w", encoding="utf-8") as f: - yaml.dump(merged_dict, f, sort_keys=False) + self._generate_custom_cicd(custom_cicd) def generate_cicd(config: Config) -> None: From 6fdb0bd153559aeb157a22731a0451dbd5fdd281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Tue, 17 Oct 2023 13:41:53 +0200 Subject: [PATCH 05/28] Removed check parameter from subprocess.run in pull_docker_image function --- scargo/commands/update.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scargo/commands/update.py b/scargo/commands/update.py index cbf73bb5..49e43676 100644 --- a/scargo/commands/update.py +++ b/scargo/commands/update.py @@ -109,7 +109,6 @@ def pull_docker_image(docker_path: Path) -> bool: cmd, cwd=docker_path, stderr=subprocess.PIPE, - check=True, ) except subprocess.CalledProcessError as e: logger.warning(e.stderr.decode()) From f84b2ab4b256f9c5392b2ba47a20563fd97a062a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Wed, 18 Oct 2023 09:48:28 +0200 Subject: [PATCH 06/28] Changed output of pull_docker_image to just an information --- scargo/commands/update.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scargo/commands/update.py b/scargo/commands/update.py index 49e43676..2a8c1eb8 100644 --- a/scargo/commands/update.py +++ b/scargo/commands/update.py @@ -109,9 +109,12 @@ def pull_docker_image(docker_path: Path) -> bool: cmd, cwd=docker_path, stderr=subprocess.PIPE, + check=True, + ) + except subprocess.CalledProcessError: + logger.info( + "No docker image does exist yet in the registry or you are not login" ) - except subprocess.CalledProcessError as e: - logger.warning(e.stderr.decode()) else: # happens for the default tag, like "myproject-dev:1.0" if ( From 82bc19030bbe897fe83b7e019ea15fe7db4a91b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Thu, 19 Oct 2023 08:35:18 +0200 Subject: [PATCH 07/28] updated it_scargo_commands_flow.py --- tests/it/it_scargo_commands_flow.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/it/it_scargo_commands_flow.py b/tests/it/it_scargo_commands_flow.py index b584673d..eefebeb0 100644 --- a/tests/it/it_scargo_commands_flow.py +++ b/tests/it/it_scargo_commands_flow.py @@ -322,18 +322,6 @@ def test_cli_build(self, test_state: ActiveTestState) -> None: for file in test_state.get_build_result_files_paths(): assert file.is_file(), f"Expected file: {file} not exist" - @pytest.mark.skip("TODO when conan version be updated to >= 2.0.0") - @pytest.mark.order(after="test_cli_build") - def test_cli_publish(self, test_state: ActiveTestState) -> None: - """This test check if call of scargo publish command will finish without error - - Note: Conan Server is required to properly execute cmd in integration test flow. The server - works for conan version 2.0.0 and higher, the same version is required for scargo conan to keep compatibility. - The test needs to be implemented when correct version of conan will be applied. - """ - # Publish command - pass - @pytest.mark.order(after="test_cli_build") def test_cli_clean(self, test_state: ActiveTestState) -> None: """This test check if call of scargo clean command will finish without error and From 9d71c5f20f87621c22a4b3253838d2f3f05edcf4 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Thu, 19 Oct 2023 14:50:50 +0200 Subject: [PATCH 08/28] Release 1.8.0 (#376) * Version 1.8.0 --- scargo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scargo/__init__.py b/scargo/__init__.py index 48e69360..d03a66e4 100644 --- a/scargo/__init__.py +++ b/scargo/__init__.py @@ -1,4 +1,4 @@ # # # @copyright Copyright (C) 2023 SpyroSoft Solutions S.A. All rights reserved. # # -__version__ = "1.7.0" +__version__ = "1.8.0" From 05745fc2204eb0f096cfa20ba562f551acac0f64 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Tue, 24 Oct 2023 09:55:31 +0200 Subject: [PATCH 09/28] Upgrade to conan2 (#375) Upgrade conan vesion to 2.x --------- Co-authored-by: Pawel Warzecha --- ci/requirements.txt | 14 +--- example_project/conanfile.py | 2 +- pyproject.toml | 2 +- scargo/cli.py | 3 +- scargo/commands/build.py | 41 ++++----- scargo/commands/check.py | 2 +- scargo/commands/flash.py | 9 +- scargo/commands/publish.py | 28 ++----- scargo/commands/test.py | 23 ++++- scargo/docker_utils.py | 10 +-- scargo/file_generators/conan_gen.py | 19 ++++- scargo/file_generators/docker_gen.py | 3 +- .../templates/conan/conanfile.py.j2 | 27 +++--- .../templates/conan/conanfiletest.j2 | 5 +- .../templates/conan/profile_atsam.j2 | 17 +--- .../templates/conan/profile_esp32.j2 | 18 ++-- .../templates/conan/profile_stm32.j2 | 24 ++---- .../templates/conan/profile_x86.j2 | 9 +- .../stm32_gcc_toolchain_wrapper.cmake.j2 | 3 +- .../file_generators/templates/esp32.cmake.j2 | 4 +- .../templates/tests/CMakeLists-test.txt.j2 | 6 +- scargo/logger.py | 5 +- tests/it/it_scargo_commands_flow.py | 10 +-- tests/ut/ut_scargo_publish.py | 83 +++++++++---------- tests/ut/ut_scargo_test.py | 36 ++++++-- tests/ut/ut_scargo_update.py | 4 + 26 files changed, 200 insertions(+), 207 deletions(-) diff --git a/ci/requirements.txt b/ci/requirements.txt index ac903ed2..532292be 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -20,8 +20,6 @@ bitstring==4.0.1 # via esptool black==23.1.0 # via scargo (pyproject.toml) -bottle==0.12.25 - # via conan certifi==2022.12.7 # via requests cffi==1.15.1 @@ -46,7 +44,7 @@ coloredlogs==15.0.1 # via scargo (pyproject.toml) commonmark==0.9.1 # via recommonmark -conan==1.59.0 +conan==2.0.13 # via scargo (pyproject.toml) contourpy==1.0.7 # via matplotlib @@ -144,8 +142,6 @@ mypy-extensions==1.0.0 # via # black # mypy -node-semver==0.6.1 - # via conan nodeenv==1.7.0 # via pre-commit numpy==1.24.2 @@ -174,8 +170,6 @@ platformdirs==3.1.1 # virtualenv pluggy==1.0.0 # via pytest -pluginbase==1.0.1 - # via conan pre-commit==3.1.1 # via scargo (pyproject.toml) pyclean==2.2.0 @@ -194,15 +188,12 @@ pyflakes==3.0.1 # via flake8 pygments==2.14.0 # via - # conan # gcovr # sphinx pyinstaller==5.9.0 # via scargo (pyproject.toml) pyinstaller-hooks-contrib==2023.0 # via pyinstaller -pyjwt==2.6.0 - # via conan pylint==2.17.0 # via scargo (pyproject.toml) pynacl==1.5.0 @@ -257,7 +248,6 @@ shellingham==1.5.0.post1 # via scargo (pyproject.toml) six==1.16.0 # via - # conan # ecdsa # pytest-assume # python-dateutil @@ -301,8 +291,6 @@ tomlkit==0.11.6 # via # pylint # scargo (pyproject.toml) -tqdm==4.65.0 - # via conan typer==0.7.0 # via scargo (pyproject.toml) types-clang==0.14.3 diff --git a/example_project/conanfile.py b/example_project/conanfile.py index 72da362a..9592acf2 100644 --- a/example_project/conanfile.py +++ b/example_project/conanfile.py @@ -1,4 +1,4 @@ -from conans import CMake, ConanFile, tools # type: ignore[import] +from conan import CMake, ConanFile, tools # type: ignore[import] class Example_projectConan(ConanFile): # type: ignore[misc, no-any-unimported] diff --git a/pyproject.toml b/pyproject.toml index b1ac5bd3..24c57a35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "clang==16.0.1.1", "cmake==3.25.2", "coloredlogs==15.0.1", - "conan==1.59.0", + "conan==2.0.13", "docker==6.0.1", "esptool==4.5.1", "jinja2==3.1.2", diff --git a/scargo/cli.py b/scargo/cli.py index ecc378e8..0ca5b4ff 100644 --- a/scargo/cli.py +++ b/scargo/cli.py @@ -224,7 +224,8 @@ def flash( ), port: Optional[str] = Option( None, - help="(esp32 only) port where the target device of the command is connected to, e.g. /dev/ttyUSB0", + help="(esp32 only) port where the target device of the command is" + " connected to, e.g. /dev/ttyUSB0", ), no_erase: bool = Option(False, help="(stm32 only) Don't erase target memory"), base_dir: Optional[Path] = BASE_DIR_OPTION, diff --git a/scargo/commands/build.py b/scargo/commands/build.py index 5be33ebf..3e2f23d5 100644 --- a/scargo/commands/build.py +++ b/scargo/commands/build.py @@ -7,13 +7,9 @@ import sys from pathlib import Path -from scargo.commands.publish import ( - conan_add_conancenter, - conan_add_remote, - conan_clean_remote, - conan_source, -) +from scargo.commands.publish import conan_add_remote, conan_source from scargo.config_utils import prepare_config +from scargo.file_generators.conan_gen import conan_add_default_profile_if_missing from scargo.logger import get_logger logger = get_logger() @@ -41,49 +37,48 @@ def scargo_build(profile: str) -> None: build_dir = Path(project_dir, "build", profile) build_dir.mkdir(parents=True, exist_ok=True) - conan_clean_remote() - + conan_add_default_profile_if_missing() conan_add_remote(project_dir, config) - conan_add_conancenter() conan_source(project_dir) try: - subprocess.check_call( + subprocess.run( [ "conan", "install", ".", - "-if", - build_dir, - "-of", - build_dir, - "-pr:b", - "default", - "-pr:h", + "-pr", f"./config/conan/profiles/{config.project.target.family}_{profile}", + "-of", + f"build/{profile}", "-b", "missing", ], cwd=project_dir, + check=True, ) - subprocess.check_call( + subprocess.run( [ "conan", "build", - f"{project_dir}", - "-bf", - build_dir, + ".", + "-pr", + f"./config/conan/profiles/{config.project.target.family}_{profile}", + "-of", + f"build/{profile}", ], - cwd=build_dir, + cwd=project_dir, + check=True, ) get_logger().info("Copying artifacts...") # This is a workaround so that different profiles can work together with conan # Conan always calls CMake with ' - subprocess.check_call( + subprocess.run( f"cp -r -l -f {build_dir}/build/{config.profiles[profile].cmake_build_type}/* .", cwd=build_dir, shell=True, + check=True, ) get_logger().info("Artifacts copied") diff --git a/scargo/commands/check.py b/scargo/commands/check.py index dd767e1f..096addf4 100644 --- a/scargo/commands/check.py +++ b/scargo/commands/check.py @@ -20,7 +20,7 @@ logger = get_logger() -def scargo_check( +def scargo_check( # pylint: disable=too-many-branches clang_format: bool, clang_tidy: bool, copy_right: bool, diff --git a/scargo/commands/flash.py b/scargo/commands/flash.py index 26ba8869..275ce8fb 100644 --- a/scargo/commands/flash.py +++ b/scargo/commands/flash.py @@ -9,13 +9,12 @@ from pathlib import Path from typing import Optional -import scargo.target_helpers.atsam_helper as atsam_helper -import scargo.target_helpers.stm32_helper as stm32_helper from scargo.config import Config from scargo.config_utils import prepare_config from scargo.file_generators.vscode_gen import generate_launch_json from scargo.logger import get_logger from scargo.path_utils import find_program_path +from scargo.target_helpers import atsam_helper, stm32_helper if platform.system() == "Windows": from subprocess import DETACHED_PROCESS # type: ignore[attr-defined] @@ -121,13 +120,13 @@ def flash_stm32( logger.info("Define flash-start in scargo.toml under stm32 section") else: if erase_memory: - command = ["st-flash"] + command = ["sudo", "st-flash"] if port: command.append(f"--serial={port}") command.append("erase") subprocess.check_call(command) - command = ["st-flash", "--reset"] + command = ["sudo", "st-flash", "--reset"] if port: command.append(f"--serial={port}") command.extend(["write", str(bin_path), flash_start]) @@ -181,7 +180,7 @@ def flash_atsam( creationflags=DETACHED_PROCESS, ) else: - openocd_process = subprocess.Popen( + openocd_process = subprocess.Popen( # pylint: disable=consider-using-with [ "sudo", openocd_path, diff --git a/scargo/commands/publish.py b/scargo/commands/publish.py index 3125a757..841cae38 100644 --- a/scargo/commands/publish.py +++ b/scargo/commands/publish.py @@ -35,9 +35,7 @@ def scargo_publish(repo: str, profile: str = "Release") -> None: logger.info(f"Did you run 'scargo build --profile {profile}'?") sys.exit(1) - conan_clean_remote() conan_add_remote(project_path, config) - conan_add_conancenter() conan_source(project_path) # Export package @@ -47,7 +45,7 @@ def scargo_publish(repo: str, profile: str = "Release") -> None: "conan", "export-pkg", ".", - "-if", + "-of", str(build_dir), "-pr:b", "default", @@ -110,29 +108,19 @@ def conan_add_remote(project_path: Path, config: Config) -> None: conan_repo = config.conan.repo for repo_name, repo_url in conan_repo.items(): try: - subprocess.check_call( + subprocess.run( ["conan", "remote", "add", repo_name, repo_url], cwd=project_path, + check=True, + stderr=subprocess.PIPE, ) - except subprocess.CalledProcessError: - logger.error("Unable to add remote repository") + except subprocess.CalledProcessError as e: + if b"already exists in remotes" not in e.stderr: + logger.error(e.stderr.decode().strip()) + logger.error("Unable to add remote repository") conan_add_user(repo_name) -def conan_add_conancenter() -> None: - """ - Add conancenter remote repository - - :return: None - """ - try: - subprocess.check_call( - "conan remote add conancenter https://center.conan.io", shell=True - ) - except subprocess.CalledProcessError: - logger.error("Unable to add conancenter remote repository") - - def conan_clean_remote() -> None: """ Clean all remote repositories diff --git a/scargo/commands/test.py b/scargo/commands/test.py index 8fe36667..b2caf1e4 100644 --- a/scargo/commands/test.py +++ b/scargo/commands/test.py @@ -10,6 +10,7 @@ from scargo.config import Config from scargo.config_utils import prepare_config +from scargo.file_generators.conan_gen import conan_add_default_profile_if_missing from scargo.logger import get_logger logger = get_logger() @@ -41,23 +42,37 @@ def scargo_test( sys.exit(1) test_build_dir.mkdir(parents=True, exist_ok=True) + conan_add_default_profile_if_missing() try: # Run CMake and build tests. - subprocess.check_call( + subprocess.run( [ "conan", "install", tests_src_dir, - "-if", + "-of", test_build_dir, f"-sbuild_type={profile}", + "-b", + "missing", ], cwd=project_dir, + check=True, ) - subprocess.check_call( - ["conan", "build", tests_src_dir, "-bf", test_build_dir], + subprocess.run( + [ + "conan", + "build", + "-of", + test_build_dir, + tests_src_dir, + f"-sbuild_type={profile}", + "-b", + "missing", + ], cwd=project_dir, + check=True, ) except subprocess.CalledProcessError: logger.error("Failed to build tests.") diff --git a/scargo/docker_utils.py b/scargo/docker_utils.py index 38721d25..c02585ee 100644 --- a/scargo/docker_utils.py +++ b/scargo/docker_utils.py @@ -48,13 +48,9 @@ def run_scargo_again_in_docker( return cmd_args = sys.argv[1:] - try: - # Correct base-dir arg for command in docker - for idx, val in enumerate(cmd_args): - if val == "-B" or val == "--base-dir": - cmd_args[idx + 1] = "." - except Exception as e: - logger.error(e) + for idx, val in enumerate(cmd_args): + if val in ("-B", "--base-dir"): + cmd_args[idx + 1] = "." result = run_command_in_docker( command=["scargo", *cmd_args], **prepare_docker(project_config, project_path) diff --git a/scargo/file_generators/conan_gen.py b/scargo/file_generators/conan_gen.py index 104d455b..f97b90e8 100644 --- a/scargo/file_generators/conan_gen.py +++ b/scargo/file_generators/conan_gen.py @@ -2,6 +2,8 @@ # @copyright Copyright (C) 2023 SpyroSoft Solutions S.A. All rights reserved. # # +import subprocess + from scargo.config import Config from scargo.file_generators.base_gen import create_file_from_template @@ -28,7 +30,7 @@ def generate_conanprofile(config: Config) -> None: create_file_from_template( "conan/stm32_gcc_toolchain_wrapper.cmake.j2", "config/conan/profiles/stm32_gcc_toolchain_wrapper.cmake", - template_params={}, + template_params={"config": config}, config=config, ) @@ -42,3 +44,18 @@ def generate_conanprofile(config: Config) -> None: }, config=config, ) + + +def conan_add_default_profile_if_missing() -> None: + result = subprocess.run( + ["conan", "profile", "list"], + stdout=subprocess.PIPE, + check=True, + ) + if b"default" not in result.stdout.splitlines(): + subprocess.run( + ["conan", "profile", "detect"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) diff --git a/scargo/file_generators/docker_gen.py b/scargo/file_generators/docker_gen.py index 392ab946..68c93075 100644 --- a/scargo/file_generators/docker_gen.py +++ b/scargo/file_generators/docker_gen.py @@ -7,12 +7,11 @@ from pathlib import Path from typing import Any, Dict -import scargo.target_helpers.atsam_helper as atsam_helper -import scargo.target_helpers.stm32_helper as stm32_helper from scargo import __version__ from scargo.config import Config from scargo.file_generators.base_gen import create_file_from_template from scargo.global_values import SCARGO_PKG_PATH +from scargo.target_helpers import atsam_helper, stm32_helper class _DockerComposeTemplate: diff --git a/scargo/file_generators/templates/conan/conanfile.py.j2 b/scargo/file_generators/templates/conan/conanfile.py.j2 index 6999540c..68f1c6e9 100644 --- a/scargo/file_generators/templates/conan/conanfile.py.j2 +++ b/scargo/file_generators/templates/conan/conanfile.py.j2 @@ -4,11 +4,11 @@ # from conan import ConanFile -from conans import tools -from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout +from conan.tools.scm import Git +from conan.tools.files import copy, get +from conan.tools.cmake import CMake, cmake_layout {% if config.project.target.family == "stm32" %} -from conans import tools import os {% endif %} @@ -19,7 +19,7 @@ class {{ config.project.name|capitalize|replace("-", "") }}Conan(ConanFile): # description = "{{ config.project.description }}" settings = "os", "compiler", "build_type", "arch" url = "{{ config.project.homepage_url }}" - generators = "CMakeDeps" + generators = "CMakeToolchain", "CMakeDeps" exports_sources = ["{{ config.source_dir_path.relative_to(config.project_root) }}/*", "CMakeLists.txt", "include/*", "config/*"] {% if config.dependencies.build or config.dependencies.tool %} def build_requirements(self) -> None: @@ -40,9 +40,10 @@ class {{ config.project.name|capitalize|replace("-", "") }}Conan(ConanFile): # def source(self) -> None: {% if config.project.target.family == "stm32" %} - git = tools.Git(folder="third-party/stm32-cmake") - if not os.listdir(git.folder): - git.clone("https://github.com/ObKo/stm32-cmake.git", "master") + target_dir = "third-party/stm32-cmake" + if not os.path.isdir(target_dir): + git = Git(self) + git.clone("https://github.com/ObKo/stm32-cmake.git", target=target_dir) {% elif config.project.target.family == "atsam" %} import requests import re @@ -64,11 +65,11 @@ class {{ config.project.name|capitalize|replace("-", "") }}Conan(ConanFile): # re.MULTILINE | re.DOTALL, ) dfp_filename = match_atpack.group(1) - tools.get(f"{packs_url}/{dfp_filename}", destination=dfp_outdir) + get(self, f"{packs_url}/{dfp_filename}", destination=dfp_outdir) if not cmsis_outdir.is_dir(): cmsis_filename = "ARM.CMSIS.5.4.0.atpack" - tools.get(f"{packs_url}/{cmsis_filename}", destination=cmsis_outdir) + get(self, f"{packs_url}/{cmsis_filename}", destination=cmsis_outdir) {% else %} pass {% endif %} @@ -76,10 +77,6 @@ class {{ config.project.name|capitalize|replace("-", "") }}Conan(ConanFile): # def layout(self) -> None: cmake_layout(self) - def generate(self) -> None: - tc = CMakeToolchain(self) - tc.generate() - def build(self) -> None: cmake = CMake(self) cmake.configure() @@ -90,8 +87,8 @@ class {{ config.project.name|capitalize|replace("-", "") }}Conan(ConanFile): # cmake = CMake(self) cmake.install() {% else %} - self.copy("*.bin") - self.copy("*.elf") + copy("*.bin") + copy("*.elf") {% endif %} {% if config.project.lib_name %} diff --git a/scargo/file_generators/templates/conan/conanfiletest.j2 b/scargo/file_generators/templates/conan/conanfiletest.j2 index e07b2546..550ac068 100644 --- a/scargo/file_generators/templates/conan/conanfiletest.j2 +++ b/scargo/file_generators/templates/conan/conanfiletest.j2 @@ -3,7 +3,8 @@ # This file is generated by `scargo update`. # -from conans import CMake, ConanFile, tools # type: ignore[import] +from conan import ConanFile +from conan.tools.cmake import CMake # type: ignore[import] class {{ config.project.name|capitalize|replace("-", "") }}TestConan(ConanFile): # type: ignore[misc, no-any-unimported] @@ -12,7 +13,7 @@ class {{ config.project.name|capitalize|replace("-", "") }}TestConan(ConanFile): settings = "os", "compiler", "build_type", "arch" description = "Tests for {{ config.project.name }}" url = "{{ config.project.homepage_url }}" - generators = "cmake_find_package", "cmake" + generators = "CMakeToolchain", "CMakeDeps" def build(self) -> None: cmake = CMake(self) diff --git a/scargo/file_generators/templates/conan/profile_atsam.j2 b/scargo/file_generators/templates/conan/profile_atsam.j2 index d960db73..a366b7cf 100644 --- a/scargo/file_generators/templates/conan/profile_atsam.j2 +++ b/scargo/file_generators/templates/conan/profile_atsam.j2 @@ -1,22 +1,16 @@ -target_host=arm-none-eabi -standalone_toolchain=arm-none-eabi -cc_compiler=gcc -cxx_compiler=g++ - [settings] os=baremetal -arch=armv8 +arch=armv6 compiler=gcc compiler.version=10 compiler.libcxx=libstdc++ compiler.cppstd={{ config.project.cxxstandard }} -arch_build=x86_64 build_type={{ config.profiles[profile].cmake_build_type }} -[env] +[buildenv] CONAN_CMAKE_FIND_ROOT_PATH=/opt/gcc-arm-none-eabi/lib/ -CC=$target_host-$cc_compiler -CXX=$target_host-$cxx_compiler +CC=arm-none-eabi-gcc +CXX=arm-none-eabi-g++ [conf] tools.build:cflags=["{{ config.project.cflags if config.project.cflags}} {{config.profiles.get(profile).cflags if config.profiles.get(profile).cflags}}"] @@ -25,6 +19,3 @@ tools.build:cxxflags=["{{ config.project.cxxflags if config.project.cxxflags }} {% if config.project.max_build_jobs != None %} tools.build:jobs={{config.project.max_build_jobs}} {% endif %} - -[options] -[build_requires] diff --git a/scargo/file_generators/templates/conan/profile_esp32.j2 b/scargo/file_generators/templates/conan/profile_esp32.j2 index 5f280ce5..fd9d066e 100644 --- a/scargo/file_generators/templates/conan/profile_esp32.j2 +++ b/scargo/file_generators/templates/conan/profile_esp32.j2 @@ -1,22 +1,16 @@ -[env] -WORKAROUND_FOR_ESP32_C_FLAGS="{{ config.project.cflags if config.project.cflags}} {{config.profiles.get(profile).cflags if config.profiles.get(profile).cflags}}" -WORKAROUND_FOR_ESP32_CXX_FLAGS="{{ config.project.cxxflags if config.project.cxxflags }} {{ config.profiles.get(profile).cxxflags if config.profiles.get(profile).cxxflags }}" - [settings] +os=baremetal +arch=xtensalx6 compiler=gcc compiler.version=8 compiler.cppstd={{ config.project.cxxstandard }} compiler.libcxx=libstdc++11 -os_build=Linux -arch_build=x86_64 - -os=baremetal -arch=xtensalx6 build_type={{ config.profiles[profile].cmake_build_type }} +[buildenv] +WORKAROUND_FOR_ESP32_C_FLAGS={{ config.project.cflags if config.project.cflags}} {{config.profiles.get(profile).cflags if config.profiles.get(profile).cflags}} +WORKAROUND_FOR_ESP32_CXX_FLAGS={{ config.project.cxxflags if config.project.cxxflags }} {{ config.profiles.get(profile).cxxflags if config.profiles.get(profile).cxxflags }} + [conf] tools.cmake.cmaketoolchain:user_toolchain=["/opt/esp-idf/tools/cmake/toolchain-esp32.cmake"] tools.cmake.cmaketoolchain:generator=Ninja - -[options] -[build_requires] \ No newline at end of file diff --git a/scargo/file_generators/templates/conan/profile_stm32.j2 b/scargo/file_generators/templates/conan/profile_stm32.j2 index 6420cb8a..9ddbf6dc 100644 --- a/scargo/file_generators/templates/conan/profile_stm32.j2 +++ b/scargo/file_generators/templates/conan/profile_stm32.j2 @@ -1,28 +1,22 @@ -[env] -STM32_CHIP={{config.stm32.chip}} -STM32_TOOLCHAIN_PATH=/opt/gcc-arm-none-eabi -STM32_TARGET_TRIPLET=arm-none-eabi - [settings] +os=baremetal +arch=armv7 compiler=gcc compiler.version=9 compiler.libcxx=libstdc++ compiler.cppstd={{ config.project.cxxstandard }} -os=baremetal -arch=armv7 -os_build=Linux -arch_build=x86_64 - build_type={{ config.profiles[profile].cmake_build_type }} +[buildenv] +STM32_CHIP={{config.stm32.chip}} +STM32_TOOLCHAIN_PATH=/opt/gcc-arm-none-eabi +STM32_TARGET_TRIPLET=arm-none-eabi + [conf] tools.build:cflags=["{{ config.project.cflags if config.project.cflags}} {{config.profiles.get(profile).cflags if config.profiles.get(profile).cflags}}"] tools.build:cxxflags=["{{ config.project.cxxflags if config.project.cxxflags }} {{ config.profiles.get(profile).cxxflags if config.profiles.get(profile).cxxflags }}"] - -tools.cmake.cmaketoolchain:user_toolchain=["/workspace/config/conan/profiles/stm32_gcc_toolchain_wrapper.cmake"] {% if config.project.max_build_jobs != None %} tools.build:jobs={{config.project.max_build_jobs}} {% endif %} - -[options] -[build_requires] +{% set workspace_dir = config.project_root if config.project.build_env != "docker" else "/workspace" %} +tools.cmake.cmaketoolchain:user_toolchain=["{{ workspace_dir }}/config/conan/profiles/stm32_gcc_toolchain_wrapper.cmake"] diff --git a/scargo/file_generators/templates/conan/profile_x86.j2 b/scargo/file_generators/templates/conan/profile_x86.j2 index 4d226d87..f2c80a54 100644 --- a/scargo/file_generators/templates/conan/profile_x86.j2 +++ b/scargo/file_generators/templates/conan/profile_x86.j2 @@ -1,4 +1,4 @@ -[env] +[buildenv] {% if config.profiles[profile].cc %} CC={{ config.profiles[profile].cc }} {% else %} @@ -17,18 +17,11 @@ compiler.libcxx=libstdc++ compiler.cppstd={{ config.project.cxxstandard }} os=Linux arch=x86_64 -os_build=Linux -arch_build=x86_64 - build_type={{ config.profiles[profile].cmake_build_type }} - [conf] tools.build:cflags=["{{ config.project.cflags if config.project.cflags}} {{config.profiles.get(profile).cflags if config.profiles.get(profile).cflags}}"] tools.build:cxxflags=["{{ config.project.cxxflags if config.project.cxxflags }} {{ config.profiles.get(profile).cxxflags if config.profiles.get(profile).cxxflags }}"] {% if config.project.max_build_jobs != None %} tools.build:jobs={{config.project.max_build_jobs}} {% endif %} - -[options] -[build_requires] diff --git a/scargo/file_generators/templates/conan/stm32_gcc_toolchain_wrapper.cmake.j2 b/scargo/file_generators/templates/conan/stm32_gcc_toolchain_wrapper.cmake.j2 index 2e69a39a..d0603d67 100644 --- a/scargo/file_generators/templates/conan/stm32_gcc_toolchain_wrapper.cmake.j2 +++ b/scargo/file_generators/templates/conan/stm32_gcc_toolchain_wrapper.cmake.j2 @@ -1,4 +1,5 @@ -include(/workspace/third-party/stm32-cmake/cmake/stm32_gcc.cmake) +{% set workspace_dir = config.project_root if config.project.build_env != "docker" else "/workspace" %} +include({{ workspace_dir }}/third-party/stm32-cmake/cmake/stm32_gcc.cmake) stm32_get_chip_info($ENV{STM32_CHIP} FAMILY STM32_FAMILY DEVICE STM32_DEVICE TYPE STM32_TYPE) diff --git a/scargo/file_generators/templates/esp32.cmake.j2 b/scargo/file_generators/templates/esp32.cmake.j2 index 71c2918b..2532d7df 100644 --- a/scargo/file_generators/templates/esp32.cmake.j2 +++ b/scargo/file_generators/templates/esp32.cmake.j2 @@ -1,5 +1,5 @@ -# TODO: Below lines are caused by problems with idf tolchain file which we had to use. But enforced usage of tolchain block normal pass of argument by conan generato. -# We can do it better, without such haks in code. But it is not for today task +{# TODO: Below lines are caused by problems with idf tolchain file which we had to use. But enforced usage of tolchain block normal pass of argument by conan generato. +We can do it better, without such haks in code. But it is not for today task #} set (CMAKE_CXX_STANDARD {{ config.project.cxxstandard }}) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} $ENV{WORKAROUND_FOR_ESP32_C_FLAGS}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} $ENV{WORKAROUND_FOR_ESP32_CXX_FLAGS}") diff --git a/scargo/file_generators/templates/tests/CMakeLists-test.txt.j2 b/scargo/file_generators/templates/tests/CMakeLists-test.txt.j2 index 8ff9acfa..3409f88e 100644 --- a/scargo/file_generators/templates/tests/CMakeLists-test.txt.j2 +++ b/scargo/file_generators/templates/tests/CMakeLists-test.txt.j2 @@ -32,10 +32,8 @@ SET({{ key }} {{ value }}) project(tests LANGUAGES C CXX ASM) -if(EXISTS ${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) - include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) - conan_basic_setup() -endif() +find_package(GTest REQUIRED) +link_libraries(GTest::GTest GTest::Main) enable_testing() diff --git a/scargo/logger.py b/scargo/logger.py index f6441445..ce12a265 100644 --- a/scargo/logger.py +++ b/scargo/logger.py @@ -24,7 +24,10 @@ def __get_logging_config() -> Tuple[int, int]: console_log_level = logging.getLevelName(scargo_config.console_log_level) file_log_level = logging.getLevelName(scargo_config.file_log_level) finally: - return console_log_level, file_log_level # pylint: disable=lost-exception + return ( # pylint: disable=[lost-exception,return-in-finally] + console_log_level, + file_log_level, + ) def get_logger(name: str = "scargo") -> logging.Logger: diff --git a/tests/it/it_scargo_commands_flow.py b/tests/it/it_scargo_commands_flow.py index eefebeb0..24ad4f54 100644 --- a/tests/it/it_scargo_commands_flow.py +++ b/tests/it/it_scargo_commands_flow.py @@ -505,14 +505,12 @@ def test_cli_update_after_adding_new_profile( result.exit_code == 0 ), f"Command 'update' end with non zero exit code: {result.exit_code}" if test_state.target_id == TargetIds.esp32: - c_flags_re = ( - r"WORKAROUND_FOR_ESP32_C_FLAGS=\"(.+?) " + NEW_PROFILE_CFLAGS + r"\"" - ) + c_flags_re = r"WORKAROUND_FOR_ESP32_C_FLAGS=(.+?)" + NEW_PROFILE_CFLAGS + cxx_flags_re = ( - r"WORKAROUND_FOR_ESP32_CXX_FLAGS=\"(.+?) " - + NEW_PROFILE_CXXFLAGS - + r"\"" + r"WORKAROUND_FOR_ESP32_CXX_FLAGS=(.+?)" + NEW_PROFILE_CXXFLAGS ) + else: c_flags_re = ( r"tools\.build:cflags=\[\"(.+?) " + NEW_PROFILE_CFLAGS + r"\"\]" diff --git a/tests/ut/ut_scargo_publish.py b/tests/ut/ut_scargo_publish.py index 9ec9bd5c..ba06893b 100644 --- a/tests/ut/ut_scargo_publish.py +++ b/tests/ut/ut_scargo_publish.py @@ -1,18 +1,14 @@ import os +import subprocess from pathlib import Path from unittest.mock import patch import pytest from _pytest.logging import LogCaptureFixture from pytest_subprocess import FakeProcess +from pytest_subprocess.fake_popen import FakePopen -from scargo.commands.publish import ( - conan_add_conancenter, - conan_add_remote, - conan_add_user, - conan_clean_remote, - scargo_publish, -) +from scargo.commands.publish import conan_add_remote, conan_add_user, scargo_publish from scargo.config import Config from tests.ut.utils import get_test_project_config @@ -24,6 +20,10 @@ ENV_CONAN_PASSWORD = "env_conan_password" +def conan_remote_add_error(process: FakePopen, stderr: str) -> None: + raise subprocess.CalledProcessError(1, process.args, b"", stderr.encode()) + + @pytest.fixture def config(monkeypatch: pytest.MonkeyPatch) -> Config: test_project_config = get_test_project_config() @@ -58,7 +58,7 @@ def test_publish(config: Config, fp: FakeProcess) -> None: "conan", "export-pkg", ".", - "-if", + "-of", str(build_path), "-pr:b", "default", @@ -100,17 +100,15 @@ def test_publish(config: Config, fp: FakeProcess) -> None: scargo_publish(REPO_NAME) # ASSERT - assert fp.calls[0] == conan_clean_cmd - assert fp.calls[1] == conan_add_remote_1_cmd - assert fp.calls[2] == conan_user_cmd - assert fp.calls[3] == conan_add_remote_2_cmd - assert fp.calls[4] == conan_user_cmd - assert fp.calls[5] == conan_add_conacenter_cmd - assert fp.calls[6] == conan_source_cmd - assert fp.calls[7] == conan_export_pkg_cmd - assert fp.calls[8] == conan_test_cmd - assert fp.calls[9] == conan_upload_cmd - assert len(fp.calls) == 10 + assert fp.calls[0] == conan_add_remote_1_cmd + assert fp.calls[1] == conan_user_cmd + assert fp.calls[2] == conan_add_remote_2_cmd + assert fp.calls[3] == conan_user_cmd + assert fp.calls[4] == conan_source_cmd + assert fp.calls[5] == conan_export_pkg_cmd + assert fp.calls[6] == conan_test_cmd + assert fp.calls[7] == conan_upload_cmd + assert len(fp.calls) == 8 def test_conan_add_user(fp: FakeProcess) -> None: @@ -140,7 +138,7 @@ def test_conan_add_user(fp: FakeProcess) -> None: assert len(fp.calls) == 2 -def test_conan_add_remote_fail( +def test_conan_add_remote_already_exists( config: Config, caplog: pytest.LogCaptureFixture, fp: FakeProcess, @@ -148,39 +146,40 @@ def test_conan_add_remote_fail( # ARRANGE fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) fp.register("conan user", occurrences=2) + error = f"ERROR: Remote '{REMOTE_REPO_NAME_2}' already exists in remotes (use --force to continue)" fp.register( - ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], returncode=1 + ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], + callback=conan_remote_add_error, + callback_kwargs={"stderr": error}, ) # ACT conan_add_remote(Path("some_path"), config) # ASSERT - assert "Unable to add remote repository" in caplog.text + assert "Unable to add remote repository" not in caplog.text -def test_conan_add_conancenter_fail(caplog: LogCaptureFixture, fp: FakeProcess) -> None: - # ARRANGE - cmd = "conan remote add conancenter https://center.conan.io" - fp.register(cmd, returncode=1) - - # ACT - conan_add_conancenter() - - # ASSERT - assert "Unable to add conancenter remote repository" in caplog.text - - -def test_conan_clean_remote_fail(caplog: LogCaptureFixture, fp: FakeProcess) -> None: +def test_conan_add_remote_fail( + config: Config, + caplog: pytest.LogCaptureFixture, + fp: FakeProcess, +) -> None: # ARRANGE - cmd = "conan remote clean" - fp.register(cmd, returncode=1) + fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) + fp.register("conan user", occurrences=2) + error = "ERROR: not remote add failure" + fp.register( + ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], + callback=conan_remote_add_error, + callback_kwargs={"stderr": error}, + ) # ACT - conan_clean_remote() + conan_add_remote(Path("some_path"), config) # ASSERT - assert "Unable to clean remote repository list" in caplog.text + assert "Unable to add remote repository" in caplog.text def test_create_package_fail( @@ -211,7 +210,7 @@ def test_create_package_fail( "conan", "export-pkg", ".", - "-if", + "-of", str(build_path), "-pr:b", "default", @@ -266,11 +265,9 @@ def test_upload_package_fail( build_path = Path(f"{config.project_root}/build/Release") build_path.mkdir(parents=True, exist_ok=True) - fp.register("conan remote clean") fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL]) fp.register("conan user", occurrences=2) - fp.register("conan remote add conancenter https://center.conan.io") fp.register( [ "conan", @@ -283,7 +280,7 @@ def test_upload_package_fail( "conan", "export-pkg", ".", - "-if", + "-of", str(build_path), "-pr:b", "default", diff --git a/tests/ut/ut_scargo_test.py b/tests/ut/ut_scargo_test.py index 6ce340f3..672150de 100644 --- a/tests/ut/ut_scargo_test.py +++ b/tests/ut/ut_scargo_test.py @@ -35,25 +35,49 @@ def test_scargo_test_no_cmake_file( # type: ignore[no-any-unimported] def test_scargo_test( # type: ignore[no-any-unimported] fp: FakeProcess, fs: FakeFilesystem, mock_prepare_config: MagicMock ) -> None: - fp.register("conan install tests -if build/tests -sbuild_type=Debug") - fp.register("conan build tests -bf build/tests") - fp.register("ctest") + fp.register("conan profile list") + fp.register("conan profile detect") + fp.register("conan install tests -of build/tests -sbuild_type=Debug -b missing") + fp.register("conan build -of build/tests tests -sbuild_type=Debug -b missing") fp.register("gcovr -r ut . -f src --html=ut-coverage.html") + fp.register("ctest") os.mkdir("tests") with open("tests/CMakeLists.txt", "w"): pass scargo_test(False) + assert fp.calls[0] == [ + "conan", + "profile", + "list", + ] + assert fp.calls[1] == [ + "conan", + "profile", + "detect", + ] + assert fp.calls[2] == [ "conan", "install", Path("tests"), - "-if", + "-of", Path("build/tests"), "-sbuild_type=Debug", + "-b", + "missing", ] - assert fp.calls[1] == ["conan", "build", Path("tests"), "-bf", Path("build/tests")] - assert fp.calls[2] == ["ctest"] assert fp.calls[3] == [ + "conan", + "build", + "-of", + Path("build/tests"), + Path("tests"), + "-sbuild_type=Debug", + "-b", + "missing", + ] + assert fp.calls[4] == ["ctest"] + assert fp.calls[5] == [ "gcovr", "-r", "ut", diff --git a/tests/ut/ut_scargo_update.py b/tests/ut/ut_scargo_update.py index afd61eb1..0f736a86 100644 --- a/tests/ut/ut_scargo_update.py +++ b/tests/ut/ut_scargo_update.py @@ -54,6 +54,8 @@ def test_update_project_content_with_docker(tmp_path: Path, fp: FakeProcess) -> called_subprocess_cmd = get_docker_compose_command() called_subprocess_cmd.extend(["pull"]) fp.register(called_subprocess_cmd) + fp.register(["conan", "profile", "list"]) + fp.register(["conan", "profile", "detect"]) scargo_update(Path("scargo.toml")) for path in Path().iterdir(): assert path.name in EXPECTED_FILES_AND_DIRS @@ -70,6 +72,8 @@ def test_update_project_content_with_docker__build( cmd_pull.extend(["pull"]) fp.register(cmd_pull, returncode=1) cmd_build = get_docker_compose_command() + fp.register(["conan", "profile", "list"]) + fp.register(["conan", "profile", "detect"]) cmd_build.extend(["build"]) fp.register(cmd_build) scargo_update(Path("scargo.toml")) From 8db36a412bbce43a01b1e1a1819946b02361bf4a Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Tue, 24 Oct 2023 15:58:48 +0200 Subject: [PATCH 10/28] Update atsam doc (#383) --- docs/source/guides/scargo-atsam.rst | 12 ++++++++++-- docs/source/scargo/index.rst | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/source/guides/scargo-atsam.rst b/docs/source/guides/scargo-atsam.rst index 27e0d804..fcfbae2f 100644 --- a/docs/source/guides/scargo-atsam.rst +++ b/docs/source/guides/scargo-atsam.rst @@ -15,8 +15,16 @@ It's also possible to change the chip in *scargo.toml* file and run scargo updat Flashing -------- -Flashing the Atmel SAM series is currently not supported, but it will be implemented +Flashing the Atmel SAM series is currently supported by using openocd in the background. +Run the :doc:`scargo flash command ` to flash the board. + +This flashing procedure might not work for all boards. +If you have any problems you can `open issue on github `_ or contact Spyrosoft members via email (e.g. aak@spyro-soft.com). Debugging --------- -Debugging the Atmel SAM series is currently not supported, but it will be implemented +If you plan on debugging make sure that your board has debugger or you are connected to the board using debugger. +First build the project in Debug and then you can run :doc:`scargo debug command `: :: + + scargo build --profile Debug + scargo debug diff --git a/docs/source/scargo/index.rst b/docs/source/scargo/index.rst index 1c236f4a..4d11e6c8 100644 --- a/docs/source/scargo/index.rst +++ b/docs/source/scargo/index.rst @@ -27,7 +27,7 @@ To see all options for a particular command, append ``-h`` to the command name. doc Create project documentation docker Manage the docker environment for the project fix Fix violations reported by the command `check`. - flash Flash the target (only available for esp32 for now). + flash Flash the target. gen Manage the auto file generator new Create a new project template. publish Upload conan pkg to repo From b1c0290892742c23f124eebc0bbc40fb25db8b48 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Wed, 25 Oct 2023 10:14:32 +0200 Subject: [PATCH 11/28] Update example project (#384) --- .../.devcontainer/.gitlab-ci-custom.yml | 2 + example_project/.devcontainer/Dockerfile | 40 ++++++------------- .../.devcontainer/devcontainer.json | 38 ++++++++++++++++-- .../.devcontainer/docker-compose.yaml | 18 ++++----- example_project/.gitlab-ci.yml | 8 +++- example_project/CMakeLists.txt | 37 ++++------------- example_project/conanfile.py | 30 +++++++++----- .../config/conan/profiles/x86_Debug | 16 ++++++++ .../config/conan/profiles/x86_MinSizeRel | 16 ++++++++ .../config/conan/profiles/x86_RelWithDebInfo | 16 ++++++++ .../config/conan/profiles/x86_Release | 16 ++++++++ example_project/scargo.lock | 4 +- example_project/scargo.toml | 2 +- example_project/tests/CMakeLists.txt | 11 +++-- example_project/tests/conanfile.py | 12 ++++-- pyproject.toml | 4 ++ .../templates/conan/conanfile.py.j2 | 10 ++--- .../templates/conan/conanfiletest.j2 | 4 +- setup.cfg | 3 ++ 19 files changed, 186 insertions(+), 101 deletions(-) create mode 100644 example_project/.devcontainer/.gitlab-ci-custom.yml create mode 100644 example_project/config/conan/profiles/x86_Debug create mode 100644 example_project/config/conan/profiles/x86_MinSizeRel create mode 100644 example_project/config/conan/profiles/x86_RelWithDebInfo create mode 100644 example_project/config/conan/profiles/x86_Release diff --git a/example_project/.devcontainer/.gitlab-ci-custom.yml b/example_project/.devcontainer/.gitlab-ci-custom.yml new file mode 100644 index 00000000..6d525a8c --- /dev/null +++ b/example_project/.devcontainer/.gitlab-ci-custom.yml @@ -0,0 +1,2 @@ +# user can add his cicd code below which will be integrated with +# main cicd file and not overwritten by scargo update \ No newline at end of file diff --git a/example_project/.devcontainer/Dockerfile b/example_project/.devcontainer/Dockerfile index b2b2f84a..22e092bb 100644 --- a/example_project/.devcontainer/Dockerfile +++ b/example_project/.devcontainer/Dockerfile @@ -9,30 +9,22 @@ FROM ${DOCKER_IMAGE_ROOT} as base ENV DEBIAN_FRONTEND noninteractive RUN apt update --fix-missing && \ - apt -y install python3.8 python3.8-venv python3-pip \ - binutils git wget \ - scons build-essential pkg-config \ - unzip graphviz nano vim && \ + apt -y install --no-install-recommends python3 python3-pip \ + git build-essential pkg-config graphviz && \ apt -y install sudo && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ - update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 + rm -rf /var/lib/apt/lists/* && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 FROM base AS cpp -RUN apt -y install cppcheck bzr lib32z1 \ - clang clang-format clang-tidy valgrind \ - gcovr doxygen curl libcurl4-openssl-dev \ - libcmocka0 libcmocka-dev plantuml +RUN apt update --fix-missing && apt -y --no-install-recommends install cppcheck lib32z1 \ + make cmake clang clang-format clang-tidy gcovr doxygen libcmocka0 libcmocka-dev gdb screen wget && \ + rm -rf /var/lib/apt/lists/* FROM cpp AS x86 ARG SSH_PORT ENV DEBIAN_FRONTEND noninteractive - -# configure wxWidget -RUN apt update --fix-missing && \ - apt install -y adwaita-icon-theme-full libwxgtk3.0-gtk3-dev dbus-x11 \ - bzr lib32z1 libncurses5:amd64 usbutils xterm minicom WORKDIR /opt FROM x86 AS custom_example_project @@ -47,16 +39,17 @@ ARG USER_PASSWORD=user ARG UID_NUMBER=1000 ARG GID_NUMBER=1000 ARG SSH_PORT=2000 +ARG CONAN_LOGIN_USERNAME="" +ARG CONAN_PASSWORD="" -ARG CONAN_LOGIN_USERNAME -ARG CONAN_PASSWORD ENV CONAN_LOGIN_USERNAME=${CONAN_LOGIN_USERNAME} ENV CONAN_PASSWORD=${CONAN_PASSWORD} WORKDIR /opt # configure ssh -RUN apt install -y openssh-server ssh-askpass && apt install -y rsync grsync && \ +RUN apt update --fix-missing && apt install -y --no-install-recommends openssh-server && \ + rm -rf /var/lib/apt/lists/* && \ ssh-keygen -A && mkdir -p /run/sshd && \ echo "Port $SSH_PORT" >> /etc/ssh/sshd_config @@ -69,16 +62,7 @@ RUN printf "\n\nADDING USER $USER_NAME TO SUDOERS - DEV ENV ONLY!\n\n" >&2 && \ echo "$USER_NAME:$USER_PASSWORD" | chpasswd && \ echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -# install scargo from spyro rep (temporarily before pip repo will be available) -COPY requirements.txt requirements.txt -RUN pip3 install -r requirements.txt && \ - rm requirements.txt - -ENV SCARGO_VERSION="release_1.0.0" -ENV SCARGO_PATH="/opt/scargo" -ENV SCARGO_REPO_URL="https://github.com/Spyro-Soft/scargo.git" -RUN git clone --single-branch --branch ${SCARGO_VERSION} ${SCARGO_REPO_URL} ${SCARGO_PATH} && \ - cd scargo && pip install . +RUN pip3 install scargo==1.8.0 RUN usermod -a -G dialout ${USER_NAME} USER ${USER_NAME} diff --git a/example_project/.devcontainer/devcontainer.json b/example_project/.devcontainer/devcontainer.json index eff28a26..4dd60c2b 100644 --- a/example_project/.devcontainer/devcontainer.json +++ b/example_project/.devcontainer/devcontainer.json @@ -1,6 +1,36 @@ { - "name": "example_project_dev", - "dockerComposeFile": "docker-compose.yaml", - "service": "example_project_dev", - "workspaceFolder": "/workspace" + "name": "example_project_dev", + "dockerComposeFile": "docker-compose.yaml", + "service": "example_project_dev", + "workspaceFolder": "/workspace", + "customizations": { + "vscode": { + "settings": { + "files.insertFinalNewline": true, + "editor.formatOnSave": true, + "C_Cpp.default.includePath": [ + "${workspaceFolder}/**" + ], + "cSpell.words": [ + "spyro", + "spyrosoft", + "cppcheck", + "lvgl", + "gpio", + "scargo", + "devcontainer", + "usbip", + "cyclomatic", + "microcontroller" + ] + }, + "extensions": [ + "xaver.clang-format", + "ms-vscode.cpptools-extension-pack", + "streetsidesoftware.code-spell-checker", + "ciarant.vscode-structurizr", + "coolchyni.beyond-debug" + ] + } + } } \ No newline at end of file diff --git a/example_project/.devcontainer/docker-compose.yaml b/example_project/.devcontainer/docker-compose.yaml index 3c9cb504..79496f01 100644 --- a/example_project/.devcontainer/docker-compose.yaml +++ b/example_project/.devcontainer/docker-compose.yaml @@ -4,23 +4,21 @@ ### version: '3.9' -x-project_config: &project_config - USER_NAME: ${USER_NAME} - USER_PASSWORD: ${USER_PASSWORD} - GID_NUMBER: ${GID_NUMBER} - UID_NUMBER: ${UID_NUMBER} - DOCKER_IMAGE_ROOT: ${DOCKER_IMAGE_ROOT} - CONAN_LOGIN_USERNAME: ${CONAN_LOGIN_USERNAME} - CONAN_PASSWORD: ${CONAN_PASSWORD} - services: example_project_dev: image: example_project-dev:1.0 platform: linux/amd64 build: context: . + # in case of missing .env file (eg. CI/CD) system will use default values args: - <<: *project_config + USER_NAME: ${USER_NAME:-user} + USER_PASSWORD: ${USER_PASSWORD:-user} + GID_NUMBER: ${GID_NUMBER:-1000} + UID_NUMBER: ${UID_NUMBER:-1000} + DOCKER_IMAGE_ROOT: ${DOCKER_IMAGE_ROOT:-ubuntu:20.04} + CONAN_LOGIN_USERNAME: ${CONAN_LOGIN_USERNAME:-""} + CONAN_PASSWORD: ${CONAN_PASSWORD:-""} privileged: true volumes: - ..:/workspace diff --git a/example_project/.gitlab-ci.yml b/example_project/.gitlab-ci.yml index 170717dc..08c50ac3 100644 --- a/example_project/.gitlab-ci.yml +++ b/example_project/.gitlab-ci.yml @@ -36,7 +36,9 @@ debug: stage: build extends: .merge_request script: - - scargo build + - CONAN_LOGIN_USERNAME=ci_user + - CONAN_PASSWORD=${CI_JOB_TOKEN} + - scargo build --profile Debug artifacts: paths: - build/Debug/bin @@ -49,10 +51,12 @@ tests: extends: .merge_request allow_failure: true script: + - CONAN_LOGIN_USERNAME=ci_user + - CONAN_PASSWORD=${CI_JOB_TOKEN} - scargo test artifacts: paths: - - build/doc/ut_coverage_html + - build/tests/ut-coverage.html #______________________________________________________________________________________________CLANG_FORMAT clang format: diff --git a/example_project/CMakeLists.txt b/example_project/CMakeLists.txt index 158123ff..e6766445 100644 --- a/example_project/CMakeLists.txt +++ b/example_project/CMakeLists.txt @@ -1,9 +1,16 @@ +# # +# Copyright (C) 2023 SpyroSoft Solutions S.A. All rights reserved. +# # + # # DO NOT EDIT THIS FILE! # This file is generated by `scargo update`. # -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.22) +list(APPEND CMAKE_PREFIX_PATH "${CMAKE_BINARY_DIR}/generators/") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) project( example_project VERSION 0.1.0 @@ -12,32 +19,4 @@ project( example_project LANGUAGES C CXX ASM ) -if(EXISTS ${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) - include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) - conan_basic_setup() -endif() -set(CMAKE_C_COMPILER "gcc") -set(CMAKE_CXX_COMPILER "g++") - -set(CMAKE_CXX_STANDARD 17) - -set(CMAKE_C_FLAGS "-Wall -Wextra") -set(CMAKE_CXX_FLAGS "-Wall -Wextra") -# Debug -set(CMAKE_C_FLAGS_DEBUG "-g") -set(CMAKE_CXX_FLAGS_DEBUG "-g") - -# Release -set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") -set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") - -# RelWithDebInfo -set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG") -set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG") - -# MinSizeRel -set(CMAKE_C_FLAGS_MINSIZEREL "-Os -DNDEBUG") -set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG") - - add_subdirectory(src) diff --git a/example_project/conanfile.py b/example_project/conanfile.py index 9592acf2..49d716cf 100644 --- a/example_project/conanfile.py +++ b/example_project/conanfile.py @@ -1,25 +1,35 @@ -from conan import CMake, ConanFile, tools # type: ignore[import] +# +# DO NOT EDIT THIS FILE! +# This file is generated by `scargo update`. +# + +from conan import ConanFile +from conan.tools.cmake import CMake, cmake_layout +from conan.tools.files import copy, get +from conan.tools.scm import Git class Example_projectConan(ConanFile): # type: ignore[misc, no-any-unimported] name = "example_project" version = "0.1.0" - settings = "os", "compiler", "build_type", "arch" + license = "MIT" description = "Project description." + settings = "os", "compiler", "build_type", "arch" url = "www.hello-world.com" - generators = "cmake_find_package", "cmake" + generators = "CMakeToolchain", "CMakeDeps" + exports_sources = ["src/*", "CMakeLists.txt", "include/*", "config/*"] - def package(self) -> None: - self.copy("*", src="build/Debug/bin/", dst="bin", keep_path=False) - self.copy("*", src="build/Debug/lib/", dst="lib", keep_path=False) + def source(self) -> None: + pass - def package_info(self) -> None: - self.cpp_info.libs = tools.collect_libs(self) + def layout(self) -> None: + cmake_layout(self) def build(self) -> None: cmake = CMake(self) cmake.configure() cmake.build() - def source(self) -> None: - pass + def package(self) -> None: + copy("*.bin") + copy("*.elf") diff --git a/example_project/config/conan/profiles/x86_Debug b/example_project/config/conan/profiles/x86_Debug new file mode 100644 index 00000000..54fcd668 --- /dev/null +++ b/example_project/config/conan/profiles/x86_Debug @@ -0,0 +1,16 @@ +[buildenv] +CC=gcc +CXX=g++ + +[settings] +compiler=gcc +compiler.version=9 +compiler.libcxx=libstdc++ +compiler.cppstd=17 +os=Linux +arch=x86_64 +build_type=Debug + +[conf] +tools.build:cflags=["-Wall -Wextra -g"] +tools.build:cxxflags=["-Wall -Wextra -g"] diff --git a/example_project/config/conan/profiles/x86_MinSizeRel b/example_project/config/conan/profiles/x86_MinSizeRel new file mode 100644 index 00000000..32645b27 --- /dev/null +++ b/example_project/config/conan/profiles/x86_MinSizeRel @@ -0,0 +1,16 @@ +[buildenv] +CC=gcc +CXX=g++ + +[settings] +compiler=gcc +compiler.version=9 +compiler.libcxx=libstdc++ +compiler.cppstd=17 +os=Linux +arch=x86_64 +build_type=MinSizeRel + +[conf] +tools.build:cflags=["-Wall -Wextra -Os -DNDEBUG"] +tools.build:cxxflags=["-Wall -Wextra -Os -DNDEBUG"] diff --git a/example_project/config/conan/profiles/x86_RelWithDebInfo b/example_project/config/conan/profiles/x86_RelWithDebInfo new file mode 100644 index 00000000..f7147262 --- /dev/null +++ b/example_project/config/conan/profiles/x86_RelWithDebInfo @@ -0,0 +1,16 @@ +[buildenv] +CC=gcc +CXX=g++ + +[settings] +compiler=gcc +compiler.version=9 +compiler.libcxx=libstdc++ +compiler.cppstd=17 +os=Linux +arch=x86_64 +build_type=RelWithDebInfo + +[conf] +tools.build:cflags=["-Wall -Wextra -O2 -g -DNDEBUG"] +tools.build:cxxflags=["-Wall -Wextra -O2 -g -DNDEBUG"] diff --git a/example_project/config/conan/profiles/x86_Release b/example_project/config/conan/profiles/x86_Release new file mode 100644 index 00000000..5bcbdf89 --- /dev/null +++ b/example_project/config/conan/profiles/x86_Release @@ -0,0 +1,16 @@ +[buildenv] +CC=gcc +CXX=g++ + +[settings] +compiler=gcc +compiler.version=9 +compiler.libcxx=libstdc++ +compiler.cppstd=17 +os=Linux +arch=x86_64 +build_type=Release + +[conf] +tools.build:cflags=["-Wall -Wextra -O3 -DNDEBUG"] +tools.build:cxxflags=["-Wall -Wextra -O3 -DNDEBUG"] diff --git a/example_project/scargo.lock b/example_project/scargo.lock index 1d83646c..51a5ef26 100644 --- a/example_project/scargo.lock +++ b/example_project/scargo.lock @@ -72,7 +72,7 @@ gcov-executable = "" # Empty string -> use default gcov executable general = [] build = [] tool = [] -test = ["gtest/cci.20210126"] +test = ["gtest/1.13.0"] [conan.repo] # conancenter = "https://center.conan.io" @@ -82,4 +82,4 @@ test = ["gtest/cci.20210126"] console-log-level = "INFO" file-log-level = "WARNING" update-exclude = [] -version = "1.0.0" +version = "1.8.0" diff --git a/example_project/scargo.toml b/example_project/scargo.toml index 0f1f05d1..d431e5af 100644 --- a/example_project/scargo.toml +++ b/example_project/scargo.toml @@ -72,7 +72,7 @@ gcov-executable = "" # Empty string -> use default gcov executable general = [] build = [] tool = [] -test = ["gtest/cci.20210126"] +test = ["gtest/1.13.0"] [conan.repo] # conancenter = "https://center.conan.io" diff --git a/example_project/tests/CMakeLists.txt b/example_project/tests/CMakeLists.txt index cf8904bf..e4d9df49 100644 --- a/example_project/tests/CMakeLists.txt +++ b/example_project/tests/CMakeLists.txt @@ -5,17 +5,20 @@ cmake_minimum_required(VERSION 3.22) set(CMAKE_C_COMPILER "gcc") + set(CMAKE_CXX_COMPILER "g++") +set(CMAKE_CXX_STANDARD 17) + set(CMAKE_C_FLAGS "-Wall -Wextra -Og --coverage -fkeep-inline-functions -fkeep-static-consts") + set(CMAKE_CXX_FLAGS "-Wall -Wextra -Og --coverage -fkeep-inline-functions -fkeep-static-consts") + project(tests LANGUAGES C CXX ASM) -if(EXISTS ${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) - include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) - conan_basic_setup() -endif() +find_package(GTest REQUIRED) +link_libraries(GTest::GTest GTest::Main) enable_testing() diff --git a/example_project/tests/conanfile.py b/example_project/tests/conanfile.py index 5f404d36..c6fe82c7 100644 --- a/example_project/tests/conanfile.py +++ b/example_project/tests/conanfile.py @@ -1,4 +1,10 @@ -from conans import CMake, ConanFile # type: ignore[import] +# +# DO NOT EDIT THIS FILE! +# This file is generated by `scargo update`. +# + +from conan import ConanFile +from conan.tools.cmake import CMake class Example_projectTestConan(ConanFile): # type: ignore[misc, no-any-unimported] @@ -7,7 +13,7 @@ class Example_projectTestConan(ConanFile): # type: ignore[misc, no-any-unimport settings = "os", "compiler", "build_type", "arch" description = "Tests for example_project" url = "www.hello-world.com" - generators = "cmake_find_package", "cmake" + generators = "CMakeToolchain", "CMakeDeps" def build(self) -> None: cmake = CMake(self) @@ -19,4 +25,4 @@ def test(self) -> None: self.cmake.build(target=target_test) def requirements(self) -> None: - self.requires("gtest/cci.20210126") + self.requires("gtest/1.13.0") diff --git a/pyproject.toml b/pyproject.toml index 24c57a35..5ceb8787 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,3 +128,7 @@ ignore_missing_imports = true [[tool.mypy.overrides]] module = "pyfakefs.*" ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "conan.*" +ignore_missing_imports = true diff --git a/scargo/file_generators/templates/conan/conanfile.py.j2 b/scargo/file_generators/templates/conan/conanfile.py.j2 index 68f1c6e9..617a53e2 100644 --- a/scargo/file_generators/templates/conan/conanfile.py.j2 +++ b/scargo/file_generators/templates/conan/conanfile.py.j2 @@ -4,9 +4,9 @@ # from conan import ConanFile -from conan.tools.scm import Git -from conan.tools.files import copy, get from conan.tools.cmake import CMake, cmake_layout +from conan.tools.files import copy, get +from conan.tools.scm import Git {% if config.project.target.family == "stm32" %} import os @@ -21,6 +21,7 @@ class {{ config.project.name|capitalize|replace("-", "") }}Conan(ConanFile): # url = "{{ config.project.homepage_url }}" generators = "CMakeToolchain", "CMakeDeps" exports_sources = ["{{ config.source_dir_path.relative_to(config.project_root) }}/*", "CMakeLists.txt", "include/*", "config/*"] + {% if config.dependencies.build or config.dependencies.tool %} def build_requirements(self) -> None: {% for dep in config.dependencies.build %} @@ -29,15 +30,15 @@ class {{ config.project.name|capitalize|replace("-", "") }}Conan(ConanFile): # {% for tool_dep in config.dependencies.tool %} self.tool_requires("{{ tool_dep }}") {% endfor %} - {% endif %} + {% endif %} {% if config.dependencies.general %} def requirements(self) -> None: {% for dep in config.dependencies.general %} self.requires("{{ dep }}") {% endfor %} - {% endif %} + {% endif %} def source(self) -> None: {% if config.project.target.family == "stm32" %} target_dir = "third-party/stm32-cmake" @@ -90,7 +91,6 @@ class {{ config.project.name|capitalize|replace("-", "") }}Conan(ConanFile): # copy("*.bin") copy("*.elf") {% endif %} - {% if config.project.lib_name %} def package_info(self) -> None: self.cpp_info.libs = ["{{ config.project.name }}"] diff --git a/scargo/file_generators/templates/conan/conanfiletest.j2 b/scargo/file_generators/templates/conan/conanfiletest.j2 index 550ac068..04efe087 100644 --- a/scargo/file_generators/templates/conan/conanfiletest.j2 +++ b/scargo/file_generators/templates/conan/conanfiletest.j2 @@ -4,7 +4,7 @@ # from conan import ConanFile -from conan.tools.cmake import CMake # type: ignore[import] +from conan.tools.cmake import CMake class {{ config.project.name|capitalize|replace("-", "") }}TestConan(ConanFile): # type: ignore[misc, no-any-unimported] @@ -33,7 +33,6 @@ class {{ config.project.name|capitalize|replace("-", "") }}TestConan(ConanFile): self.requires("{{ dep }}") {% endfor %} {% endif %} - {% if config.dependencies.build or config.dependencies.tool %} def build_requirements(self) -> None: {% for dep in config.dependencies.build %} @@ -43,4 +42,3 @@ class {{ config.project.name|capitalize|replace("-", "") }}TestConan(ConanFile): self.tool_requires("{{ tool_dep }}") {% endfor %} {% endif %} - diff --git a/setup.cfg b/setup.cfg index 3d84d994..a4c9b2cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,5 +15,8 @@ per-file-ignores = # allow definition from star imports in docs config docs/conf_common.py: F405, + # potential unused imports + conanfile.py:F401 + # Compatibility with Black max-line-length = 120 From c2281e2e23e079856d7a65422560b46b57faa7e7 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Wed, 25 Oct 2023 15:24:39 +0200 Subject: [PATCH 12/28] Improve pylint (#385) --- .github/workflows/workflow-common.yml | 2 +- .pre-commit-config.yaml | 4 ++ ci/requirements.txt | 8 +--- clean.py | 9 ++-- common_dev/scripts/copyrights.py | 60 ++++++++++++++------------- common_dev/scripts/pylintchecker.py | 18 +++----- common_dev/scripts/test_report.py | 2 +- common_dev/scripts/todo_check.py | 4 +- docs/source/conf.py | 6 +-- pyproject.toml | 41 +++++++++++++++--- run.py | 11 +++-- scargo/target_helpers/__init__.py | 0 scargo/target_helpers/atsam_helper.py | 4 +- 13 files changed, 96 insertions(+), 73 deletions(-) create mode 100644 scargo/target_helpers/__init__.py diff --git a/.github/workflows/workflow-common.yml b/.github/workflows/workflow-common.yml index d5258320..6d7f637f 100644 --- a/.github/workflows/workflow-common.yml +++ b/.github/workflows/workflow-common.yml @@ -61,7 +61,7 @@ jobs: continue-on-error: true run: ./common_dev/scripts/cyclomatic.py - name: Run pylint - run: ./common_dev/scripts/pylintchecker.py -c scargo/ -s 9.95 --exclude=tests/ + run: pylint scargo run.py clean.py - name: Check typing with mypy run: mypy --explicit-package-bases scargo tests common_dev run.py clean.py unit_tests: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c4fb58f3..da76dff4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,10 @@ repos: - id: flake8 additional_dependencies: - flake8-absolute-import + - repo: https://github.com/pylint-dev/pylint + rev: v3.0.2 + hooks: + - id: pylint - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.0.1 hooks: diff --git a/ci/requirements.txt b/ci/requirements.txt index 532292be..3d39f0ae 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.13 # via sphinx altgraph==0.17.3 # via pyinstaller -astroid==2.15.0 +astroid==3.0.1 # via pylint attrs==22.2.0 # via pytest @@ -118,8 +118,6 @@ jinja2==3.1.2 # sphinx kiwisolver==1.4.4 # via matplotlib -lazy-object-proxy==1.9.0 - # via astroid libclang==16.0.0 # via scargo (pyproject.toml) lizard==1.17.10 @@ -194,7 +192,7 @@ pyinstaller==5.9.0 # via scargo (pyproject.toml) pyinstaller-hooks-contrib==2023.0 # via pyinstaller -pylint==2.17.0 +pylint==3.0.2 # via scargo (pyproject.toml) pynacl==1.5.0 # via paramiko @@ -318,8 +316,6 @@ virtualenv==20.21.0 # via pre-commit websocket-client==0.59.0 # via docker -wrapt==1.15.0 - # via astroid # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/clean.py b/clean.py index 45b3e4f3..3972b1d6 100755 --- a/clean.py +++ b/clean.py @@ -51,10 +51,9 @@ def list_files(path: str) -> Optional[List[str]]: if path: if os.path.isdir(path): return [path] - else: - raise argparse.ArgumentTypeError( - f"readable_dir:schemas/{path} is not a valid path" - ) + raise argparse.ArgumentTypeError( + f"readable_dir:schemas/{path} is not a valid path" + ) return None @@ -66,7 +65,7 @@ def clean(output_file_names: Sequence[str]) -> None: print("Pyclean fail") for item in output_file_names: - subprocess.run(["rm", "-rf", item]) + subprocess.run(["rm", "-rf", item], check=True) def main() -> None: diff --git a/common_dev/scripts/copyrights.py b/common_dev/scripts/copyrights.py index db6511e3..8dcce1a6 100755 --- a/common_dev/scripts/copyrights.py +++ b/common_dev/scripts/copyrights.py @@ -101,7 +101,9 @@ class FilesToCheck: lang_data: List[str] = [] lang_other: List[str] = [] - def add_file_to_check(self, file: str, repo_path: str) -> int: + def add_file_to_check( # pylint: disable=too-many-return-statements + self, file: str, repo_path: str + ) -> int: if os.path.islink(file): # ignore symbolic links (they are likely to be found as file again) return 0 @@ -147,7 +149,7 @@ def add_file_to_check(self, file: str, repo_path: str) -> int: def get_all_files(repo_path: str, exclude_dir: Sequence[str]) -> FilesToCheck: files_to_check = FilesToCheck() files = [] - for root, d_names, f_names in os.walk(repo_path): + for root, _, f_names in os.walk(repo_path): for f in f_names: fname = os.path.join(root, f) if exclude_dir: @@ -166,14 +168,15 @@ def check_correct_copyright_embedded( enable_verbose_print: bool = False, ) -> Optional[int]: if file.endswith(file_extensions): - for line in open(file, encoding="utf-8"): - copyright_regex = COPYRIGHT_CHECK_REGEX - copyright_check_with_doxygen = COPYRIGHT_CHECK_WITH_DOXYGEN_REGEX - if require_doxygen_tag: - copyright_regex = copyright_check_with_doxygen - if re.search(copyright_regex, line) is not None: - # matching copyright found - return 0 + with open(file, encoding="utf-8") as f: + for line in f.readlines(): + copyright_regex = COPYRIGHT_CHECK_REGEX + copyright_check_with_doxygen = COPYRIGHT_CHECK_WITH_DOXYGEN_REGEX + if require_doxygen_tag: + copyright_regex = copyright_check_with_doxygen + if re.search(copyright_regex, line) is not None: + # matching copyright found + return 0 if enable_verbose_print: print(PRINT_PREFIX + "No matching copyright found in file: " + file) # no matching copyright found @@ -214,28 +217,27 @@ def check_copyrights(files_to_check: FilesToCheck) -> int: ): fail_count += 1 - if fail_count > 0: - return 1 - else: - return 0 + return fail_count def is_any_copyright_embedded(file: str) -> bool: - for line in open(file): - if re.search(COPYRIGHT_ANY_CHECK_REGEX, line) is not None: - # some copyright found - return True - # no copyright found - return False + with open(file, encoding="utf-8") as f: + for line in f.readlines(): + if re.search(COPYRIGHT_ANY_CHECK_REGEX, line) is not None: + # some copyright found + return True + # no copyright found + return False def is_correct_copyright_with_any_year_embedded(file: str) -> bool: - for line in open(file): - if re.search(COPYRIGHT_CHECK_REGEX, line) is not None: - # some copyright found - return True - # no copyright found - return False + with open(file, encoding="utf-8") as f: + for line in f.readlines(): + if re.search(COPYRIGHT_CHECK_REGEX, line) is not None: + # some copyright found + return True + # no copyright found + return False class FixStats: @@ -271,10 +273,10 @@ def fix_copyright_common( if f.isfirstline(): if not check_shebang or re.match(SHEBANG_REGEX, line) is None: # start with copyright if no shebang matched - print(copyright_string + line, end=""), + print(copyright_string + line, end="") else: # keep shebang first before copyright - print(line + copyright_string, end=""), + print(line + copyright_string, end="") else: # rewrite file print(line, end="") @@ -407,7 +409,7 @@ def fix_copyrights(files_to_check: FilesToCheck) -> int: def main() -> None: - (args, unknown_args) = option_parser_init() + (args, _) = option_parser_init() files_to_check = get_all_files(args.workdir, args.exclude) diff --git a/common_dev/scripts/pylintchecker.py b/common_dev/scripts/pylintchecker.py index bd62fcbd..ac3125b9 100755 --- a/common_dev/scripts/pylintchecker.py +++ b/common_dev/scripts/pylintchecker.py @@ -6,7 +6,7 @@ import sys from typing import Sequence -from pylint.lint import Run # type: ignore[import] +from pylint.lint import Run def run_pylint_on_specific_directory( @@ -22,25 +22,17 @@ def run_pylint_on_specific_directory( "y", "-f", "colorized", - "--disable=C0103", # snake_case naming style, disallows names like "e" or "fs" - "--disable=C0114,C0115,C0116", # disable missing docstring rules - "--disable=R0902", # too many instance attributes - "--disable=R0903", # too few public methods - "--disable=R0913", # too many arguments - "--disable=W1203", # lazy % formatting in logging functions - "--extension-pkg-whitelist=pydantic", # ignore "No name 'BaseModel' in module 'pydantic'" ] args.extend(ignore_pattern) - # `exit` is deprecated, use `do_exit` instead - results = Run([directory] + args, do_exit=False) + results = Run([directory] + args, exit=False) result = results.linter.stats.global_note if float(result) >= score: print("PASSED!\n") return 0 - else: - print("FAILED \n") - return 1 + + print("FAILED \n") + return 1 def get_cmdline_arguments() -> argparse.Namespace: diff --git a/common_dev/scripts/test_report.py b/common_dev/scripts/test_report.py index c48d7dfc..c99f56f2 100755 --- a/common_dev/scripts/test_report.py +++ b/common_dev/scripts/test_report.py @@ -99,7 +99,7 @@ def add_test_pie_chart(ax: plt.Axes, test_data: TestData) -> None: # type: igno ax.set_title("Test Case Status") marks = [test_data.passed, test_data.failures, test_data.errors, test_data.skipped] colors = ["green", "red", "yellow", "grey"] - patches, texts = ax.pie(marks, colors=colors) + patches, _ = ax.pie(marks, colors=colors) white_circle = plt.Circle((0, 0), 0.7, color="white") ax.add_artist(white_circle) labels = ["passed", "failed", "errors", "skipped"] diff --git a/common_dev/scripts/todo_check.py b/common_dev/scripts/todo_check.py index c7f31341..a7af7739 100755 --- a/common_dev/scripts/todo_check.py +++ b/common_dev/scripts/todo_check.py @@ -58,7 +58,7 @@ def search_multiple_strings_in_file( def process(path: str, exclude_dir: Sequence[str]) -> int: todo_count = 0 - for root, d_names, f_names in os.walk(path): + for root, _, f_names in os.walk(path): for f in f_names: fname = os.path.join(root, f) if exclude_dir: @@ -74,7 +74,7 @@ def process(path: str, exclude_dir: Sequence[str]) -> int: def main() -> None: - (args, unknown_args) = option_parser_init() + (args, _) = option_parser_init() todo_count = 0 for workdir in args.workdirs: diff --git a/docs/source/conf.py b/docs/source/conf.py index d2cf0539..5c85fae2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,8 +23,8 @@ sys.path.extend(src_paths) # -- Project information ----------------------------------------------------- -project = "scargo" -copyright = "2022, Spyrosoft Solution S.A." +project = "scargo" # pylint: disable=redefined-builtin +copyright = "2022, Spyrosoft Solution S.A." # pylint: disable=redefined-builtin author = "Spyrosoft Solution S.A." # The full version, including alpha/beta/rc tags @@ -50,7 +50,7 @@ "sphinxcontrib.plantuml", ] autosummary_generate = True # Turn on sphinx.ext.autosummary -inheritance_graph_attrs = dict(rankdir="TB", size='""') # TB=Top to bottom view +inheritance_graph_attrs = {"rankdir": "TB", "size": '""'} # TB=Top to bottom view graphviz_output_format = "svg" plantuml_output_format = "svg" diff --git a/pyproject.toml b/pyproject.toml index 5ceb8787..055a45c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,12 +5,10 @@ build-backend = "flit_core.buildapi" [project] name = "scargo" description = "C/C++ package and software development life cycle manager inspired by RUST cargo idea." -authors = [ - {name = "Spyrosoft Solutions S.A.", email = "aak@spyro-soft.com"} -] +authors = [{ name = "Spyrosoft Solutions S.A.", email = "aak@spyro-soft.com" }] readme = "README.md" requires-python = ">=3.8" -license = {file = "LICENSE"} +license = { file = "LICENSE" } classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -62,7 +60,7 @@ dev = [ "pyelftools", "pyfakefs", "PyInstaller", - "pylint", + "pylint==3.0.2", "pytest-cov", "pytest-mock", "pytest-subprocess", @@ -107,6 +105,39 @@ exclude = [ [tool.isort] profile = "black" +[tool.pylint] +ignore-paths = ["example_project", "tests"] +max-line-length = 120 +output-format = "colorized" +ignored-modules = [ + "clang.cindex", + "clang", + "coloredlogs", + "docker", + "jinja2", + "matplotlib.pyplot", + "pydantic", + "toml", + "typer", + "yaml", +] + +[tool.pylint.messages_control] +disable = [ + "C0103", # snake_case naming style, disallows names like "e" or "fs" + "C0114", + "C0115", + "C0116", # disable missing docstring rules + "R0801", # Simmilar lines + "R0902", # too many instance attributes + "R0903", # too few public methods + "R0913", # too many arguments + "W1203", # lazy % formatting in logging functions +] + +[tool.pylint.extensions] +extension-pkg-whitelist = ["pydantic"] + [tool.mypy] strict = true disallow_any_unimported = true diff --git a/run.py b/run.py index bf82ffb2..2b77511e 100755 --- a/run.py +++ b/run.py @@ -252,11 +252,10 @@ def run_all_code_checkers() -> bool: def run_pylint() -> None: # can add "tests", optionally command = [ - "./common_dev/scripts/pylintchecker.py", - "-c", - "scargo/", - "-s", - "9.95", + "pylint", + "scargo", + "run.py", + "clean.py", ] subprocess.check_call(command) @@ -316,7 +315,7 @@ def run_mypy() -> None: subprocess.check_call(mypy_command) -def main() -> None: +def main() -> None: # pylint: disable=R0912 args = get_cmdline_arguments() if not len(sys.argv) > 1: args.run_all = True diff --git a/scargo/target_helpers/__init__.py b/scargo/target_helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scargo/target_helpers/atsam_helper.py b/scargo/target_helpers/atsam_helper.py index 95f1ee3a..9f8e212c 100644 --- a/scargo/target_helpers/atsam_helper.py +++ b/scargo/target_helpers/atsam_helper.py @@ -45,7 +45,7 @@ def get_atsam_cpu(chip_label: str) -> Optional[str]: def get_openocd_script_name(chip_label: str) -> Optional[str]: if chip_label.startswith("atsamd"): return "at91samdXX.cfg" - elif chip_label.startswith("atsaml1"): + if chip_label.startswith("atsaml1"): return "atsaml1x.cfg" # Not sure about this return None @@ -55,7 +55,7 @@ def get_openocd_flash_driver_name(chip_label: str) -> Optional[str]: # This should probably be done differently if chip_label.startswith("atsamd"): return "at91samd" - elif chip_label.startswith("atsaml1"): + if chip_label.startswith("atsaml1"): return "at91samd" return None From 0097acacc2a20e890440dadffd3da9e5dcf0e0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Fri, 27 Oct 2023 09:39:13 +0200 Subject: [PATCH 13/28] Changes implemented to issue_360 --- scargo/commands/build.py | 2 +- scargo/commands/publish.py | 76 +------------------- scargo/commands/test.py | 4 ++ scargo/commands/update.py | 66 +++++++++++++++++ tests/ut/ut_scargo_publish.py | 105 +-------------------------- tests/ut/ut_scargo_test.py | 43 +++++++++-- tests/ut/ut_scargo_update.py | 132 +++++++++++++++++++++++++++++++++- 7 files changed, 241 insertions(+), 187 deletions(-) diff --git a/scargo/commands/build.py b/scargo/commands/build.py index 3e2f23d5..351182c3 100644 --- a/scargo/commands/build.py +++ b/scargo/commands/build.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -from scargo.commands.publish import conan_add_remote, conan_source +from scargo.commands.update import conan_add_remote, conan_source from scargo.config_utils import prepare_config from scargo.file_generators.conan_gen import conan_add_default_profile_if_missing from scargo.logger import get_logger diff --git a/scargo/commands/publish.py b/scargo/commands/publish.py index 841cae38..1c6ed180 100644 --- a/scargo/commands/publish.py +++ b/scargo/commands/publish.py @@ -4,12 +4,11 @@ """Publish conan package into repository""" -import os import subprocess import sys from pathlib import Path -from scargo.config import Config +from scargo.commands.update import conan_add_remote, conan_source from scargo.config_utils import prepare_config from scargo.logger import get_logger @@ -95,76 +94,3 @@ def scargo_publish(repo: str, profile: str = "Release") -> None: except subprocess.CalledProcessError: logger.error("Unable to publish package") sys.exit(1) - - -def conan_add_remote(project_path: Path, config: Config) -> None: - """ - Add conan remote repository - - :param Path project_path: path to project - :param Config config: - :return: None - """ - conan_repo = config.conan.repo - for repo_name, repo_url in conan_repo.items(): - try: - subprocess.run( - ["conan", "remote", "add", repo_name, repo_url], - cwd=project_path, - check=True, - stderr=subprocess.PIPE, - ) - except subprocess.CalledProcessError as e: - if b"already exists in remotes" not in e.stderr: - logger.error(e.stderr.decode().strip()) - logger.error("Unable to add remote repository") - conan_add_user(repo_name) - - -def conan_clean_remote() -> None: - """ - Clean all remote repositories - - :return: None - """ - try: - subprocess.check_call("conan remote clean", shell=True) - except subprocess.CalledProcessError: - logger.error("Unable to clean remote repository list") - - -def conan_add_user(remote: str) -> None: - """ - Add conan user - - :param str remote: name of remote repository - :return: None - """ - conan_user = subprocess.run( - "conan user", capture_output=True, shell=True, check=False - ).stdout.decode("utf-8") - - env_conan_user = os.environ.get("CONAN_LOGIN_USERNAME", "") - env_conan_passwd = os.environ.get("CONAN_PASSWORD", "") - - if env_conan_user not in conan_user: - try: - subprocess.check_call( - ["conan", "user", "-p", env_conan_passwd, "-r", remote, env_conan_user], - ) - except subprocess.CalledProcessError: - logger.error("Unable to add user") - - -def conan_source(project_dir: Path) -> None: - try: - subprocess.check_call( - [ - "conan", - "source", - ".", - ], - cwd=project_dir, - ) - except subprocess.CalledProcessError: - logger.error("Unable to source") diff --git a/scargo/commands/test.py b/scargo/commands/test.py index b2caf1e4..3017caf6 100644 --- a/scargo/commands/test.py +++ b/scargo/commands/test.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import List, Union +from scargo.commands.update import conan_add_remote, conan_source from scargo.config import Config from scargo.config_utils import prepare_config from scargo.file_generators.conan_gen import conan_add_default_profile_if_missing @@ -44,6 +45,9 @@ def scargo_test( test_build_dir.mkdir(parents=True, exist_ok=True) conan_add_default_profile_if_missing() + conan_add_remote(project_dir, config) + conan_source(project_dir) + try: # Run CMake and build tests. subprocess.run( diff --git a/scargo/commands/update.py b/scargo/commands/update.py index 3d352ac1..067c6c12 100644 --- a/scargo/commands/update.py +++ b/scargo/commands/update.py @@ -3,12 +3,14 @@ # # """Update project""" +import os import shutil import subprocess import sys from pathlib import Path from scargo.commands.docker import get_docker_compose_command, scargo_docker_build +from scargo.config import Config from scargo.config_utils import add_version_to_scargo_lock, get_scargo_config_or_exit from scargo.file_generators.cicd_gen import generate_cicd from scargo.file_generators.cmake_gen import generate_cmake @@ -78,6 +80,9 @@ def scargo_update(config_file_path: Path) -> None: generate_conanfile(config) generate_conanprofile(config) + conan_add_remote(project_path, config) + conan_source(project_path) + if target.family == "esp32": Path(config.source_dir_path, "fs").mkdir(parents=True, exist_ok=True) with open(Path(project_path, "version.txt"), "w", encoding="utf-8") as out: @@ -128,3 +133,64 @@ def pull_docker_image(docker_path: Path) -> bool: logger.info("Docker image pulled successfully") return True return False + + +def conan_add_remote(project_path: Path, config: Config) -> None: + """ + Add conan remote repository + + :param Path project_path: path to project + :param Config config: + :return: None + """ + conan_repo = config.conan.repo + for repo_name, repo_url in conan_repo.items(): + try: + subprocess.run( + ["conan", "remote", "add", repo_name, repo_url], + cwd=project_path, + check=True, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as e: + if b"already exists in remotes" not in e.stderr: + logger.error(e.stderr.decode().strip()) + logger.error("Unable to add remote repository") + conan_add_user(repo_name) + + +def conan_add_user(remote: str) -> None: + """ + Add conan user + + :param str remote: name of remote repository + :return: None + """ + conan_user = subprocess.run( + "conan user", capture_output=True, shell=True, check=False + ).stdout.decode("utf-8") + + env_conan_user = os.environ.get("CONAN_LOGIN_USERNAME", "") + env_conan_passwd = os.environ.get("CONAN_PASSWORD", "") + + if env_conan_user not in conan_user: + try: + subprocess.check_call( + ["conan", "user", "-p", env_conan_passwd, "-r", remote, env_conan_user], + ) + except subprocess.CalledProcessError: + logger.error("Unable to add user") + + +def conan_source(project_dir: Path) -> None: + try: + subprocess.check_call( + [ + "conan", + "source", + ".", + ], + cwd=project_dir, + ) + except subprocess.CalledProcessError: + logger.error("Unable to source") diff --git a/tests/ut/ut_scargo_publish.py b/tests/ut/ut_scargo_publish.py index ba06893b..3bb3d1a9 100644 --- a/tests/ut/ut_scargo_publish.py +++ b/tests/ut/ut_scargo_publish.py @@ -1,14 +1,10 @@ -import os -import subprocess from pathlib import Path -from unittest.mock import patch import pytest from _pytest.logging import LogCaptureFixture from pytest_subprocess import FakeProcess -from pytest_subprocess.fake_popen import FakePopen -from scargo.commands.publish import conan_add_remote, conan_add_user, scargo_publish +from scargo.commands.publish import scargo_publish from scargo.config import Config from tests.ut.utils import get_test_project_config @@ -20,10 +16,6 @@ ENV_CONAN_PASSWORD = "env_conan_password" -def conan_remote_add_error(process: FakePopen, stderr: str) -> None: - raise subprocess.CalledProcessError(1, process.args, b"", stderr.encode()) - - @pytest.fixture def config(monkeypatch: pytest.MonkeyPatch) -> Config: test_project_config = get_test_project_config() @@ -44,11 +36,9 @@ def test_publish(config: Config, fp: FakeProcess) -> None: build_path = Path(f"{config.project_root}/build/Release") build_path.mkdir(parents=True, exist_ok=True) - conan_clean_cmd = "conan remote clean" conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] conan_user_cmd = "conan user" - conan_add_conacenter_cmd = "conan remote add conancenter https://center.conan.io" conan_source_cmd = [ "conan", "source", @@ -86,11 +76,9 @@ def test_publish(config: Config, fp: FakeProcess) -> None: "--confirm", ] - fp.register(conan_clean_cmd) fp.register(conan_add_remote_1_cmd) fp.register(conan_add_remote_2_cmd) fp.register(conan_user_cmd, occurrences=2) - fp.register(conan_add_conacenter_cmd) fp.register(conan_source_cmd) fp.register(conan_export_pkg_cmd) fp.register(conan_test_cmd) @@ -111,77 +99,6 @@ def test_publish(config: Config, fp: FakeProcess) -> None: assert len(fp.calls) == 8 -def test_conan_add_user(fp: FakeProcess) -> None: - # ARRANGE - conan_user_cmd = "conan user" - add_user_cmd = [ - "conan", - "user", - "-p", - ENV_CONAN_PASSWORD, - "-r", - REPO_NAME, - ENV_CONAN_USER, - ] - fp.register(conan_user_cmd, occurrences=2) - fp.register(add_user_cmd) - - # ACT - with patch.dict( - os.environ, - {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, - ): - conan_add_user(REPO_NAME) - - assert fp.calls[0] == conan_user_cmd - assert fp.calls[1] == add_user_cmd - assert len(fp.calls) == 2 - - -def test_conan_add_remote_already_exists( - config: Config, - caplog: pytest.LogCaptureFixture, - fp: FakeProcess, -) -> None: - # ARRANGE - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) - error = f"ERROR: Remote '{REMOTE_REPO_NAME_2}' already exists in remotes (use --force to continue)" - fp.register( - ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], - callback=conan_remote_add_error, - callback_kwargs={"stderr": error}, - ) - - # ACT - conan_add_remote(Path("some_path"), config) - - # ASSERT - assert "Unable to add remote repository" not in caplog.text - - -def test_conan_add_remote_fail( - config: Config, - caplog: pytest.LogCaptureFixture, - fp: FakeProcess, -) -> None: - # ARRANGE - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) - error = "ERROR: not remote add failure" - fp.register( - ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], - callback=conan_remote_add_error, - callback_kwargs={"stderr": error}, - ) - - # ACT - conan_add_remote(Path("some_path"), config) - - # ASSERT - assert "Unable to add remote repository" in caplog.text - - def test_create_package_fail( config: Config, caplog: LogCaptureFixture, @@ -192,11 +109,9 @@ def test_create_package_fail( build_path = Path(f"{config.project_root}/build/Release") build_path.mkdir(parents=True, exist_ok=True) - fp.register("conan remote clean") fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL]) fp.register("conan user", occurrences=2) - fp.register("conan remote add conancenter https://center.conan.io") fp.register( [ "conan", @@ -265,6 +180,7 @@ def test_upload_package_fail( build_path = Path(f"{config.project_root}/build/Release") build_path.mkdir(parents=True, exist_ok=True) + fp.register(["conan", "remote", "clean"]) fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL]) fp.register("conan user", occurrences=2) @@ -321,20 +237,3 @@ def test_upload_package_fail( # ASSERT assert "Unable to publish package" in caplog.text assert error.value.code == 1 - - -def test_conan_add_user_fail(caplog: pytest.LogCaptureFixture, fp: FakeProcess) -> None: - # ARRANGE - cmd = ["conan", "user", "-p", ENV_CONAN_PASSWORD, "-r", REPO_NAME, ENV_CONAN_USER] - fp.register(cmd, returncode=1) - fp.register("conan user", stdout="user_name") - - # ACT - with patch.dict( - os.environ, - {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, - ): - conan_add_user(REPO_NAME) - - # ASSERT - assert "Unable to add user" in caplog.text diff --git a/tests/ut/ut_scargo_test.py b/tests/ut/ut_scargo_test.py index 672150de..a3481473 100644 --- a/tests/ut/ut_scargo_test.py +++ b/tests/ut/ut_scargo_test.py @@ -9,11 +9,21 @@ from scargo.commands.test import scargo_test from scargo.config import Config +from tests.ut.ut_scargo_publish import ( + EXAMPLE_URL, + REMOTE_REPO_NAME_1, + REMOTE_REPO_NAME_2, + config, +) def test_scargo_test_no_test_dir( # type: ignore[no-any-unimported] - fs: FakeFilesystem, caplog: pytest.LogCaptureFixture, mock_prepare_config: MagicMock + config: Config, + fs: FakeFilesystem, + caplog: pytest.LogCaptureFixture, + mock_prepare_config: MagicMock, ) -> None: + config.project_root = Path(".") with pytest.raises(SystemExit) as e: scargo_test(False) assert e.value.code == 1 @@ -21,10 +31,12 @@ def test_scargo_test_no_test_dir( # type: ignore[no-any-unimported] def test_scargo_test_no_cmake_file( # type: ignore[no-any-unimported] + config: Config, caplog: pytest.LogCaptureFixture, mock_prepare_config: MagicMock, fs: FakeFilesystem, ) -> None: + config.project_root = Path(".") os.mkdir("tests") with pytest.raises(SystemExit) as e: scargo_test(False) @@ -33,8 +45,22 @@ def test_scargo_test_no_cmake_file( # type: ignore[no-any-unimported] def test_scargo_test( # type: ignore[no-any-unimported] - fp: FakeProcess, fs: FakeFilesystem, mock_prepare_config: MagicMock + config: Config, fp: FakeProcess, fs: FakeFilesystem, mock_prepare_config: MagicMock ) -> None: + config.project_root = Path(".") + conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] + conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] + conan_user_cmd = "conan user" + conan_source_cmd = [ + "conan", + "source", + ".", + ] + + fp.register(conan_add_remote_1_cmd) + fp.register(conan_add_remote_2_cmd) + fp.register(conan_user_cmd, occurrences=2) + fp.register(conan_source_cmd) fp.register("conan profile list") fp.register("conan profile detect") fp.register("conan install tests -of build/tests -sbuild_type=Debug -b missing") @@ -56,7 +82,12 @@ def test_scargo_test( # type: ignore[no-any-unimported] "profile", "detect", ] - assert fp.calls[2] == [ + assert fp.calls[2] == conan_add_remote_1_cmd + assert fp.calls[3] == conan_user_cmd + assert fp.calls[4] == conan_add_remote_2_cmd + assert fp.calls[5] == conan_user_cmd + assert fp.calls[6] == conan_source_cmd + assert fp.calls[7] == [ "conan", "install", Path("tests"), @@ -66,7 +97,7 @@ def test_scargo_test( # type: ignore[no-any-unimported] "-b", "missing", ] - assert fp.calls[3] == [ + assert fp.calls[8] == [ "conan", "build", "-of", @@ -76,8 +107,8 @@ def test_scargo_test( # type: ignore[no-any-unimported] "-b", "missing", ] - assert fp.calls[4] == ["ctest"] - assert fp.calls[5] == [ + assert fp.calls[9] == ["ctest"] + assert fp.calls[10] == [ "gcovr", "-r", "ut", diff --git a/tests/ut/ut_scargo_update.py b/tests/ut/ut_scargo_update.py index 0f736a86..9d7b178b 100644 --- a/tests/ut/ut_scargo_update.py +++ b/tests/ut/ut_scargo_update.py @@ -1,12 +1,25 @@ import os +import subprocess from pathlib import Path +from unittest.mock import patch +import pytest from pytest_subprocess import FakeProcess +from pytest_subprocess.fake_popen import FakePopen from scargo.commands.docker import get_docker_compose_command from scargo.commands.new import scargo_new -from scargo.commands.update import scargo_update -from scargo.config import Target +from scargo.commands.update import conan_add_remote, conan_add_user, scargo_update +from scargo.config import Config, Target +from tests.ut.ut_scargo_publish import ( + ENV_CONAN_PASSWORD, + ENV_CONAN_USER, + EXAMPLE_URL, + REMOTE_REPO_NAME_1, + REMOTE_REPO_NAME_2, + REPO_NAME, + config, +) EXPECTED_FILES_AND_DIRS = [ ".clang-format", @@ -30,6 +43,10 @@ TARGET_X86 = Target.get_target_by_id("x86") +def conan_remote_add_error(process: FakePopen, stderr: str) -> None: + raise subprocess.CalledProcessError(1, process.args, b"", stderr.encode()) + + def test_update_project_content_without_docker(tmp_path: Path) -> None: os.chdir(tmp_path) project_name = "test_project" @@ -47,6 +64,19 @@ def test_update_project_content_without_docker(tmp_path: Path) -> None: def test_update_project_content_with_docker(tmp_path: Path, fp: FakeProcess) -> None: + conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] + conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] + conan_user_cmd = "conan user" + conan_source_cmd = [ + "conan", + "source", + ".", + ] + + fp.register(conan_add_remote_1_cmd) + fp.register(conan_add_remote_2_cmd) + fp.register(conan_user_cmd, occurrences=2) + fp.register(conan_source_cmd) os.chdir(tmp_path) project_name = "test_project_with_docker" scargo_new(project_name, None, None, TARGET_X86, True, False, None) @@ -64,6 +94,19 @@ def test_update_project_content_with_docker(tmp_path: Path, fp: FakeProcess) -> def test_update_project_content_with_docker__build( tmp_path: Path, fp: FakeProcess ) -> None: + conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] + conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] + conan_user_cmd = "conan user" + conan_source_cmd = [ + "conan", + "source", + ".", + ] + + fp.register(conan_add_remote_1_cmd) + fp.register(conan_add_remote_2_cmd) + fp.register(conan_user_cmd, occurrences=2) + fp.register(conan_source_cmd) os.chdir(tmp_path) project_name = "test_project_with_docker" scargo_new(project_name, None, None, TARGET_X86, True, False, None) @@ -80,3 +123,88 @@ def test_update_project_content_with_docker__build( assert fp.call_count(cmd_build) == 1 for path in Path().iterdir(): assert path.name in EXPECTED_FILES_AND_DIRS + + +def test_conan_add_user(fp: FakeProcess) -> None: + # ARRANGE + conan_user_cmd = "conan user" + add_user_cmd = [ + "conan", + "user", + "-p", + ENV_CONAN_PASSWORD, + "-r", + REPO_NAME, + ENV_CONAN_USER, + ] + fp.register(conan_user_cmd, occurrences=2) + fp.register(add_user_cmd) + + # ACT + with patch.dict( + os.environ, + {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, + ): + conan_add_user(REPO_NAME) + + assert fp.calls[0] == conan_user_cmd + assert fp.calls[1] == add_user_cmd + assert len(fp.calls) == 2 + + +def test_conan_add_remote_fail( + config: Config, + caplog: pytest.LogCaptureFixture, + fp: FakeProcess, +) -> None: + # ARRANGE + fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) + fp.register("conan user", occurrences=2) + fp.register( + ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], returncode=1 + ) + + # ACT + conan_add_remote(Path("some_path"), config) + + # ASSERT + assert "Unable to add remote repository" in caplog.text + + +def test_conan_add_user_fail(caplog: pytest.LogCaptureFixture, fp: FakeProcess) -> None: + # ARRANGE + cmd = ["conan", "user", "-p", ENV_CONAN_PASSWORD, "-r", REPO_NAME, ENV_CONAN_USER] + fp.register(cmd, returncode=1) + fp.register("conan user", stdout="user_name") + + # ACT + with patch.dict( + os.environ, + {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, + ): + conan_add_user(REPO_NAME) + + # ASSERT + assert "Unable to add user" in caplog.text + + +def test_conan_add_remote_already_exists( + config: Config, + caplog: pytest.LogCaptureFixture, + fp: FakeProcess, +) -> None: + # ARRANGE + fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) + fp.register("conan user", occurrences=2) + error = f"ERROR: Remote '{REMOTE_REPO_NAME_2}' already exists in remotes (use --force to continue)" + fp.register( + ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], + callback=conan_remote_add_error, + callback_kwargs={"stderr": error}, + ) + + # ACT + conan_add_remote(Path("some_path"), config) + + # ASSERT + assert "Unable to add remote repository" not in caplog.text From 535a977bd268e93849d0bfdbb5897ce1f42ba167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Fri, 27 Oct 2023 10:43:12 +0200 Subject: [PATCH 14/28] Moved conan_source function to build.py --- scargo/commands/build.py | 16 +++++++++++++++- scargo/commands/update.py | 1 - tests/ut/ut_scargo_update.py | 12 ------------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/scargo/commands/build.py b/scargo/commands/build.py index 351182c3..48bff4cb 100644 --- a/scargo/commands/build.py +++ b/scargo/commands/build.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -from scargo.commands.update import conan_add_remote, conan_source +from scargo.commands.update import conan_add_remote from scargo.config_utils import prepare_config from scargo.file_generators.conan_gen import conan_add_default_profile_if_missing from scargo.logger import get_logger @@ -85,3 +85,17 @@ def scargo_build(profile: str) -> None: except subprocess.CalledProcessError: logger.error("Unable to build exec file") sys.exit(1) + + +def conan_source(project_dir: Path) -> None: + try: + subprocess.check_call( + [ + "conan", + "source", + ".", + ], + cwd=project_dir, + ) + except subprocess.CalledProcessError: + logger.error("Unable to source") \ No newline at end of file diff --git a/scargo/commands/update.py b/scargo/commands/update.py index 067c6c12..a74123e7 100644 --- a/scargo/commands/update.py +++ b/scargo/commands/update.py @@ -81,7 +81,6 @@ def scargo_update(config_file_path: Path) -> None: generate_conanprofile(config) conan_add_remote(project_path, config) - conan_source(project_path) if target.family == "esp32": Path(config.source_dir_path, "fs").mkdir(parents=True, exist_ok=True) diff --git a/tests/ut/ut_scargo_update.py b/tests/ut/ut_scargo_update.py index 9d7b178b..688f92b2 100644 --- a/tests/ut/ut_scargo_update.py +++ b/tests/ut/ut_scargo_update.py @@ -67,16 +67,10 @@ def test_update_project_content_with_docker(tmp_path: Path, fp: FakeProcess) -> conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] conan_user_cmd = "conan user" - conan_source_cmd = [ - "conan", - "source", - ".", - ] fp.register(conan_add_remote_1_cmd) fp.register(conan_add_remote_2_cmd) fp.register(conan_user_cmd, occurrences=2) - fp.register(conan_source_cmd) os.chdir(tmp_path) project_name = "test_project_with_docker" scargo_new(project_name, None, None, TARGET_X86, True, False, None) @@ -97,16 +91,10 @@ def test_update_project_content_with_docker__build( conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] conan_user_cmd = "conan user" - conan_source_cmd = [ - "conan", - "source", - ".", - ] fp.register(conan_add_remote_1_cmd) fp.register(conan_add_remote_2_cmd) fp.register(conan_user_cmd, occurrences=2) - fp.register(conan_source_cmd) os.chdir(tmp_path) project_name = "test_project_with_docker" scargo_new(project_name, None, None, TARGET_X86, True, False, None) From 4a809bd32f962fb191f5f66a2b7a951fade0f5b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Fri, 27 Oct 2023 10:49:28 +0200 Subject: [PATCH 15/28] Fixed some black issues --- scargo/commands/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scargo/commands/build.py b/scargo/commands/build.py index 48bff4cb..fa5e5787 100644 --- a/scargo/commands/build.py +++ b/scargo/commands/build.py @@ -98,4 +98,4 @@ def conan_source(project_dir: Path) -> None: cwd=project_dir, ) except subprocess.CalledProcessError: - logger.error("Unable to source") \ No newline at end of file + logger.error("Unable to source") From dfd65491a9ca0233a19eba8267abff24146d16db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Fri, 27 Oct 2023 11:12:24 +0200 Subject: [PATCH 16/28] Moved conan_add_remote, conan_add_user and conan_source to test.py --- scargo/commands/build.py | 16 +---- scargo/commands/publish.py | 2 +- scargo/commands/test.py | 63 +++++++++++++++++- scargo/commands/update.py | 65 ------------------- tests/ut/ut_scargo_test.py | 98 +++++++++++++++++++++++++++- tests/ut/ut_scargo_update.py | 120 +---------------------------------- 6 files changed, 162 insertions(+), 202 deletions(-) diff --git a/scargo/commands/build.py b/scargo/commands/build.py index fa5e5787..a4380a16 100644 --- a/scargo/commands/build.py +++ b/scargo/commands/build.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -from scargo.commands.update import conan_add_remote +from scargo.commands.test import conan_add_remote, conan_source from scargo.config_utils import prepare_config from scargo.file_generators.conan_gen import conan_add_default_profile_if_missing from scargo.logger import get_logger @@ -85,17 +85,3 @@ def scargo_build(profile: str) -> None: except subprocess.CalledProcessError: logger.error("Unable to build exec file") sys.exit(1) - - -def conan_source(project_dir: Path) -> None: - try: - subprocess.check_call( - [ - "conan", - "source", - ".", - ], - cwd=project_dir, - ) - except subprocess.CalledProcessError: - logger.error("Unable to source") diff --git a/scargo/commands/publish.py b/scargo/commands/publish.py index 1c6ed180..3f70c456 100644 --- a/scargo/commands/publish.py +++ b/scargo/commands/publish.py @@ -8,7 +8,7 @@ import sys from pathlib import Path -from scargo.commands.update import conan_add_remote, conan_source +from scargo.commands.test import conan_add_remote, conan_source from scargo.config_utils import prepare_config from scargo.logger import get_logger diff --git a/scargo/commands/test.py b/scargo/commands/test.py index 3017caf6..8c669e6b 100644 --- a/scargo/commands/test.py +++ b/scargo/commands/test.py @@ -3,12 +3,12 @@ # # """Run test from project""" +import os import subprocess import sys from pathlib import Path from typing import List, Union -from scargo.commands.update import conan_add_remote, conan_source from scargo.config import Config from scargo.config_utils import prepare_config from scargo.file_generators.conan_gen import conan_add_default_profile_if_missing @@ -156,3 +156,64 @@ def run_it(config: Config, verbose: bool) -> None: subprocess.check_call(cmd) subprocess.check_call("cat it-coverage.txt", shell=True) + + +def conan_add_remote(project_path: Path, config: Config) -> None: + """ + Add conan remote repository + + :param Path project_path: path to project + :param Config config: + :return: None + """ + conan_repo = config.conan.repo + for repo_name, repo_url in conan_repo.items(): + try: + subprocess.run( + ["conan", "remote", "add", repo_name, repo_url], + cwd=project_path, + check=True, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as e: + if b"already exists in remotes" not in e.stderr: + logger.error(e.stderr.decode().strip()) + logger.error("Unable to add remote repository") + conan_add_user(repo_name) + + +def conan_add_user(remote: str) -> None: + """ + Add conan user + + :param str remote: name of remote repository + :return: None + """ + conan_user = subprocess.run( + "conan user", capture_output=True, shell=True, check=False + ).stdout.decode("utf-8") + + env_conan_user = os.environ.get("CONAN_LOGIN_USERNAME", "") + env_conan_passwd = os.environ.get("CONAN_PASSWORD", "") + + if env_conan_user not in conan_user: + try: + subprocess.check_call( + ["conan", "user", "-p", env_conan_passwd, "-r", remote, env_conan_user], + ) + except subprocess.CalledProcessError: + logger.error("Unable to add user") + + +def conan_source(project_dir: Path) -> None: + try: + subprocess.check_call( + [ + "conan", + "source", + ".", + ], + cwd=project_dir, + ) + except subprocess.CalledProcessError: + logger.error("Unable to source") diff --git a/scargo/commands/update.py b/scargo/commands/update.py index a74123e7..3d352ac1 100644 --- a/scargo/commands/update.py +++ b/scargo/commands/update.py @@ -3,14 +3,12 @@ # # """Update project""" -import os import shutil import subprocess import sys from pathlib import Path from scargo.commands.docker import get_docker_compose_command, scargo_docker_build -from scargo.config import Config from scargo.config_utils import add_version_to_scargo_lock, get_scargo_config_or_exit from scargo.file_generators.cicd_gen import generate_cicd from scargo.file_generators.cmake_gen import generate_cmake @@ -80,8 +78,6 @@ def scargo_update(config_file_path: Path) -> None: generate_conanfile(config) generate_conanprofile(config) - conan_add_remote(project_path, config) - if target.family == "esp32": Path(config.source_dir_path, "fs").mkdir(parents=True, exist_ok=True) with open(Path(project_path, "version.txt"), "w", encoding="utf-8") as out: @@ -132,64 +128,3 @@ def pull_docker_image(docker_path: Path) -> bool: logger.info("Docker image pulled successfully") return True return False - - -def conan_add_remote(project_path: Path, config: Config) -> None: - """ - Add conan remote repository - - :param Path project_path: path to project - :param Config config: - :return: None - """ - conan_repo = config.conan.repo - for repo_name, repo_url in conan_repo.items(): - try: - subprocess.run( - ["conan", "remote", "add", repo_name, repo_url], - cwd=project_path, - check=True, - stderr=subprocess.PIPE, - ) - except subprocess.CalledProcessError as e: - if b"already exists in remotes" not in e.stderr: - logger.error(e.stderr.decode().strip()) - logger.error("Unable to add remote repository") - conan_add_user(repo_name) - - -def conan_add_user(remote: str) -> None: - """ - Add conan user - - :param str remote: name of remote repository - :return: None - """ - conan_user = subprocess.run( - "conan user", capture_output=True, shell=True, check=False - ).stdout.decode("utf-8") - - env_conan_user = os.environ.get("CONAN_LOGIN_USERNAME", "") - env_conan_passwd = os.environ.get("CONAN_PASSWORD", "") - - if env_conan_user not in conan_user: - try: - subprocess.check_call( - ["conan", "user", "-p", env_conan_passwd, "-r", remote, env_conan_user], - ) - except subprocess.CalledProcessError: - logger.error("Unable to add user") - - -def conan_source(project_dir: Path) -> None: - try: - subprocess.check_call( - [ - "conan", - "source", - ".", - ], - cwd=project_dir, - ) - except subprocess.CalledProcessError: - logger.error("Unable to source") diff --git a/tests/ut/ut_scargo_test.py b/tests/ut/ut_scargo_test.py index a3481473..dae19794 100644 --- a/tests/ut/ut_scargo_test.py +++ b/tests/ut/ut_scargo_test.py @@ -1,22 +1,31 @@ import os +import subprocess from pathlib import Path -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest from pyfakefs.fake_filesystem import FakeFilesystem from pytest_mock import MockerFixture from pytest_subprocess import FakeProcess +from pytest_subprocess.fake_popen import FakePopen -from scargo.commands.test import scargo_test +from scargo.commands.test import conan_add_remote, conan_add_user, scargo_test from scargo.config import Config from tests.ut.ut_scargo_publish import ( + ENV_CONAN_PASSWORD, + ENV_CONAN_USER, EXAMPLE_URL, REMOTE_REPO_NAME_1, REMOTE_REPO_NAME_2, + REPO_NAME, config, ) +def conan_remote_add_error(process: FakePopen, stderr: str) -> None: + raise subprocess.CalledProcessError(1, process.args, b"", stderr.encode()) + + def test_scargo_test_no_test_dir( # type: ignore[no-any-unimported] config: Config, fs: FakeFilesystem, @@ -122,3 +131,88 @@ def test_scargo_test( # type: ignore[no-any-unimported] @pytest.fixture def mock_prepare_config(mocker: MockerFixture, config: Config) -> MagicMock: return mocker.patch(f"{scargo_test.__module__}.prepare_config", return_value=config) + + +def test_conan_add_user(fp: FakeProcess) -> None: + # ARRANGE + conan_user_cmd = "conan user" + add_user_cmd = [ + "conan", + "user", + "-p", + ENV_CONAN_PASSWORD, + "-r", + REPO_NAME, + ENV_CONAN_USER, + ] + fp.register(conan_user_cmd, occurrences=2) + fp.register(add_user_cmd) + + # ACT + with patch.dict( + os.environ, + {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, + ): + conan_add_user(REPO_NAME) + + assert fp.calls[0] == conan_user_cmd + assert fp.calls[1] == add_user_cmd + assert len(fp.calls) == 2 + + +def test_conan_add_remote_fail( + config: Config, + caplog: pytest.LogCaptureFixture, + fp: FakeProcess, +) -> None: + # ARRANGE + fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) + fp.register("conan user", occurrences=2) + fp.register( + ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], returncode=1 + ) + + # ACT + conan_add_remote(Path("some_path"), config) + + # ASSERT + assert "Unable to add remote repository" in caplog.text + + +def test_conan_add_user_fail(caplog: pytest.LogCaptureFixture, fp: FakeProcess) -> None: + # ARRANGE + cmd = ["conan", "user", "-p", ENV_CONAN_PASSWORD, "-r", REPO_NAME, ENV_CONAN_USER] + fp.register(cmd, returncode=1) + fp.register("conan user", stdout="user_name") + + # ACT + with patch.dict( + os.environ, + {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, + ): + conan_add_user(REPO_NAME) + + # ASSERT + assert "Unable to add user" in caplog.text + + +def test_conan_add_remote_already_exists( + config: Config, + caplog: pytest.LogCaptureFixture, + fp: FakeProcess, +) -> None: + # ARRANGE + fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) + fp.register("conan user", occurrences=2) + error = f"ERROR: Remote '{REMOTE_REPO_NAME_2}' already exists in remotes (use --force to continue)" + fp.register( + ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], + callback=conan_remote_add_error, + callback_kwargs={"stderr": error}, + ) + + # ACT + conan_add_remote(Path("some_path"), config) + + # ASSERT + assert "Unable to add remote repository" not in caplog.text diff --git a/tests/ut/ut_scargo_update.py b/tests/ut/ut_scargo_update.py index 688f92b2..0f736a86 100644 --- a/tests/ut/ut_scargo_update.py +++ b/tests/ut/ut_scargo_update.py @@ -1,25 +1,12 @@ import os -import subprocess from pathlib import Path -from unittest.mock import patch -import pytest from pytest_subprocess import FakeProcess -from pytest_subprocess.fake_popen import FakePopen from scargo.commands.docker import get_docker_compose_command from scargo.commands.new import scargo_new -from scargo.commands.update import conan_add_remote, conan_add_user, scargo_update -from scargo.config import Config, Target -from tests.ut.ut_scargo_publish import ( - ENV_CONAN_PASSWORD, - ENV_CONAN_USER, - EXAMPLE_URL, - REMOTE_REPO_NAME_1, - REMOTE_REPO_NAME_2, - REPO_NAME, - config, -) +from scargo.commands.update import scargo_update +from scargo.config import Target EXPECTED_FILES_AND_DIRS = [ ".clang-format", @@ -43,10 +30,6 @@ TARGET_X86 = Target.get_target_by_id("x86") -def conan_remote_add_error(process: FakePopen, stderr: str) -> None: - raise subprocess.CalledProcessError(1, process.args, b"", stderr.encode()) - - def test_update_project_content_without_docker(tmp_path: Path) -> None: os.chdir(tmp_path) project_name = "test_project" @@ -64,13 +47,6 @@ def test_update_project_content_without_docker(tmp_path: Path) -> None: def test_update_project_content_with_docker(tmp_path: Path, fp: FakeProcess) -> None: - conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] - conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] - conan_user_cmd = "conan user" - - fp.register(conan_add_remote_1_cmd) - fp.register(conan_add_remote_2_cmd) - fp.register(conan_user_cmd, occurrences=2) os.chdir(tmp_path) project_name = "test_project_with_docker" scargo_new(project_name, None, None, TARGET_X86, True, False, None) @@ -88,13 +64,6 @@ def test_update_project_content_with_docker(tmp_path: Path, fp: FakeProcess) -> def test_update_project_content_with_docker__build( tmp_path: Path, fp: FakeProcess ) -> None: - conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] - conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] - conan_user_cmd = "conan user" - - fp.register(conan_add_remote_1_cmd) - fp.register(conan_add_remote_2_cmd) - fp.register(conan_user_cmd, occurrences=2) os.chdir(tmp_path) project_name = "test_project_with_docker" scargo_new(project_name, None, None, TARGET_X86, True, False, None) @@ -111,88 +80,3 @@ def test_update_project_content_with_docker__build( assert fp.call_count(cmd_build) == 1 for path in Path().iterdir(): assert path.name in EXPECTED_FILES_AND_DIRS - - -def test_conan_add_user(fp: FakeProcess) -> None: - # ARRANGE - conan_user_cmd = "conan user" - add_user_cmd = [ - "conan", - "user", - "-p", - ENV_CONAN_PASSWORD, - "-r", - REPO_NAME, - ENV_CONAN_USER, - ] - fp.register(conan_user_cmd, occurrences=2) - fp.register(add_user_cmd) - - # ACT - with patch.dict( - os.environ, - {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, - ): - conan_add_user(REPO_NAME) - - assert fp.calls[0] == conan_user_cmd - assert fp.calls[1] == add_user_cmd - assert len(fp.calls) == 2 - - -def test_conan_add_remote_fail( - config: Config, - caplog: pytest.LogCaptureFixture, - fp: FakeProcess, -) -> None: - # ARRANGE - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) - fp.register( - ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], returncode=1 - ) - - # ACT - conan_add_remote(Path("some_path"), config) - - # ASSERT - assert "Unable to add remote repository" in caplog.text - - -def test_conan_add_user_fail(caplog: pytest.LogCaptureFixture, fp: FakeProcess) -> None: - # ARRANGE - cmd = ["conan", "user", "-p", ENV_CONAN_PASSWORD, "-r", REPO_NAME, ENV_CONAN_USER] - fp.register(cmd, returncode=1) - fp.register("conan user", stdout="user_name") - - # ACT - with patch.dict( - os.environ, - {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, - ): - conan_add_user(REPO_NAME) - - # ASSERT - assert "Unable to add user" in caplog.text - - -def test_conan_add_remote_already_exists( - config: Config, - caplog: pytest.LogCaptureFixture, - fp: FakeProcess, -) -> None: - # ARRANGE - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) - error = f"ERROR: Remote '{REMOTE_REPO_NAME_2}' already exists in remotes (use --force to continue)" - fp.register( - ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], - callback=conan_remote_add_error, - callback_kwargs={"stderr": error}, - ) - - # ACT - conan_add_remote(Path("some_path"), config) - - # ASSERT - assert "Unable to add remote repository" not in caplog.text From 28b5ff44b9c501388eab205b1dbde412e0dc29d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Fri, 27 Oct 2023 14:12:01 +0200 Subject: [PATCH 17/28] Moved conan_add_remote, conan_add_user and conan_source to scargo\conan_utils.py --- scargo/commands/build.py | 2 +- scargo/commands/test.py | 63 +--------------------------------- scargo/conan_utils.py | 69 ++++++++++++++++++++++++++++++++++++++ tests/ut/ut_scargo_test.py | 3 +- 4 files changed, 73 insertions(+), 64 deletions(-) create mode 100644 scargo/conan_utils.py diff --git a/scargo/commands/build.py b/scargo/commands/build.py index a4380a16..a9df0550 100644 --- a/scargo/commands/build.py +++ b/scargo/commands/build.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -from scargo.commands.test import conan_add_remote, conan_source +from scargo.conan_utils import conan_add_remote, conan_source from scargo.config_utils import prepare_config from scargo.file_generators.conan_gen import conan_add_default_profile_if_missing from scargo.logger import get_logger diff --git a/scargo/commands/test.py b/scargo/commands/test.py index 8c669e6b..5a36b911 100644 --- a/scargo/commands/test.py +++ b/scargo/commands/test.py @@ -3,12 +3,12 @@ # # """Run test from project""" -import os import subprocess import sys from pathlib import Path from typing import List, Union +from scargo.conan_utils import conan_add_remote, conan_source from scargo.config import Config from scargo.config_utils import prepare_config from scargo.file_generators.conan_gen import conan_add_default_profile_if_missing @@ -156,64 +156,3 @@ def run_it(config: Config, verbose: bool) -> None: subprocess.check_call(cmd) subprocess.check_call("cat it-coverage.txt", shell=True) - - -def conan_add_remote(project_path: Path, config: Config) -> None: - """ - Add conan remote repository - - :param Path project_path: path to project - :param Config config: - :return: None - """ - conan_repo = config.conan.repo - for repo_name, repo_url in conan_repo.items(): - try: - subprocess.run( - ["conan", "remote", "add", repo_name, repo_url], - cwd=project_path, - check=True, - stderr=subprocess.PIPE, - ) - except subprocess.CalledProcessError as e: - if b"already exists in remotes" not in e.stderr: - logger.error(e.stderr.decode().strip()) - logger.error("Unable to add remote repository") - conan_add_user(repo_name) - - -def conan_add_user(remote: str) -> None: - """ - Add conan user - - :param str remote: name of remote repository - :return: None - """ - conan_user = subprocess.run( - "conan user", capture_output=True, shell=True, check=False - ).stdout.decode("utf-8") - - env_conan_user = os.environ.get("CONAN_LOGIN_USERNAME", "") - env_conan_passwd = os.environ.get("CONAN_PASSWORD", "") - - if env_conan_user not in conan_user: - try: - subprocess.check_call( - ["conan", "user", "-p", env_conan_passwd, "-r", remote, env_conan_user], - ) - except subprocess.CalledProcessError: - logger.error("Unable to add user") - - -def conan_source(project_dir: Path) -> None: - try: - subprocess.check_call( - [ - "conan", - "source", - ".", - ], - cwd=project_dir, - ) - except subprocess.CalledProcessError: - logger.error("Unable to source") diff --git a/scargo/conan_utils.py b/scargo/conan_utils.py new file mode 100644 index 00000000..ec0497ee --- /dev/null +++ b/scargo/conan_utils.py @@ -0,0 +1,69 @@ +import os +import subprocess +from pathlib import Path + +from scargo.config import Config +from scargo.logger import get_logger + +logger = get_logger() + + +def conan_add_remote(project_path: Path, config: Config) -> None: + """ + Add conan remote repository + + :param Path project_path: path to project + :param Config config: + :return: None + """ + conan_repo = config.conan.repo + for repo_name, repo_url in conan_repo.items(): + try: + subprocess.run( + ["conan", "remote", "add", repo_name, repo_url], + cwd=project_path, + check=True, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as e: + if b"already exists in remotes" not in e.stderr: + logger.error(e.stderr.decode().strip()) + logger.error("Unable to add remote repository") + conan_add_user(repo_name) + + +def conan_add_user(remote: str) -> None: + """ + Add conan user + + :param str remote: name of remote repository + :return: None + """ + conan_user = subprocess.run( + "conan user", capture_output=True, shell=True, check=False + ).stdout.decode("utf-8") + + env_conan_user = os.environ.get("CONAN_LOGIN_USERNAME", "") + env_conan_passwd = os.environ.get("CONAN_PASSWORD", "") + + if env_conan_user not in conan_user: + try: + subprocess.check_call( + ["conan", "user", "-p", env_conan_passwd, "-r", remote, env_conan_user], + ) + except subprocess.CalledProcessError: + logger.error("Unable to add user") + + +def conan_source(project_dir: Path) -> None: + try: + subprocess.check_call( + [ + "conan", + "source", + ".", + ], + cwd=project_dir, + ) + except subprocess.CalledProcessError: + logger.error("Unable to source") diff --git a/tests/ut/ut_scargo_test.py b/tests/ut/ut_scargo_test.py index dae19794..5189cd07 100644 --- a/tests/ut/ut_scargo_test.py +++ b/tests/ut/ut_scargo_test.py @@ -9,7 +9,8 @@ from pytest_subprocess import FakeProcess from pytest_subprocess.fake_popen import FakePopen -from scargo.commands.test import conan_add_remote, conan_add_user, scargo_test +from scargo.commands.test import scargo_test +from scargo.conan_utils import conan_add_remote, conan_add_user from scargo.config import Config from tests.ut.ut_scargo_publish import ( ENV_CONAN_PASSWORD, From 30096385976eac0dd08f46e7df1f9688df6a6213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Fri, 27 Oct 2023 14:19:38 +0200 Subject: [PATCH 18/28] Created a conan_utils.py file for unit tests --- tests/ut/conan_utils.py | 109 +++++++++++++++++++++++++++++++++++++ tests/ut/ut_scargo_test.py | 97 +-------------------------------- 2 files changed, 110 insertions(+), 96 deletions(-) create mode 100644 tests/ut/conan_utils.py diff --git a/tests/ut/conan_utils.py b/tests/ut/conan_utils.py new file mode 100644 index 00000000..f2dbc998 --- /dev/null +++ b/tests/ut/conan_utils.py @@ -0,0 +1,109 @@ +import os +import subprocess +from pathlib import Path +from unittest.mock import patch + +import pytest +from pytest_subprocess import FakeProcess +from pytest_subprocess.fake_popen import FakePopen + +from scargo.conan_utils import conan_add_remote, conan_add_user +from scargo.config import Config +from tests.ut.ut_scargo_publish import ( + ENV_CONAN_PASSWORD, + ENV_CONAN_USER, + EXAMPLE_URL, + REMOTE_REPO_NAME_1, + REMOTE_REPO_NAME_2, + REPO_NAME, + config, +) + + +def conan_remote_add_error(process: FakePopen, stderr: str) -> None: + raise subprocess.CalledProcessError(1, process.args, b"", stderr.encode()) + + +def test_conan_add_user(fp: FakeProcess) -> None: + # ARRANGE + conan_user_cmd = "conan user" + add_user_cmd = [ + "conan", + "user", + "-p", + ENV_CONAN_PASSWORD, + "-r", + REPO_NAME, + ENV_CONAN_USER, + ] + fp.register(conan_user_cmd, occurrences=2) + fp.register(add_user_cmd) + + # ACT + with patch.dict( + os.environ, + {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, + ): + conan_add_user(REPO_NAME) + + assert fp.calls[0] == conan_user_cmd + assert fp.calls[1] == add_user_cmd + assert len(fp.calls) == 2 + + +def test_conan_add_remote_fail( + config: Config, + caplog: pytest.LogCaptureFixture, + fp: FakeProcess, +) -> None: + # ARRANGE + fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) + fp.register("conan user", occurrences=2) + fp.register( + ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], returncode=1 + ) + + # ACT + conan_add_remote(Path("some_path"), config) + + # ASSERT + assert "Unable to add remote repository" in caplog.text + + +def test_conan_add_user_fail(caplog: pytest.LogCaptureFixture, fp: FakeProcess) -> None: + # ARRANGE + cmd = ["conan", "user", "-p", ENV_CONAN_PASSWORD, "-r", REPO_NAME, ENV_CONAN_USER] + fp.register(cmd, returncode=1) + fp.register("conan user", stdout="user_name") + + # ACT + with patch.dict( + os.environ, + {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, + ): + conan_add_user(REPO_NAME) + + # ASSERT + assert "Unable to add user" in caplog.text + + +def test_conan_add_remote_already_exists( + config: Config, + caplog: pytest.LogCaptureFixture, + fp: FakeProcess, +) -> None: + # ARRANGE + fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) + fp.register("conan user", occurrences=2) + error = f"ERROR: Remote '{REMOTE_REPO_NAME_2}' already exists in remotes (use --force to continue)" + fp.register( + ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], + callback=conan_remote_add_error, + callback_kwargs={"stderr": error}, + ) + + # ACT + conan_add_remote(Path("some_path"), config) + + # ASSERT + assert "Unable to add remote repository" not in caplog.text diff --git a/tests/ut/ut_scargo_test.py b/tests/ut/ut_scargo_test.py index 5189cd07..a3481473 100644 --- a/tests/ut/ut_scargo_test.py +++ b/tests/ut/ut_scargo_test.py @@ -1,32 +1,22 @@ import os -import subprocess from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import pytest from pyfakefs.fake_filesystem import FakeFilesystem from pytest_mock import MockerFixture from pytest_subprocess import FakeProcess -from pytest_subprocess.fake_popen import FakePopen from scargo.commands.test import scargo_test -from scargo.conan_utils import conan_add_remote, conan_add_user from scargo.config import Config from tests.ut.ut_scargo_publish import ( - ENV_CONAN_PASSWORD, - ENV_CONAN_USER, EXAMPLE_URL, REMOTE_REPO_NAME_1, REMOTE_REPO_NAME_2, - REPO_NAME, config, ) -def conan_remote_add_error(process: FakePopen, stderr: str) -> None: - raise subprocess.CalledProcessError(1, process.args, b"", stderr.encode()) - - def test_scargo_test_no_test_dir( # type: ignore[no-any-unimported] config: Config, fs: FakeFilesystem, @@ -132,88 +122,3 @@ def test_scargo_test( # type: ignore[no-any-unimported] @pytest.fixture def mock_prepare_config(mocker: MockerFixture, config: Config) -> MagicMock: return mocker.patch(f"{scargo_test.__module__}.prepare_config", return_value=config) - - -def test_conan_add_user(fp: FakeProcess) -> None: - # ARRANGE - conan_user_cmd = "conan user" - add_user_cmd = [ - "conan", - "user", - "-p", - ENV_CONAN_PASSWORD, - "-r", - REPO_NAME, - ENV_CONAN_USER, - ] - fp.register(conan_user_cmd, occurrences=2) - fp.register(add_user_cmd) - - # ACT - with patch.dict( - os.environ, - {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, - ): - conan_add_user(REPO_NAME) - - assert fp.calls[0] == conan_user_cmd - assert fp.calls[1] == add_user_cmd - assert len(fp.calls) == 2 - - -def test_conan_add_remote_fail( - config: Config, - caplog: pytest.LogCaptureFixture, - fp: FakeProcess, -) -> None: - # ARRANGE - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) - fp.register( - ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], returncode=1 - ) - - # ACT - conan_add_remote(Path("some_path"), config) - - # ASSERT - assert "Unable to add remote repository" in caplog.text - - -def test_conan_add_user_fail(caplog: pytest.LogCaptureFixture, fp: FakeProcess) -> None: - # ARRANGE - cmd = ["conan", "user", "-p", ENV_CONAN_PASSWORD, "-r", REPO_NAME, ENV_CONAN_USER] - fp.register(cmd, returncode=1) - fp.register("conan user", stdout="user_name") - - # ACT - with patch.dict( - os.environ, - {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, - ): - conan_add_user(REPO_NAME) - - # ASSERT - assert "Unable to add user" in caplog.text - - -def test_conan_add_remote_already_exists( - config: Config, - caplog: pytest.LogCaptureFixture, - fp: FakeProcess, -) -> None: - # ARRANGE - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) - error = f"ERROR: Remote '{REMOTE_REPO_NAME_2}' already exists in remotes (use --force to continue)" - fp.register( - ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], - callback=conan_remote_add_error, - callback_kwargs={"stderr": error}, - ) - - # ACT - conan_add_remote(Path("some_path"), config) - - # ASSERT - assert "Unable to add remote repository" not in caplog.text From 79ecdba197d7e705bcf588600870d3c3735e09d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Fri, 27 Oct 2023 14:27:18 +0200 Subject: [PATCH 19/28] Changed import of conan_add_remote and conan_source in publish.py --- scargo/commands/publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scargo/commands/publish.py b/scargo/commands/publish.py index 3f70c456..f8449c58 100644 --- a/scargo/commands/publish.py +++ b/scargo/commands/publish.py @@ -8,7 +8,7 @@ import sys from pathlib import Path -from scargo.commands.test import conan_add_remote, conan_source +from scargo.conan_utils import conan_add_remote, conan_source from scargo.config_utils import prepare_config from scargo.logger import get_logger From 2388734e7f4c8f604a94cece72e9885ee5ff1abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B6lzl?= Date: Tue, 31 Oct 2023 10:36:53 +0100 Subject: [PATCH 20/28] Renamed ut\conan_utils.py to ut\ut_conan_utils.py --- tests/ut/{conan_utils.py => ut_conan_utils.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/ut/{conan_utils.py => ut_conan_utils.py} (100%) diff --git a/tests/ut/conan_utils.py b/tests/ut/ut_conan_utils.py similarity index 100% rename from tests/ut/conan_utils.py rename to tests/ut/ut_conan_utils.py From 849e304c0026129f4cd2b34d6f45bbce077965c1 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Tue, 31 Oct 2023 14:20:52 +0100 Subject: [PATCH 21/28] Sync mypy precommit hook with CI execution (#386) --- .github/workflows/workflow-common.yml | 2 +- .pre-commit-config.yaml | 23 ++++++----------------- ci/requirements.txt | 2 +- common_dev/scripts/cyclomatic.py | 2 +- pyproject.toml | 17 ++++------------- scargo/config.py | 15 +++++++-------- tests/it/it_scargo_commands_flow.py | 2 +- tests/ut/conftest.py | 7 ++++--- 8 files changed, 25 insertions(+), 45 deletions(-) diff --git a/.github/workflows/workflow-common.yml b/.github/workflows/workflow-common.yml index 6d7f637f..065377c9 100644 --- a/.github/workflows/workflow-common.yml +++ b/.github/workflows/workflow-common.yml @@ -63,7 +63,7 @@ jobs: - name: Run pylint run: pylint scargo run.py clean.py - name: Check typing with mypy - run: mypy --explicit-package-bases scargo tests common_dev run.py clean.py + run: mypy scargo tests common_dev run.py clean.py unit_tests: runs-on: ubuntu-latest needs: prepare_matrix diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da76dff4..69156b76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,26 +47,15 @@ repos: rev: v3.0.2 hooks: - id: pylint + language: system - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.0.1 + rev: v1.6.1 hooks: - id: mypy - args: [--explicit-package-bases] - additional_dependencies: [ - click, - jinja2, - matplotlib, - pydantic, - pytest, - pytest_subprocess, - pytest-mock, - python-dotenv, - tomlkit, - typer, - types-clang==0.14.3, - types-toml, - types-openssl-python, - ] + exclude: conanfile.py + args: + - --config-file=pyproject.toml + language: system # sets up .pre-commit-ci.yaml to ensure pre-commit dependencies stay up to date ci: diff --git a/ci/requirements.txt b/ci/requirements.txt index 3d39f0ae..f3ea9818 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -134,7 +134,7 @@ mccabe==0.7.0 # via # flake8 # pylint -mypy==1.0.1 +mypy==1.6.1 # via scargo (pyproject.toml) mypy-extensions==1.0.0 # via diff --git a/common_dev/scripts/cyclomatic.py b/common_dev/scripts/cyclomatic.py index 2587a2db..d6dff722 100755 --- a/common_dev/scripts/cyclomatic.py +++ b/common_dev/scripts/cyclomatic.py @@ -3,7 +3,7 @@ # @copyright Copyright (C) 2022 SpyroSoft Solutions S.A. All rights reserved. # # -import lizard # type: ignore[import] +import lizard # type: ignore[import-untyped] # to exclude directory use -x param params = ["lizard", "/scargo/", "-C", "25", "-w"] diff --git a/pyproject.toml b/pyproject.toml index 055a45c4..94761fe3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ dev = [ "gcovr>=5.2", "isort==5.11.4", "matplotlib", - "mypy==1.0.1", + "mypy==1.6.1", "pre-commit", "pyclean==2.2.0", "pyelftools", @@ -109,18 +109,6 @@ profile = "black" ignore-paths = ["example_project", "tests"] max-line-length = 120 output-format = "colorized" -ignored-modules = [ - "clang.cindex", - "clang", - "coloredlogs", - "docker", - "jinja2", - "matplotlib.pyplot", - "pydantic", - "toml", - "typer", - "yaml", -] [tool.pylint.messages_control] disable = [ @@ -143,6 +131,9 @@ strict = true disallow_any_unimported = true no_implicit_optional = true show_error_codes = true +explicit_package_bases = true +scripts_are_modules = false +exclude = ["conanfile.py"] [[tool.mypy.overrides]] module = "coloredlogs" diff --git a/scargo/config.py b/scargo/config.py index 60be0897..2c2915c5 100644 --- a/scargo/config.py +++ b/scargo/config.py @@ -102,7 +102,7 @@ class ProjectConfig(BaseModel): name: str version: str description: Optional[str] - homepage_url: Optional[str] = Field(None, alias="homepage-url") + homepage_url: Optional[str] = Field(default=None, alias="homepage-url") bin_name: Optional[str] lib_name: Optional[str] @@ -119,7 +119,7 @@ class ProjectConfig(BaseModel): cflags: str cxxflags: str - max_build_jobs: Optional[int] = Field(None, alias="max-build-jobs") + max_build_jobs: Optional[int] = Field(default=None, alias="max-build-jobs") cmake_variables: Dict[str, str] = Field( default_factory=dict, alias="cmake-variables" @@ -168,7 +168,6 @@ def get_target_by_id(cls, target_id: str) -> "Target": ), } - # for typer ScargoTargets = Enum( # type: ignore[misc] "ScargoTargets", {target.id: target.id for target in TARGETS.values()} @@ -180,7 +179,7 @@ class ProfileConfig(BaseModel, extra=Extra.allow): cxxflags: Optional[str] cc: Optional[str] = None cxx: Optional[str] = None - cmake_build_type: Optional[str] = Field(None, alias="cmake-build-type") + cmake_build_type: Optional[str] = Field(default=None, alias="cmake-build-type") @property def extras(self) -> Dict[str, Any]: @@ -202,7 +201,7 @@ class ChecksConfig(BaseModel): class CheckConfig(BaseModel): - description: Optional[str] + description: Optional[str] = None exclude: List[str] = Field(default_factory=list) @@ -264,10 +263,10 @@ class Esp32Config(BaseModel): class ScargoConfig(BaseModel): - console_log_level: str = Field("INFO", alias="console-log-level") - file_log_level: str = Field("WARNING", alias="file-log-level") + console_log_level: str = Field(default="INFO", alias="console-log-level") + file_log_level: str = Field(default="WARNING", alias="file-log-level") update_exclude: List[str] = Field(alias="update-exclude", default_factory=list) - version: Optional[str] + version: Optional[str] = None class DockerComposeConfig(BaseModel): diff --git a/tests/it/it_scargo_commands_flow.py b/tests/it/it_scargo_commands_flow.py index 24ad4f54..f14963c7 100644 --- a/tests/it/it_scargo_commands_flow.py +++ b/tests/it/it_scargo_commands_flow.py @@ -232,7 +232,7 @@ def test_cli_help( """Simple test which checks if scargo new -h command can be invoked and do not return any error""" # create temporary dir for new project test_state.proj_path = tmpdir_factory.mktemp(test_state.proj_name) - os.chdir(test_state.proj_path) + os.chdir(test_state.proj_path) # type: ignore # New Help result = test_state.runner.invoke(cli, ["new", "-h"]) diff --git a/tests/ut/conftest.py b/tests/ut/conftest.py index 6256fb0a..e15c0117 100644 --- a/tests/ut/conftest.py +++ b/tests/ut/conftest.py @@ -44,7 +44,7 @@ def config(fs: FakeFilesystem) -> Config: # type: ignore[no-any-unimported] return Config( project=ProjectConfig( - **{ + **{ # type: ignore "name": "test_project", "version": "0.1.0", "description": "Project description.", @@ -72,7 +72,7 @@ def config(fs: FakeFilesystem) -> Config: # type: ignore[no-any-unimported] "MinSizeRel": ProfileConfig(cflags="-Os -DNDEBUG", cxxflags="-Os -DNDEBUG"), }, check=ChecksConfig( - **{ + **{ # type: ignore "exclude": [], "pragma": CheckConfig(description=None, exclude=[]), "copyright": CheckConfig(description="Copyright", exclude=[]), @@ -98,9 +98,10 @@ def config(fs: FakeFilesystem) -> Config: # type: ignore[no-any-unimported] general=[], build=[], tool=[], test=["gtest/cci.20210126"] ), conan=ConanConfig(repo={}), + atsam=None, stm32=None, esp32=None, - scargo=ScargoConfig( + scargo=ScargoConfig( # type: ignore console_log_level="INFO", file_log_level="WARNING", update_exclude=[], From b68fdbb9b95b0a21eab2c5e3d0bf0370ec847013 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:17:11 +0100 Subject: [PATCH 22/28] Scargo gen unit tests using mocks (#388) --- scargo/commands/gen.py | 104 +-- scargo/target_helpers/esp32_helper.py | 97 ++ tests/ut/data/scargo_test_esp32.toml | 107 +++ ...{scargo_test.toml => scargo_test_x86.toml} | 0 tests/ut/target_helpers/ut_esp32_helper.py | 160 ++++ tests/ut/ut_scargo_gen.py | 829 +++++------------- tests/ut/utils.py | 8 +- 7 files changed, 605 insertions(+), 700 deletions(-) create mode 100644 scargo/target_helpers/esp32_helper.py create mode 100644 tests/ut/data/scargo_test_esp32.toml rename tests/ut/data/{scargo_test.toml => scargo_test_x86.toml} (100%) create mode 100644 tests/ut/target_helpers/ut_esp32_helper.py diff --git a/scargo/commands/gen.py b/scargo/commands/gen.py index bc6116d5..76becf07 100644 --- a/scargo/commands/gen.py +++ b/scargo/commands/gen.py @@ -2,9 +2,6 @@ # @copyright Copyright (C) 2023 SpyroSoft Solutions S.A. All rights reserved. # # -import os -import re -import shutil import subprocess import sys from pathlib import Path @@ -17,11 +14,14 @@ from scargo.file_generators.ut_gen import generate_ut from scargo.global_values import SCARGO_PKG_PATH from scargo.logger import get_logger +from scargo.target_helpers.esp32_helper import ( + OUT_FS_DIR, + gen_fs_esp32, + gen_single_binary_esp32, +) logger = get_logger() -OUT_FS_DIR = Path("build", "fs") - def scargo_gen( profile: str, @@ -57,8 +57,7 @@ def scargo_gen( generate_fs(config) if single_bin: - project_profile_path = config.project_root / "build" / profile - gen_single_binary(project_profile_path, config) + gen_single_binary(profile, config) def generate_certs( @@ -76,9 +75,7 @@ def generate_certs( if not certs_passwd: certs_passwd = "1234" - if mode_for_certs == "all": - mode_for_certs = "All-certificates" - elif mode_for_certs == "device": + if mode_for_certs == "device": mode_for_certs = "Device-certificate" else: mode_for_certs = "All-certificates" @@ -135,92 +132,9 @@ def generate_fs(config: Config) -> None: logger.warning("Gen --fs command not supported for this target yet.") -def gen_single_binary(project_profile_path: Path, config: Config) -> None: +def gen_single_binary(build_profile: str, config: Config) -> None: target = config.project.target if target.family == "esp32": - gen_single_binary_esp32(project_profile_path, config) + gen_single_binary_esp32(build_profile, config) else: logger.warning("Gen --bin command not supported for this target yet.") - - -def gen_fs_esp32(config: Config) -> None: - command = [] - partition_list = config.get_esp32_config().partitions - fs_size = 0 - for i in partition_list: - split_list = i.split(",") - if "spiffs" in split_list[0]: - fs_size = int(re.sub(",", "", split_list[4]), 16) - - try: - project_path = config.project_root - fs_in_dir = project_path / "main/fs" - fs_out_dir = project_path / OUT_FS_DIR - fs_out_bin = project_path / "build/spiffs.bin" - fs_in_dir.mkdir(parents=True, exist_ok=True) - fs_out_dir.mkdir(parents=True, exist_ok=True) - - shutil.copytree(fs_in_dir, fs_out_dir, dirs_exist_ok=True) - - idf_path = os.environ.get("IDF_PATH") - command = [ - f"{idf_path}/components/spiffs/spiffsgen.py", - str(fs_size), - str(fs_out_dir), - str(fs_out_bin), - ] - - subprocess.check_call(command, cwd=project_path) - logger.info("Generated %s of size:%s", fs_out_bin, fs_size) - - except subprocess.CalledProcessError: - logger.error("%s fail", command) - sys.exit(1) - - -def gen_single_binary_esp32(project_profile_path: Path, config: Config) -> None: - partition_list = config.get_esp32_config().partitions - chip = config.get_esp32_config().chip - - flasher_args_path = project_profile_path / "flash_args" - if not flasher_args_path.is_file(): - logger.warning("%s does not exists", flasher_args_path) - sys.exit(1) - - line_list = flasher_args_path.read_text().split() - - spiffs_addr = None - for i in partition_list: - if "spiffs" in i: - split_list = i.split(",") - spiffs_addr = split_list[3].strip() - break - - flash_size = "4MB" - for index, arg in enumerate(line_list): - if "flash_size" in arg: - flash_size = line_list[index + 1] - if arg.endswith(".bin"): - line_list[index] = str(project_profile_path / arg) - - command = [ - "esptool.py", - "--chip", - chip, - "merge_bin", - "-o", - "build/flash_image.bin", - "--fill-flash-size", - f"{flash_size}", - ] - command.extend(line_list) - - if spiffs_addr: - command.extend([spiffs_addr, "build/spiffs.bin"]) - - try: - logger.info("Running: %s", " ".join(command)) - subprocess.check_call(command, cwd=config.project_root) - except subprocess.CalledProcessError: - logger.error("Generation of single binary failed") - sys.exit(1) diff --git a/scargo/target_helpers/esp32_helper.py b/scargo/target_helpers/esp32_helper.py new file mode 100644 index 00000000..ddc937ab --- /dev/null +++ b/scargo/target_helpers/esp32_helper.py @@ -0,0 +1,97 @@ +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +from scargo.config import Config +from scargo.logger import get_logger + +logger = get_logger() +OUT_FS_DIR = Path("build", "fs") + + +def gen_fs_esp32(config: Config) -> None: + command = [] + partition_list = config.get_esp32_config().partitions + fs_size = 0 + for i in partition_list: + split_list = i.split(",") + if "spiffs" in split_list[0]: + fs_size = int(re.sub(",", "", split_list[4]), 16) + + try: + project_path = config.project_root + fs_in_dir = project_path / "main/fs" + fs_out_dir = project_path / OUT_FS_DIR + fs_out_bin = project_path / "build/spiffs.bin" + fs_in_dir.mkdir(parents=True, exist_ok=True) + fs_out_dir.mkdir(parents=True, exist_ok=True) + + shutil.copytree(fs_in_dir, fs_out_dir, dirs_exist_ok=True) + + idf_path = os.environ.get("IDF_PATH") + command = [ + f"{idf_path}/components/spiffs/spiffsgen.py", + str(fs_size), + str(fs_out_dir), + str(fs_out_bin), + ] + + subprocess.run(command, cwd=project_path, check=True) + logger.info("Generated %s of size:%s", fs_out_bin, fs_size) + + except subprocess.CalledProcessError: + logger.error("%s fail", command) + sys.exit(1) + + +def gen_single_binary_esp32(build_profile: str, config: Config) -> None: + partition_list = config.get_esp32_config().partitions + chip = config.get_esp32_config().chip + + build_dir = config.project_root / "build" / build_profile + flasher_args_path = build_dir / "flash_args" + if not flasher_args_path.is_file(): + logger.warning("%s does not exists", flasher_args_path) + logger.info("Did you run scargo build --profile %s", build_profile) + sys.exit(1) + + line_list = flasher_args_path.read_text().split() + + spiffs_addr = None + for i in partition_list: + if "spiffs" in i: + split_list = i.split(",") + spiffs_addr = split_list[3].strip() + break + + flash_size = "4MB" + for index, arg in enumerate(line_list): + if "flash_size" in arg: + flash_size = line_list[index + 1] + if arg.endswith(".bin"): + line_list[index] = str(build_dir / arg) + + command = [ + "esptool.py", + "--chip", + chip, + "merge_bin", + "-o", + "build/flash_image.bin", + "--fill-flash-size", + f"{flash_size}", + ] + command.extend(line_list) + + if spiffs_addr: + command.extend([spiffs_addr, "build/spiffs.bin"]) + + try: + logger.info("Running: %s", " ".join(command)) + subprocess.run(command, cwd=config.project_root, check=True) + except subprocess.CalledProcessError: + logger.error("Generation of single binary failed") + sys.exit(1) diff --git a/tests/ut/data/scargo_test_esp32.toml b/tests/ut/data/scargo_test_esp32.toml new file mode 100644 index 00000000..b3588637 --- /dev/null +++ b/tests/ut/data/scargo_test_esp32.toml @@ -0,0 +1,107 @@ +[project] +name = "esp32proj" +version = "0.1.0" +description = "Project description." +homepage-url = "www.hello-world.com" + +bin_name = "esp32proj" +target = "esp32" +build-env = "docker" +docker-file = ".devcontainer/Dockerfile-custom" +docker-image-tag = "esp32proj-dev:1.0" + +cxxstandard = "17" + +cflags = "-Wall -Wextra" +cxxflags = "-Wall -Wextra" + +in-repo-conan-cache = false + +[profile.Debug] +cflags = "-g" +cxxflags = "-g" + +[profile.Release] +cflags = "-O3 -DNDEBUG" +cxxflags = "-O3 -DNDEBUG" + +[profile.RelWithDebInfo] +cflags = "-O2 -g -DNDEBUG" +cxxflags = "-O2 -g -DNDEBUG" + +[profile.MinSizeRel] +cflags = "-Os -DNDEBUG" +cxxflags = "-Os -DNDEBUG" + +[check] +exclude = [] + +[check.pragma] +exclude = [] + +[check.copyright] +description = "Copyright" +exclude = [] + +[check.todo] +keywords = ["tbd", "todo", "TODO", "fixme"] +exclude = [] + +[check.clang-format] +exclude = [] + +[check.clang-tidy] +exclude = [] + +[check.cyclomatic] +exclude = [] + +[tests] +cc = "gcc" +cxx = "g++" + +cflags = "-Wall -Wextra -Og --coverage -fkeep-inline-functions -fkeep-static-consts" +cxxflags = "-Wall -Wextra -Og --coverage -fkeep-inline-functions -fkeep-static-consts" + +gcov-executable = "" # Empty string -> use default gcov executable + +# Underhood scargo use conan. All string valid for conan tool are valid here. eg "gtest/1.13.0" +[dependencies] +#general -> public conan dependencies of project they will be added to package info (eg. bianry dynamic linkage libary) they will be added also to scargo test. +general = [ +] +#build -> private conan dependencies usedn only during build process(eg. private static linkage library) +build = [ +] +#tool -> special conan "not library" dependencies like cmake/3.22. +tool = [ +] +#test-> conan dependencies used only for testing targets +test = [ + "gtest/1.13.0" +] + +[conan.repo] +#Passing conancenter here is not nessary as scargo adds it by default +#Below example of private binary artifacts repository +#gitlab = "https://git.my_extra_gitlab_server.com" + +[esp32] +chip = "esp32" +extra_component_dirs=[] +partitions = [ + "nvs, data, nvs, 0x9000, 0x4000,", + "otadata, data, ota, 0xd000, 0x2000,", + "phy_init, data, phy, 0xf000, 0x1000,", + "ota_0, app, ota_0, 0x10000, 0x180000,", + "ota_1, app, ota_1, 0x190000,0x180000,", + "spiffs, data, spiffs, 0x310000,0x6000," +] + +[scargo] +console-log-level = "INFO" +file-log-level = "WARNING" +update-exclude = [] + +[docker-compose] +ports = [] \ No newline at end of file diff --git a/tests/ut/data/scargo_test.toml b/tests/ut/data/scargo_test_x86.toml similarity index 100% rename from tests/ut/data/scargo_test.toml rename to tests/ut/data/scargo_test_x86.toml diff --git a/tests/ut/target_helpers/ut_esp32_helper.py b/tests/ut/target_helpers/ut_esp32_helper.py new file mode 100644 index 00000000..30e0f267 --- /dev/null +++ b/tests/ut/target_helpers/ut_esp32_helper.py @@ -0,0 +1,160 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from pytest_subprocess import FakeProcess + +from scargo.target_helpers.esp32_helper import gen_fs_esp32, gen_single_binary_esp32 +from tests.ut.ut_scargo_gen import mock_prepare_config_esp32 # noqa: F401 + + +def test_gen_fs_esp32( + fp: FakeProcess, mock_prepare_config_esp32: MagicMock # noqa: F811 +) -> None: + project_dir = Path.cwd() + + fp.register( + [ + "idf_path/components/spiffs/spiffsgen.py", + "24576", + f"{project_dir}/build/fs", + f"{project_dir}/build/spiffs.bin", + ] + ) + + gen_fs_esp32(mock_prepare_config_esp32.return_value) + + +def test_gen_fs_esp32_fails( + fp: FakeProcess, mock_prepare_config_esp32: MagicMock # noqa: F811 +) -> None: + project_dir = Path.cwd() + + fp.register( + [ + "idf_path/components/spiffs/spiffsgen.py", + "24576", + f"{project_dir}/build/fs", + f"{project_dir}/build/spiffs.bin", + ], + returncode=1, + ) + + with pytest.raises(SystemExit): + gen_fs_esp32(mock_prepare_config_esp32.return_value) + + +def test_gen_single_bin_esp32_flash_args_not_exists( + caplog: pytest.LogCaptureFixture, + mock_prepare_config_esp32: MagicMock, # noqa: F811 +) -> None: + with pytest.raises(SystemExit) as e: + gen_single_binary_esp32("Debug", mock_prepare_config_esp32.return_value) + + assert "flash_args does not exists" in caplog.text + assert "Did you run scargo build --profile Debug" in caplog.text + assert e.type == SystemExit + + +def test_gen_single_bin_esp32_esptool_fails( + fp: FakeProcess, + mock_prepare_config_esp32: MagicMock, # noqa: F811 + caplog: pytest.LogCaptureFixture, +) -> None: + build_profile = "Debug" + flash_args_path = Path(f"build/{build_profile}/flash_args") + flash_args_path.parent.mkdir(parents=True) + flash_args_path.touch() + + fp.register( + [ + "esptool.py", + "--chip", + "esp32", + "merge_bin", + "-o", + "build/flash_image.bin", + "--fill-flash-size", + "4MB", + "0x310000", + "build/spiffs.bin", + ], + returncode=1, + ) + + with pytest.raises(SystemExit): + gen_single_binary_esp32("Debug", mock_prepare_config_esp32.return_value) + assert "Generation of single binary failed" in caplog.text + + +def test_gen_single_bin_esp32_default( + fp: FakeProcess, + mock_prepare_config_esp32: MagicMock, # noqa: F811 +) -> None: + build_profile = "Debug" + flash_args_path = Path(f"build/{build_profile}/flash_args") + flash_args_path.parent.mkdir(parents=True) + flash_args_path.touch() + + fp.register( + [ + "esptool.py", + "--chip", + "esp32", + "merge_bin", + "-o", + "build/flash_image.bin", + "--fill-flash-size", + "4MB", + "0x310000", + "build/spiffs.bin", + ] + ) + + gen_single_binary_esp32("Debug", mock_prepare_config_esp32.return_value) + + +def test_gen_single_bin_esp32_flash_args( + fp: FakeProcess, + mock_prepare_config_esp32: MagicMock, # noqa: F811 +) -> None: + build_profile = "Debug" + flash_args_content = """ +--flash_mode dio --flash_freq 40m --flash_size 2MB +0x1000 bootloader/bootloader.bin +0x10000 app_esp32.bin +0x8000 partition_table/partition-table.bin +""" + build_dir = Path(f"build/{build_profile}").absolute() + flash_args_path = build_dir / "flash_args" + flash_args_path.parent.mkdir(parents=True) + flash_args_path.write_text(flash_args_content) + + fp.register( + [ + "esptool.py", + "--chip", + "esp32", + "merge_bin", + "-o", + "build/flash_image.bin", + "--fill-flash-size", + "2MB", + "--flash_mode", + "dio", + "--flash_freq", + "40m", + "--flash_size", + "2MB", + "0x1000", + f"{build_dir}/bootloader/bootloader.bin", + "0x10000", + f"{build_dir}/app_esp32.bin", + "0x8000", + f"{build_dir}/partition_table/partition-table.bin", + "0x310000", + "build/spiffs.bin", + ] + ) + + gen_single_binary_esp32("Debug", mock_prepare_config_esp32.return_value) diff --git a/tests/ut/ut_scargo_gen.py b/tests/ut/ut_scargo_gen.py index f2e5f206..8bc2087d 100644 --- a/tests/ut/ut_scargo_gen.py +++ b/tests/ut/ut_scargo_gen.py @@ -1,108 +1,78 @@ import os -import shutil +from dataclasses import dataclass from pathlib import Path -from typing import Dict +from typing import Optional +from unittest.mock import MagicMock import pytest -from clang import native # type: ignore[attr-defined] -from clang.cindex import Config as ClangConfig -from OpenSSL import crypto from pytest_mock import MockerFixture +from pytest_subprocess import FakeProcess, fake_popen from scargo.commands.gen import scargo_gen -from scargo.commands.update import scargo_update -from scargo.config_utils import get_scargo_config_or_exit +from tests.ut.utils import get_test_project_config -NAME_OF_LIB_HPP_FILE = "lib_hpp.hpp" -NAME_OF_TEST_TXT_FILE = "test_dummy_file.txt" -NAME_OF_TEST_NO_EXTENSION_FILE = "test_dummy_no_ext_file" -NESTED_LIB_FILE_LOCATION = "libs/test_lib/" -TEST_DEVICE_ID = "com.example.my.solution:gw-01:da:device:ZWave:CA0D6357%2F4" -TEST_DEVICE_ID_2 = "com.example.my.solution:gw-01:da:device:ZWave:AAAA1111%1A1" -TEST_DEVICE_ID_3 = "com.example.my.solution:gw-01:da:device:ZWave:BBBB2222%2B2" +@pytest.fixture +def mock_prepare_config(tmpdir: Path, mocker: MockerFixture) -> MagicMock: + os.chdir(tmpdir) -CERTS_DIR = "build/certs" -FS_DIR = "build/fs" + x86_config = get_test_project_config("x86") + x86_config.project_root = Path(tmpdir) -CUSTOM_CERTS_DIR = "custom_certs" + return mocker.patch( + f"{scargo_gen.__module__}.prepare_config", return_value=x86_config + ) -@pytest.fixture(autouse=True, scope="session") -def clang_config() -> None: - ClangConfig().set_library_file(str(Path(native.__file__).with_name("libclang.so"))) +@pytest.fixture +def mock_prepare_config_esp32(tmpdir: Path, mocker: MockerFixture) -> MagicMock: + os.environ["IDF_PATH"] = "idf_path" + os.chdir(tmpdir) + esp_config = get_test_project_config("esp32") + esp_config.project_root = Path(tmpdir) -def project_add_nested_hpp_test_file(project_src_path: Path) -> None: - """This function adding to project src directory some dummy .hpp file in nested directory for stronger tests""" - os.makedirs(str(project_src_path / NESTED_LIB_FILE_LOCATION), exist_ok=True) - with open( - str(project_src_path / NESTED_LIB_FILE_LOCATION / NAME_OF_LIB_HPP_FILE), "w" - ) as fp: - fp.write("//THIS IS TMP TEST HPP FILE") + return mocker.patch( + f"{scargo_gen.__module__}.prepare_config", return_value=esp_config + ) -def project_add_extra_test_files(project_src_path: Path) -> None: - """This function adding to project src directory some .txt file and file without extension for negative tests""" - os.makedirs(str(project_src_path / NESTED_LIB_FILE_LOCATION), exist_ok=True) - with open(str(project_src_path / NAME_OF_TEST_TXT_FILE), "w") as fp: - fp.write("//THIS IS TMP TEST TXT FILE") - with open(str(project_src_path / NAME_OF_TEST_NO_EXTENSION_FILE), "w") as fp: - fp.write("//THIS IS TMP TEST NO EXTENSION FILE") +# -------------- Gen unit_tests tests -------------- +def test_gen_ut(mock_prepare_config: MagicMock, mocker: MockerFixture) -> None: + # This test just checks that generator is called + # Unit tests for unit test generator should be in seperate file + mocked_generate_ut = mocker.patch(f"{scargo_gen.__module__}.generate_ut") + path_to_srcs = Path("path_not_none") -@pytest.mark.parametrize( - "test_project_data", - [ - {"proj_path": "copy_test_project", "src_files_dir": "src"}, - {"proj_path": "copy_test_project_esp32", "src_files_dir": "main"}, - {"proj_path": "copy_test_project_stm32", "src_files_dir": "src"}, - {"proj_path": "new_project_x86", "src_files_dir": "src"}, - {"proj_path": "new_project_esp32", "src_files_dir": "main"}, - {"proj_path": "new_project_stm32", "src_files_dir": "src"}, - ], - ids=[ - "copied_proj", - "copied_esp32_proj", - "copied_stm32_proj", - "new_project_x86", - "new_project_esp32", - "new_project_stm32", - ], - scope="class", -) -class TestGenMockPositive: - """This class collects all unit tests for scargo_gen command with gen_mock option""" - - def test_gen_mock_for_simple_h_file( - self, - request: pytest.FixtureRequest, - test_project_data: Dict[str, str], - mocker: MockerFixture, - ) -> None: - """ - This test check if for .h located in src dictionary mock files will be created under tests/mocks dictionary - as a result of scargo gen command. - """ - proj_path = request.getfixturevalue(test_project_data["proj_path"]) - os.chdir(proj_path) - path_to_h_file = Path( - os.getcwd(), test_project_data["src_files_dir"], "test_lib.h" - ) + scargo_gen( + profile="Debug", + gen_ut=path_to_srcs, + gen_mock=None, + certs=None, + certs_mode=None, + certs_input=None, + certs_passwd=None, + fs=False, + single_bin=False, + ) - # make sure that files which needs to be generated do not exist - precondition - assert not os.path.exists("tests/mocks/test_lib.h") - assert not os.path.exists("tests/mocks/mock_test_lib.h") - assert not os.path.exists("tests/mocks/mock_test_lib.cpp") + mocked_generate_ut.assert_called_once_with( + path_to_srcs, mock_prepare_config.return_value + ) - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), - ) + +# -------------- Gen mocks tests -------------- +def test_gen_mocknot_header( + mock_prepare_config: MagicMock, caplog: pytest.LogCaptureFixture +) -> None: + some_path = Path("not_header_file") + + with pytest.raises(SystemExit): scargo_gen( profile="Debug", gen_ut=None, - gen_mock=path_to_h_file, + gen_mock=some_path, certs=None, certs_mode=None, certs_input=None, @@ -111,383 +81,88 @@ def test_gen_mock_for_simple_h_file( single_bin=False, ) - # test if expected files was generated - assert os.path.exists("tests/mocks/test_lib.h") - assert os.path.exists("tests/mocks/mock_test_lib.h") - assert os.path.exists("tests/mocks/mock_test_lib.cpp") - - def test_gen_mock_for_nested_h_file( - self, - request: pytest.FixtureRequest, - test_project_data: Dict[str, str], - mocker: MockerFixture, - ) -> None: - """ - This test check if for .hpp located in nested directory not just under src dictionary mock files will be created - under path mirroring the path to sources in tests/mocks dictionary as a result of scargo gen command. - """ - proj_path = request.getfixturevalue(test_project_data["proj_path"]) - os.chdir(proj_path) - project_add_nested_hpp_test_file(proj_path / test_project_data["src_files_dir"]) - path_to_nested_hpp_file = Path( - os.getcwd(), - test_project_data["src_files_dir"], - NESTED_LIB_FILE_LOCATION, - NAME_OF_LIB_HPP_FILE, - ) + assert "Not a header file. Please chose .h or .hpp file." in caplog.text - # make sure that files which needs to be generated do not exist - assert not os.path.exists( - f"tests/mocks/{NESTED_LIB_FILE_LOCATION}/{NAME_OF_LIB_HPP_FILE}" - ) - assert not os.path.exists( - f"tests/mocks/{NESTED_LIB_FILE_LOCATION}/mock_{NAME_OF_LIB_HPP_FILE}" - ) - assert not os.path.exists( - f"tests/mocks/{NESTED_LIB_FILE_LOCATION}/mock_{NAME_OF_LIB_HPP_FILE}".replace( - ".hpp", ".cpp" - ) - ) - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), - ) - scargo_gen( - profile="Debug", - gen_ut=None, - gen_mock=path_to_nested_hpp_file, - certs=None, - certs_mode=None, - certs_input=None, - certs_passwd=None, - fs=False, - single_bin=False, - ) +def test_gen_mock( + mock_prepare_config: MagicMock, + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, +) -> None: + # This test just checks that generator is called + # Unit tests for mock generator should be in seperate file - # test if expected files was generated - assert os.path.exists( - f"tests/mocks/{NESTED_LIB_FILE_LOCATION}/{NAME_OF_LIB_HPP_FILE}" - ) - assert os.path.exists( - f"tests/mocks/{NESTED_LIB_FILE_LOCATION}/mock_{NAME_OF_LIB_HPP_FILE}" - ) - assert os.path.exists( - f"tests/mocks/{NESTED_LIB_FILE_LOCATION}/mock_{NAME_OF_LIB_HPP_FILE}".replace( - ".hpp", ".cpp" - ) - ) + mocked_generate_ut = mocker.patch(f"{scargo_gen.__module__}.generate_mocks") + path_to_hdr = Path("header.h") + + scargo_gen( + profile="Debug", + gen_ut=None, + gen_mock=path_to_hdr, + certs=None, + certs_mode=None, + certs_input=None, + certs_passwd=None, + fs=False, + single_bin=False, + ) + mocked_generate_ut.assert_called_once_with( + path_to_hdr, mock_prepare_config.return_value + ) + assert f"Generated: {path_to_hdr}" in caplog.text -@pytest.mark.parametrize( - "file", - ["test_lib.cpp", NAME_OF_TEST_TXT_FILE, NAME_OF_TEST_NO_EXTENSION_FILE], - ids=["cpp file", "txt file", "file_with_no_extension"], -) -def test_gen_mock_for_unexpected_extension_file( - copy_test_project_stm32: Path, + +def test_gen_mock_fails( + mock_prepare_config: MagicMock, mocker: MockerFixture, - file: str, + caplog: pytest.LogCaptureFixture, ) -> None: - """ - This test check if for .h located in src dictionary mock files will be created under tests/mocks dictionary - as a result of scargo gen command. - """ - os.chdir(copy_test_project_stm32) - project_add_extra_test_files(copy_test_project_stm32 / "src") - path_to_file = Path(os.getcwd(), "src", file) - # make sure that files which needs to be generated do not exist - precondition - assert not os.path.exists(f"tests/mocks/{file}") - - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), + mocked_generate_ut = mocker.patch( + f"{scargo_gen.__module__}.generate_mocks", return_value=False ) - with pytest.raises(SystemExit) as scargo_gen_sys_exit: - scargo_gen( - profile="Debug", - gen_ut=None, - gen_mock=path_to_file, - certs=None, - certs_mode=None, - certs_input=None, - certs_passwd=None, - fs=False, - single_bin=False, - ) - assert scargo_gen_sys_exit.type == SystemExit - assert scargo_gen_sys_exit.value.code == 1 + path_to_hdr = Path("header.h") - # test if expected files was generated - assert not os.path.exists(f"tests/mocks/{file}") + scargo_gen( + profile="Debug", + gen_ut=None, + gen_mock=path_to_hdr, + certs=None, + certs_mode=None, + certs_input=None, + certs_passwd=None, + fs=False, + single_bin=False, + ) + mocked_generate_ut.assert_called_once_with( + path_to_hdr, mock_prepare_config.return_value + ) + assert f"Skipping: {path_to_hdr}" in caplog.text -@pytest.mark.parametrize( - "test_project_data", - [ - {"proj_path": "copy_test_project", "src_files_dir": "src"}, - {"proj_path": "copy_test_project_esp32", "src_files_dir": "main"}, - {"proj_path": "copy_test_project_stm32", "src_files_dir": "src"}, - {"proj_path": "new_project_x86", "src_files_dir": "src"}, - {"proj_path": "new_project_esp32", "src_files_dir": "main"}, - {"proj_path": "new_project_stm32", "src_files_dir": "src"}, - ], - ids=[ - "copied_proj", - "copied_esp32_proj", - "copied_stm32_proj", - "new_project_x86", - "new_project_esp32", - "new_project_stm32", - ], - scope="class", -) -class TestGenCerts: - """This class collects all unit tests for scargo_gen command with certs option""" - - def test_gen_certs_simple( - self, - request: pytest.FixtureRequest, - test_project_data: Dict[str, str], - mocker: MockerFixture, - ) -> None: - """This test simply check if command gen certs create certs files""" - proj_path = request.getfixturevalue(test_project_data["proj_path"]) - os.chdir(proj_path) - - # remove directory with certs if already present - if os.path.exists(CERTS_DIR): - shutil.rmtree(CERTS_DIR) - if os.path.exists(FS_DIR): - shutil.rmtree(FS_DIR) - - assert not os.path.exists(CERTS_DIR) - assert not os.path.exists(FS_DIR) - - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), - ) - scargo_gen( - profile="Debug", - gen_ut=None, - gen_mock=None, - certs=TEST_DEVICE_ID, - certs_mode=None, - certs_input=None, - certs_passwd=None, - fs=False, - single_bin=False, - ) - # test if expected cert files was generated - pytest.assume(os.path.exists(FS_DIR)) # type: ignore - pytest.assume(os.path.exists(CERTS_DIR)) # type: ignore - - pytest.assume(os.path.exists(f"{FS_DIR}/ca.pem")) # type: ignore - pytest.assume(os.path.exists(f"{FS_DIR}/device_cert.pem")) # type: ignore - pytest.assume(os.path.exists(f"{FS_DIR}/device_priv_key.pem")) # type: ignore - - pytest.assume(os.path.exists(f"{CERTS_DIR}/azure")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/azure/{TEST_DEVICE_ID}-root-ca.pem")) # type: ignore - - # other files in certs dir - pytest.assume(os.path.exists(f"{CERTS_DIR}/ca.pem")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/digiroot.pem")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/index.txt")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/index.txt.attr")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/index.txt.attr.old")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/index.txt.old")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/serial")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/serial.old")) # type: ignore - # - pytest.assume(os.path.exists(f"{CERTS_DIR}/certs")) # type: ignore - pytest.assume( - os.path.exists(f"{CERTS_DIR}/certs/azure-iot-test-only.chain.ca.cert.pem") - ) # type: ignore - pytest.assume( - os.path.exists( - f"{CERTS_DIR}/certs/azure-iot-test-only.intermediate.cert.pem" - ) - ) # type: ignore - pytest.assume( - os.path.exists(f"{CERTS_DIR}/certs/azure-iot-test-only.root.ca.cert.pem") - ) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/certs/iot-device.cert.pem")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/certs/iot-device.cert.pfx")) # type: ignore - - pytest.assume(os.path.exists(f"{CERTS_DIR}/csr")) # type: ignore - pytest.assume( - os.path.exists(f"{CERTS_DIR}/csr/azure-iot-test-only.intermediate.csr.pem") - ) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/csr/iot-device.csr.pem")) # type: ignore - - pytest.assume(os.path.exists(f"{CERTS_DIR}/intermediateCerts")) # type: ignore - - pytest.assume(os.path.exists(f"{CERTS_DIR}/newcerts")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/newcerts/01.pem")) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/newcerts/02.pem")) # type: ignore - - pytest.assume(os.path.exists(f"{CERTS_DIR}/private")) # type: ignore - pytest.assume( - os.path.exists( - f"{CERTS_DIR}/private/azure-iot-test-only.intermediate.key.pem" - ) - ) # type: ignore - pytest.assume( - os.path.exists(f"{CERTS_DIR}/private/azure-iot-test-only.root.ca.key.pem") - ) # type: ignore - pytest.assume(os.path.exists(f"{CERTS_DIR}/private/iot-device.key.pem")) # type: ignore - - def test_gen_certs_mode_device( - self, - request: pytest.FixtureRequest, - test_project_data: Dict[str, str], - mocker: MockerFixture, - ) -> None: - """This test check if command gen certs with certs_mode='device' will generate device certs files for - new device id""" - proj_path = request.getfixturevalue(test_project_data["proj_path"]) - os.chdir(proj_path) - - # some files need to be present before we regenerate certs for device - if not os.path.exists(f"{CERTS_DIR}/index.txt"): - pytest.fail( - "Preconditions not meet, please generate all certificates as precondition" - ) - - # removing files which should be regenerated - # from build/certs dir - if os.path.exists(f"{CERTS_DIR}/ca.pem"): - os.remove(f"{CERTS_DIR}/ca.pem") - if os.path.exists(f"{CERTS_DIR}/certs/iot-device.cert.pem"): - os.remove(f"{CERTS_DIR}/certs/iot-device.cert.pem") - if os.path.exists(f"{CERTS_DIR}/private/iot-device.key.pem"): - os.remove(f"{CERTS_DIR}/private/iot-device.key.pem") - # from build/fs dir - if os.path.exists(f"{FS_DIR}/ca.pem"): - os.remove(f"{FS_DIR}/ca.pem") - if os.path.exists(f"{FS_DIR}/device_cert.pem"): - os.remove(f"{FS_DIR}/device_cert.pem") - if os.path.exists(f"{FS_DIR}/device_priv_key.pem"): - os.remove(f"{FS_DIR}/device_priv_key.pem") - - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), - ) - # generate keys for new device - scargo_gen( - profile="Debug", - gen_ut=None, - gen_mock=None, - certs=TEST_DEVICE_ID_2, - certs_mode="device", - certs_input=None, - certs_passwd=None, - fs=False, - single_bin=False, - ) - assert os.path.exists(f"{CERTS_DIR}/ca.pem") - assert os.path.exists(f"{CERTS_DIR}/certs/iot-device.cert.pem") - assert os.path.exists(f"{CERTS_DIR}/private/iot-device.key.pem") - assert os.path.exists(f"{CERTS_DIR}/certs/azure-iot-test-only.root.ca.cert.pem") - - assert os.path.exists(f"{CERTS_DIR}/azure/{TEST_DEVICE_ID_2}-root-ca.pem") - assert os.path.exists(f"{CERTS_DIR}/azure/{TEST_DEVICE_ID}-root-ca.pem") - - assert os.path.exists(f"{FS_DIR}/ca.pem") - assert os.path.exists(f"{FS_DIR}/device_cert.pem") - assert os.path.exists(f"{FS_DIR}/device_priv_key.pem") - - @pytest.mark.skip("Not ready yet") - def test_gen_certs_mode_device_custom_input_certs_dir( - self, - request: pytest.FixtureRequest, - test_project_data: Dict[str, str], - mocker: MockerFixture, - ) -> None: - """This test check if command gen certs with certs_mode='device' will generate device certs files for - new device id based on custom intermediate certs dir""" - proj_path = request.getfixturevalue(test_project_data["proj_path"]) - os.chdir(proj_path) - - # some files need to be present before we regenerate certs for device - if not os.path.exists(f"{CERTS_DIR}/index.txt"): - pytest.fail( - "Preconditions not meet, please generate all certificates as precondition" - ) - - # move generated intermediate certs to custom location - # os.mkdir(CUSTOM_CERTS_DIR) - # shutil.move(CERTS_DIR, CUSTOM_CERTS_DIR) - shutil.copytree(CERTS_DIR, CUSTOM_CERTS_DIR) - if os.path.exists(f"{CERTS_DIR}/certs"): - shutil.rmtree(f"{CERTS_DIR}/certs") - - # removing files which should be regenerated build/fs dir - if os.path.exists(f"{FS_DIR}/ca.pem"): - os.remove(f"{FS_DIR}/ca.pem") - if os.path.exists(f"{FS_DIR}/device_cert.pem"): - os.remove(f"{FS_DIR}/device_cert.pem") - if os.path.exists(f"{FS_DIR}/device_priv_key.pem"): - os.remove(f"{FS_DIR}/device_priv_key.pem") - - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), - ) - # generate keys for new device - scargo_gen( - profile="Debug", - gen_ut=None, - gen_mock=None, - certs=TEST_DEVICE_ID_3, - certs_mode="device", - certs_input=Path(CUSTOM_CERTS_DIR).absolute() / "certs", - certs_passwd=None, - fs=False, - single_bin=False, - ) +# -------------- Gen certs tests -------------- +TEST_DEVICE_ID = r"com.example.my.solution:gw-01:da:device:ZWave:CA0D6357%2F4" - assert os.path.exists(f"{CERTS_DIR}/ca.pem") - assert os.path.exists(f"{CERTS_DIR}/certs/iot-device.cert.pem") - assert os.path.exists(f"{CERTS_DIR}/private/iot-device.key.pem") - assert os.path.exists(f"{CERTS_DIR}/certs/azure-iot-test-only.root.ca.cert.pem") - - assert os.path.exists(f"{CERTS_DIR}/azure/{TEST_DEVICE_ID_3}-root-ca.pem") - - assert os.path.exists(f"{FS_DIR}/ca.pem") - assert os.path.exists(f"{FS_DIR}/device_cert.pem") - assert os.path.exists(f"{FS_DIR}/device_priv_key.pem") - - def test_gen_certs_default_password( - self, - request: pytest.FixtureRequest, - test_project_data: Dict[str, str], - mocker: MockerFixture, - ) -> None: - """This test check if command gen certs with provided certs_passwd parameter set expected password for - regenerated certs files""" - proj_path = request.getfixturevalue(test_project_data["proj_path"]) - os.chdir(proj_path) - - # remove directory with certs if already present - if os.path.exists(CERTS_DIR): - shutil.rmtree(CERTS_DIR) - if os.path.exists(FS_DIR): - shutil.rmtree(FS_DIR) - - assert not os.path.exists(CERTS_DIR) - assert not os.path.exists(FS_DIR) - - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), - ) - default_passwd = "1234" - incorrect_passwd = "1111" - # generate certs protected by custom password + +def mock_create_certs(process: fake_popen.FakePopen, certs_dir: Path) -> None: + cert_files = [ + certs_dir / "certs/iot-device.cert.pem", + certs_dir / "private/iot-device.key.pem", + certs_dir / "ca.pem", + certs_dir / "certs/azure-iot-test-only.root.ca.cert.pem", + ] + for cert_file in cert_files: + cert_file.parent.mkdir(parents=True, exist_ok=True) + cert_file.touch() + + +def test_gen_certs_files_not_exist( + fp: FakeProcess, mock_prepare_config: MagicMock +) -> None: + fp.register([fp.any()]) + + with pytest.raises(SystemExit): scargo_gen( profile="Debug", gen_ut=None, @@ -500,190 +175,140 @@ def test_gen_certs_default_password( single_bin=False, ) - # password protected files - pwd_protected_files = [ - f"{CERTS_DIR}/private/azure-iot-test-only.root.ca.key.pem", - f"{CERTS_DIR}/private/azure-iot-test-only.intermediate.key.pem", - ] - # test if all password-protected files are encrypted with expected password - for file in pwd_protected_files: - with open(file) as fp: - file_content = fp.read() - assert ( - "ENCRYPTED" in file_content - ), f"File {file} seems to be not encrypted" - # positive test encryption with correct password - passwd_bytes = bytes(default_passwd, "utf-8") - assert crypto.load_privatekey( - type=crypto.FILETYPE_PEM, - buffer=file_content, - passphrase=passwd_bytes, - ) - # negative test encryption with incorrect password - incorrect_passwd_bytes = bytes(incorrect_passwd, "utf-8") - with pytest.raises(crypto.Error): - crypto.load_privatekey( - type=crypto.FILETYPE_PEM, - buffer=file_content, - passphrase=incorrect_passwd_bytes, - ) - assert ( - False - ), "Files decrypted with wrong password, exception wasn't raised" - - def test_gen_certs_custom_password( - self, - request: pytest.FixtureRequest, - test_project_data: Dict[str, str], - mocker: MockerFixture, - ) -> None: - """This test check if command gen certs with provided certs_passwd parameter set expected password for - regenerated certs files""" - proj_path = request.getfixturevalue(test_project_data["proj_path"]) - os.chdir(proj_path) - - # remove directory with certs if already present - if os.path.exists(CERTS_DIR): - shutil.rmtree(CERTS_DIR) - if os.path.exists(FS_DIR): - shutil.rmtree(FS_DIR) - - assert not os.path.exists(CERTS_DIR) - assert not os.path.exists(FS_DIR) - - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), - ) - custom_passwd = "TEST_PASSWD" - incorrect_passwd = "INCORECT_PW" - # generate certs protected by custom password - scargo_gen( - profile="Debug", - gen_ut=None, - gen_mock=None, - certs=TEST_DEVICE_ID, - certs_mode=None, - certs_input=None, - certs_passwd=custom_passwd, - fs=False, - single_bin=False, - ) - # password protected files - pwd_protected_files = [ - f"{CERTS_DIR}/private/azure-iot-test-only.root.ca.key.pem", - f"{CERTS_DIR}/private/azure-iot-test-only.intermediate.key.pem", - ] - # test if all password-protected files are encrypted with expected password - for file in pwd_protected_files: - with open(file, encoding="utf-8") as fp: - file_content = fp.read() - assert ( - "ENCRYPTED" in file_content - ), f"File {file} seems to be not encrypted" - # positive test encryption with correct password - passwd_bytes = bytes(custom_passwd, "utf-8") - assert crypto.load_privatekey( - type=crypto.FILETYPE_PEM, - buffer=file_content, - passphrase=passwd_bytes, - ) - # negative test encryption with incorrect password - incorrect_passwd_bytes = bytes(incorrect_passwd, "utf-8") - with pytest.raises(crypto.Error): - crypto.load_privatekey( - type=crypto.FILETYPE_PEM, - buffer=file_content, - passphrase=incorrect_passwd_bytes, - ) - assert ( - False - ), "Files decrypted with wrong password, exception wasn't raised" +@dataclass +class GenCertTestData: + mode_argument: Optional[str] + expeced_mode_argument: str + intermediate_dir: Optional[Path] = None + certs_password: Optional[str] = None @pytest.mark.parametrize( - "test_project_data", + "test_data", [ - "new_project_x86", - "new_project_esp32", - "new_project_stm32", + GenCertTestData(None, "All-certificates"), + GenCertTestData("all", "All-certificates", Path("dir1"), "4321"), + GenCertTestData("device", "Device-certificate", Path("dir2"), "0123"), ], - ids=[ - "new_project_x86", - "new_project_esp32", - "new_project_stm32", - ], - scope="class", ) -def test_gen_ut_new_project( - request: pytest.FixtureRequest, test_project_data: str, mocker: MockerFixture +def test_gen_certs( + fp: FakeProcess, + mock_prepare_config: MagicMock, + test_data: GenCertTestData, ) -> None: - proj_path = request.getfixturevalue(test_project_data) - os.chdir(proj_path) - - if test_project_data == "new_project_esp32": - test_data = Path(__file__).parents[1].joinpath("test_data") - os.environ["IDF_PATH"] = Path(test_data, "esp32_spiff").as_posix() - h_file_path = Path(os.getcwd(), "main") - else: - h_file_path = Path(os.getcwd(), "src") - - scargo_update(Path(os.getcwd(), "scargo.toml")) - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), + certs_dir = Path("build/certs").absolute() + scargo_root = Path(__file__).parent.parent.parent + script_path = scargo_root / "scargo/certs/generateAllCertificates.sh" + + fp.register( + [ + str(script_path), + "--name", + TEST_DEVICE_ID, + "--mode", + test_data.expeced_mode_argument, + "--output", + certs_dir, + "--input", + test_data.intermediate_dir or certs_dir, + "--passwd", + test_data.certs_password or "1234", + ], + callback=mock_create_certs, + callback_kwargs={"certs_dir": certs_dir}, + ) + + scargo_gen( + profile="Debug", + gen_ut=None, + gen_mock=None, + certs=TEST_DEVICE_ID, + certs_mode=test_data.mode_argument, + certs_input=test_data.intermediate_dir, + certs_passwd=test_data.certs_password, + fs=False, + single_bin=False, ) + + assert Path("build/fs/device_cert.pem").is_file() + assert Path("build/fs/device_priv_key.pem").is_file() + assert Path("build/fs/ca.pem").is_file() + assert Path(certs_dir, f"azure/{TEST_DEVICE_ID}-root-ca.pem").is_file() + + +# -------------- Gen fs tests -------------- +def test_gen_fs_unsupored_target( + caplog: pytest.LogCaptureFixture, mock_prepare_config: MagicMock +) -> None: scargo_gen( profile="Debug", - gen_ut=h_file_path, + gen_ut=None, + gen_mock=None, + certs=None, + certs_mode=None, + certs_input=None, + certs_passwd=None, fs=True, single_bin=False, + ) + + assert "Gen --fs command not supported for this target yet." in caplog.text + + +def test_gen_fs_esp32(fp: FakeProcess, mock_prepare_config_esp32: MagicMock) -> None: + fp.register([fp.any()]) + + scargo_gen( + profile="Debug", + gen_ut=None, gen_mock=None, certs=None, certs_mode=None, certs_input=None, certs_passwd=None, + fs=True, + single_bin=False, ) - ut_path = Path(os.getcwd(), "tests", "ut") - assert "ut_test_lib.cpp" in os.listdir( - ut_path - ), f"File 'ut_test_lib.cpp' should be present in {ut_path}. Files under path: {os.listdir(ut_path)}" -@pytest.mark.parametrize( - "test_project_data", - [ - "copy_test_project_esp32", - "copy_test_project_stm32", - ], -) -def test_gen_ut_copy_old_project( - request: pytest.FixtureRequest, - test_project_data: str, - mocker: MockerFixture, +# -------------- Gen single bin tests -------------- +def test_gen_single_bin_unsupored_target( + caplog: pytest.LogCaptureFixture, mock_prepare_config: MagicMock ) -> None: - test_data = Path(__file__).parents[1].joinpath("test_data") - os.environ["IDF_PATH"] = Path(test_data, "esp32_spiff").as_posix() - proj_path = request.getfixturevalue(test_project_data) - os.chdir(proj_path) - h_file_path = Path(os.getcwd(), "main" if "esp32" in test_project_data else "src") - mocker.patch( - f"{scargo_gen.__module__}.prepare_config", - return_value=get_scargo_config_or_exit(), - ) scargo_gen( profile="Debug", - gen_ut=h_file_path, - fs=True, - single_bin=False, + gen_ut=None, + gen_mock=None, + certs=None, + certs_mode=None, + certs_input=None, + certs_passwd=None, + fs=False, + single_bin=True, + ) + + assert "Gen --bin command not supported for this target yet." in caplog.text + + +def test_gen_single_bin_esp32( + fp: FakeProcess, + mock_prepare_config_esp32: MagicMock, +) -> None: + build_profile = "Debug" + flash_args_path = Path(f"build/{build_profile}/flash_args") + flash_args_path.parent.mkdir(parents=True) + flash_args_path.touch() + fp.register([fp.any()]) + + scargo_gen( + profile=build_profile, + gen_ut=None, gen_mock=None, certs=None, certs_mode=None, certs_input=None, certs_passwd=None, + fs=False, + single_bin=True, ) - ut_path = Path(os.getcwd(), "tests", "ut") - assert "ut_test_lib.cpp" in os.listdir( - ut_path - ), f"File 'ut_test_lib.cpp' should be present in {ut_path}. Files under path: {os.listdir(ut_path)}" diff --git a/tests/ut/utils.py b/tests/ut/utils.py index 63d9db8d..695da9bc 100644 --- a/tests/ut/utils.py +++ b/tests/ut/utils.py @@ -1,12 +1,14 @@ from logging import LogRecord +from pathlib import Path from typing import Iterable, List, Tuple from scargo.config import Config, parse_config -from scargo.global_values import SCARGO_PKG_PATH -def get_test_project_config() -> Config: - return parse_config(SCARGO_PKG_PATH.parent / "tests/ut/data/scargo_test.toml") +def get_test_project_config(target: str = "x86") -> Config: + config_path = Path(__file__).parent / f"data/scargo_test_{target}.toml" + assert config_path.is_file() + return parse_config(config_path) def get_log_data(records: Iterable[LogRecord]) -> List[Tuple[str, str]]: From 7b9c725640335f15a88f7d2a4f8772a6d990d765 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:17:24 +0100 Subject: [PATCH 23/28] Bump typer version to 0.9.0 (#389) --- ci/requirements.txt | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/requirements.txt b/ci/requirements.txt index f3ea9818..4a7d9b4c 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -289,7 +289,7 @@ tomlkit==0.11.6 # via # pylint # scargo (pyproject.toml) -typer==0.7.0 +typer==0.9.0 # via scargo (pyproject.toml) types-clang==0.14.3 # via scargo (pyproject.toml) @@ -305,6 +305,7 @@ typing-extensions==4.6.0 # mypy # pydantic # scargo (pyproject.toml) + # typer unittest-xml-reporting==3.2.0 # via scargo (pyproject.toml) urllib3==1.26.15 diff --git a/pyproject.toml b/pyproject.toml index 94761fe3..8b5fb6af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "shellingham==1.5.0.post1", "toml==0.10.2", "tomlkit==0.11.6", - "typer==0.7.0", + "typer==0.9.0", "typing-extensions==4.6.0", "paramiko==3.3.1", "pyyaml==6.0", From b45ee9b9f21fa55fd74fba4ecb87f76031f46bd9 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:52:02 +0100 Subject: [PATCH 24/28] Version 2.0.0 (#391) --- scargo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scargo/__init__.py b/scargo/__init__.py index d03a66e4..046c304d 100644 --- a/scargo/__init__.py +++ b/scargo/__init__.py @@ -1,4 +1,4 @@ # # # @copyright Copyright (C) 2023 SpyroSoft Solutions S.A. All rights reserved. # # -__version__ = "1.8.0" +__version__ = "2.0.0" From 73989bd4c08682390bf7dbfc218b8d0cf7b0d5c0 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:25:31 +0100 Subject: [PATCH 25/28] Fix in repo conan cache not used (#399) --- scargo/commands/update.py | 4 ++-- scargo/config.py | 5 ++++- scargo/config_utils.py | 9 ++++++++- scargo/docker_utils.py | 4 +--- .../templates/docker/docker-compose.yaml.j2 | 4 ---- scargo/templates/.gitignore | 2 +- tests/it/utils.py | 4 +--- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/scargo/commands/update.py b/scargo/commands/update.py index 3d352ac1..6920ddf2 100644 --- a/scargo/commands/update.py +++ b/scargo/commands/update.py @@ -18,7 +18,7 @@ from scargo.file_generators.readme_gen import generate_readme from scargo.file_generators.tests_gen import generate_tests from scargo.file_generators.vscode_gen import generate_vscode -from scargo.global_values import SCARGO_DOCKER_ENV, SCARGO_LOCK_FILE, SCARGO_PKG_PATH +from scargo.global_values import SCARGO_LOCK_FILE, SCARGO_PKG_PATH from scargo.logger import get_logger logger = get_logger() @@ -96,7 +96,7 @@ def scargo_update(config_file_path: Path) -> None: generate_readme(config) # do not rebuild dockers in the docker - if project_config.build_env == SCARGO_DOCKER_ENV: + if project_config.is_docker_buildenv(): if not Path("/.dockerenv").exists(): if not pull_docker_image(docker_path): scargo_docker_build([], config.project_root) diff --git a/scargo/config.py b/scargo/config.py index 2c2915c5..a1f8c2ae 100644 --- a/scargo/config.py +++ b/scargo/config.py @@ -8,7 +8,7 @@ import toml from pydantic import BaseModel, Extra, Field, root_validator -from scargo.global_values import SCARGO_DEFAULT_BUILD_ENV +from scargo.global_values import SCARGO_DEFAULT_BUILD_ENV, SCARGO_DOCKER_ENV CHIP_DEFAULTS = { "x86": "", @@ -134,6 +134,9 @@ def get_compiler_warning(self) -> Optional[str]: return "Compiler settings are ignored for this target" return None + def is_docker_buildenv(self) -> bool: + return self.build_env == SCARGO_DOCKER_ENV + class Target(BaseModel): id: str diff --git a/scargo/config_utils.py b/scargo/config_utils.py index 8e762697..449c7e5a 100644 --- a/scargo/config_utils.py +++ b/scargo/config_utils.py @@ -1,3 +1,4 @@ +import os import sys from pathlib import Path from typing import Optional @@ -14,6 +15,11 @@ logger = get_logger() +def set_up_environment_variables(config: Config) -> None: + if config.project.in_repo_conan_cache: + os.environ["CONAN_HOME"] = f"{config.project_root}/.conan2" + + def get_scargo_config_or_exit( config_file_path: Optional[Path] = None, ) -> Config: @@ -41,12 +47,13 @@ def get_scargo_config_or_exit( def prepare_config(run_in_docker: bool = True) -> Config: """ - Prepare configuration file + Prepare configuration file and set up eniromnent variables :return: project configuration """ config = get_scargo_config_or_exit() check_scargo_version(config) + set_up_environment_variables(config) if run_in_docker: run_scargo_again_in_docker(config.project, config.project_root) return config diff --git a/scargo/docker_utils.py b/scargo/docker_utils.py index c02585ee..593b3d5d 100644 --- a/scargo/docker_utils.py +++ b/scargo/docker_utils.py @@ -6,7 +6,6 @@ from docker import DockerClient from scargo.config import ProjectConfig -from scargo.global_values import SCARGO_DOCKER_ENV from scargo.logger import get_logger logger = get_logger() @@ -43,8 +42,7 @@ def run_scargo_again_in_docker( :return: None """ - build_env = project_config.build_env - if build_env != SCARGO_DOCKER_ENV or Path("/.dockerenv").exists(): + if not project_config.is_docker_buildenv() or Path("/.dockerenv").exists(): return cmd_args = sys.argv[1:] diff --git a/scargo/file_generators/templates/docker/docker-compose.yaml.j2 b/scargo/file_generators/templates/docker/docker-compose.yaml.j2 index 5cebdec4..9b0fb67f 100644 --- a/scargo/file_generators/templates/docker/docker-compose.yaml.j2 +++ b/scargo/file_generators/templates/docker/docker-compose.yaml.j2 @@ -24,10 +24,6 @@ services: - ..:/workspace - /dev:/dev command: sleep infinity - {% if config.project.in_repo_conan_cache %} - environment: - - CONAN_USER_HOME=/workspace - {% endif %} {% if config.docker_compose.ports %} ports: {% for port_ranges in config.docker_compose.ports %} diff --git a/scargo/templates/.gitignore b/scargo/templates/.gitignore index c2a7e59e..508bd140 100644 --- a/scargo/templates/.gitignore +++ b/scargo/templates/.gitignore @@ -13,7 +13,7 @@ __pycache__/ package-lock.json .env .devcontainer/.env -.conan +.conan2 ### Linux ### *~ diff --git a/tests/it/utils.py b/tests/it/utils.py index 21ccadf0..5b9ecb0d 100644 --- a/tests/it/utils.py +++ b/tests/it/utils.py @@ -10,7 +10,6 @@ from scargo.config import ProjectConfig from scargo.docker_utils import prepare_docker -from scargo.global_values import SCARGO_DOCKER_ENV from scargo.logger import get_logger logger = get_logger() @@ -149,8 +148,7 @@ def run_custom_command_in_docker( :param Path project_path: path to project root :return: None """ - build_env = project_config.build_env - if build_env != SCARGO_DOCKER_ENV or Path("/.dockerenv").exists(): + if not project_config.is_docker_buildenv() or Path("/.dockerenv").exists(): return None docker_settings = prepare_docker(project_config, project_path) From c31bae9ee36db35034b18f7b050383986d1cbe62 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:19:57 +0100 Subject: [PATCH 26/28] Fix remote and package handling (#398) * Fix conan package related commands * Remove redundant login arguments --- scargo/commands/publish.py | 25 ++--- scargo/conan_utils.py | 50 ++++++---- tests/ut/ut_conan_utils.py | 140 ++++++++++++++++++-------- tests/ut/ut_scargo_publish.py | 183 ++++++++++------------------------ tests/ut/ut_scargo_test.py | 105 +++++++------------ 5 files changed, 229 insertions(+), 274 deletions(-) diff --git a/scargo/commands/publish.py b/scargo/commands/publish.py index f8449c58..6fa4105a 100644 --- a/scargo/commands/publish.py +++ b/scargo/commands/publish.py @@ -39,19 +39,17 @@ def scargo_publish(repo: str, profile: str = "Release") -> None: # Export package try: - subprocess.check_call( + subprocess.run( [ "conan", "export-pkg", ".", - "-of", - str(build_dir), - "-pr:b", - "default", - "-pr:h", + "-pr", f"./config/conan/profiles/{config.project.target.family}_{profile}", - "-f", + "-of", + build_dir, ], + check=True, cwd=project_path, ) except subprocess.CalledProcessError: @@ -60,35 +58,34 @@ def scargo_publish(repo: str, profile: str = "Release") -> None: # Test if package has been exported successfully try: - subprocess.check_call( + subprocess.run( [ "conan", "test", "test_package", f"{project_name}/{config.project.version}", - "-pr:b", - "default", - "-pr:h", + "-pr", f"./config/conan/profiles/{config.project.target.family}_{profile}", ], + check=True, cwd=project_path, ) except subprocess.CalledProcessError: logger.error("Package test failed") sys.exit(1) - # Upload package to artifactory + # Upload package to conan remote conan_repo = ["-r", repo] if repo else [] try: - subprocess.check_call( + subprocess.run( [ "conan", "upload", f"{project_name}", *conan_repo, - "--all", "--confirm", ], + check=True, cwd=project_path, ) except subprocess.CalledProcessError: diff --git a/scargo/conan_utils.py b/scargo/conan_utils.py index ec0497ee..0cd8b36e 100644 --- a/scargo/conan_utils.py +++ b/scargo/conan_utils.py @@ -1,6 +1,6 @@ -import os import subprocess from pathlib import Path +from typing import Dict, List from scargo.config import Config from scargo.logger import get_logger @@ -16,8 +16,8 @@ def conan_add_remote(project_path: Path, config: Config) -> None: :param Config config: :return: None """ - conan_repo = config.conan.repo - for repo_name, repo_url in conan_repo.items(): + remotes_without_user = _get_remotes_without_user(config.conan.repo) + for repo_name, repo_url in config.conan.repo.items(): try: subprocess.run( ["conan", "remote", "add", repo_name, repo_url], @@ -29,30 +29,26 @@ def conan_add_remote(project_path: Path, config: Config) -> None: if b"already exists in remotes" not in e.stderr: logger.error(e.stderr.decode().strip()) logger.error("Unable to add remote repository") - conan_add_user(repo_name) + else: + pass + if repo_name in remotes_without_user: + conan_remote_login(repo_name) -def conan_add_user(remote: str) -> None: +def conan_remote_login(remote: str) -> None: """ Add conan user :param str remote: name of remote repository :return: None """ - conan_user = subprocess.run( - "conan user", capture_output=True, shell=True, check=False - ).stdout.decode("utf-8") + remote_login_command = ["conan", "remote", "login", remote] + logger.info("Login to conan remote %s", remote) - env_conan_user = os.environ.get("CONAN_LOGIN_USERNAME", "") - env_conan_passwd = os.environ.get("CONAN_PASSWORD", "") - - if env_conan_user not in conan_user: - try: - subprocess.check_call( - ["conan", "user", "-p", env_conan_passwd, "-r", remote, env_conan_user], - ) - except subprocess.CalledProcessError: - logger.error("Unable to add user") + try: + subprocess.run(remote_login_command, check=True) + except subprocess.CalledProcessError: + logger.error(f"Unable to log in to conan remote {remote}") def conan_source(project_dir: Path) -> None: @@ -67,3 +63,21 @@ def conan_source(project_dir: Path) -> None: ) except subprocess.CalledProcessError: logger.error("Unable to source") + + +def _get_remotes_without_user(conan_remotes: Dict[str, str]) -> List[str]: + result = subprocess.run( + ["conan", "remote", "list-users"], check=True, stdout=subprocess.PIPE + ) + user_list_stdout = result.stdout.decode().splitlines() + + no_users = [] + for index, line in enumerate(user_list_stdout): + if remote_line := line.strip().split(":")[0]: + if ( + remote_line in conan_remotes + and "No user" in user_list_stdout[index + 1] + ): + no_users.append(remote_line) + + return no_users diff --git a/tests/ut/ut_conan_utils.py b/tests/ut/ut_conan_utils.py index f2dbc998..577a66aa 100644 --- a/tests/ut/ut_conan_utils.py +++ b/tests/ut/ut_conan_utils.py @@ -7,7 +7,7 @@ from pytest_subprocess import FakeProcess from pytest_subprocess.fake_popen import FakePopen -from scargo.conan_utils import conan_add_remote, conan_add_user +from scargo.conan_utils import conan_add_remote, conan_remote_login, conan_source from scargo.config import Config from tests.ut.ut_scargo_publish import ( ENV_CONAN_PASSWORD, @@ -16,39 +16,21 @@ REMOTE_REPO_NAME_1, REMOTE_REPO_NAME_2, REPO_NAME, - config, ) +from tests.ut.utils import get_test_project_config -def conan_remote_add_error(process: FakePopen, stderr: str) -> None: - raise subprocess.CalledProcessError(1, process.args, b"", stderr.encode()) +@pytest.fixture +def config(fp: FakeProcess) -> Config: + test_project_config = get_test_project_config() + test_project_config.conan.repo[REMOTE_REPO_NAME_1] = EXAMPLE_URL + test_project_config.conan.repo[REMOTE_REPO_NAME_2] = EXAMPLE_URL + return test_project_config -def test_conan_add_user(fp: FakeProcess) -> None: - # ARRANGE - conan_user_cmd = "conan user" - add_user_cmd = [ - "conan", - "user", - "-p", - ENV_CONAN_PASSWORD, - "-r", - REPO_NAME, - ENV_CONAN_USER, - ] - fp.register(conan_user_cmd, occurrences=2) - fp.register(add_user_cmd) - - # ACT - with patch.dict( - os.environ, - {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, - ): - conan_add_user(REPO_NAME) - assert fp.calls[0] == conan_user_cmd - assert fp.calls[1] == add_user_cmd - assert len(fp.calls) == 2 +def conan_remote_add_error(process: FakePopen, stderr: str = "") -> None: + raise subprocess.CalledProcessError(1, process.args, b"", stderr.encode()) def test_conan_add_remote_fail( @@ -57,8 +39,8 @@ def test_conan_add_remote_fail( fp: FakeProcess, ) -> None: # ARRANGE + fp.register(["conan", "remote", "list-users"]) fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) fp.register( ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], returncode=1 ) @@ -70,21 +52,28 @@ def test_conan_add_remote_fail( assert "Unable to add remote repository" in caplog.text -def test_conan_add_user_fail(caplog: pytest.LogCaptureFixture, fp: FakeProcess) -> None: +def test_conan_add_remote_add_missing_user( + config: Config, + caplog: pytest.LogCaptureFixture, + fp: FakeProcess, +) -> None: # ARRANGE - cmd = ["conan", "user", "-p", ENV_CONAN_PASSWORD, "-r", REPO_NAME, ENV_CONAN_USER] - fp.register(cmd, returncode=1) - fp.register("conan user", stdout="user_name") + list_user_stdout = b""" +conancenter: + No user +remote_repo_name_1: + No user +remote_repo_name_2: + Username: env_conan_user_name + authenticated: True +""" + fp.register(["conan", "remote", "list-users"], stdout=list_user_stdout) + fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) + fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL]) + fp.register(["conan", "remote", "login", REMOTE_REPO_NAME_1]) # ACT - with patch.dict( - os.environ, - {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, - ): - conan_add_user(REPO_NAME) - - # ASSERT - assert "Unable to add user" in caplog.text + conan_add_remote(Path("some_path"), config) def test_conan_add_remote_already_exists( @@ -93,8 +82,8 @@ def test_conan_add_remote_already_exists( fp: FakeProcess, ) -> None: # ARRANGE + fp.register(["conan", "remote", "list-users"]) fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) error = f"ERROR: Remote '{REMOTE_REPO_NAME_2}' already exists in remotes (use --force to continue)" fp.register( ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], @@ -107,3 +96,70 @@ def test_conan_add_remote_already_exists( # ASSERT assert "Unable to add remote repository" not in caplog.text + + +def test_conan_remote_login(fp: FakeProcess) -> None: + # ARRANGE + add_user_cmd = ["conan", "remote", "login", REPO_NAME] + fp.register(add_user_cmd) + + # ACT + conan_remote_login(REPO_NAME) + + assert fp.calls.popleft() == add_user_cmd + + +def test_conan_remote_login_env_vars(fp: FakeProcess) -> None: + # ARRANGE + add_user_cmd = [ + "conan", + "remote", + "login", + REPO_NAME, + ] + fp.register(add_user_cmd) + + # ACT + with patch.dict( + os.environ, + {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, + ): + conan_remote_login(REPO_NAME) + + assert fp.calls.popleft() == add_user_cmd + + +def test_conan_remote_login_fail( + caplog: pytest.LogCaptureFixture, fp: FakeProcess +) -> None: + # ARRANGE + add_user_cmd = [ + "conan", + "remote", + "login", + REPO_NAME, + ] + fp.register(add_user_cmd, returncode=1) + + # ACT + with patch.dict( + os.environ, + {"CONAN_LOGIN_USERNAME": ENV_CONAN_USER, "CONAN_PASSWORD": ENV_CONAN_PASSWORD}, + ): + conan_remote_login(REPO_NAME) + + # ASSERT + assert "Unable to log in to conan remote repo_name" in caplog.text + + +def test_conan_source_fails( + config: Config, caplog: pytest.LogCaptureFixture, fp: FakeProcess +) -> None: + # ARRANGE + fp.register(["conan", "source", "."], returncode=1) + + # ACT + conan_source(config.project_root) + + # ASSERT + assert "Unable to source" in caplog.text diff --git a/tests/ut/ut_scargo_publish.py b/tests/ut/ut_scargo_publish.py index 3bb3d1a9..55108ff9 100644 --- a/tests/ut/ut_scargo_publish.py +++ b/tests/ut/ut_scargo_publish.py @@ -14,10 +14,18 @@ REPO_NAME = "repo_name" ENV_CONAN_USER = "env_conan_user_name" ENV_CONAN_PASSWORD = "env_conan_password" +CONAN_REMOTE_CALLS = [ + ["conan", "remote", "list-users"], + ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL], + ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL], +] +CONAN_SETUP_CALLS = CONAN_REMOTE_CALLS + [ + ["conan", "source", "."], +] @pytest.fixture -def config(monkeypatch: pytest.MonkeyPatch) -> Config: +def config(monkeypatch: pytest.MonkeyPatch, fp: FakeProcess) -> Config: test_project_config = get_test_project_config() test_project_config.conan.repo[REMOTE_REPO_NAME_1] = EXAMPLE_URL test_project_config.conan.repo[REMOTE_REPO_NAME_2] = EXAMPLE_URL @@ -27,6 +35,9 @@ def config(monkeypatch: pytest.MonkeyPatch) -> Config: lambda: test_project_config, ) + for command in CONAN_SETUP_CALLS: + fp.register(command) + return test_project_config @@ -35,68 +46,44 @@ def test_publish(config: Config, fp: FakeProcess) -> None: project_name = config.project.name build_path = Path(f"{config.project_root}/build/Release") build_path.mkdir(parents=True, exist_ok=True) + profile_path = f"./config/conan/profiles/{config.project.target.family}_Release" - conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] - conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] - conan_user_cmd = "conan user" - conan_source_cmd = [ - "conan", - "source", - ".", - ] - conan_export_pkg_cmd = [ - "conan", - "export-pkg", - ".", - "-of", - str(build_path), - "-pr:b", - "default", - "-pr:h", - f"./config/conan/profiles/{config.project.target.family}_Release", - "-f", - ] - conan_test_cmd = [ - "conan", - "test", - "test_package", - f"{project_name}/{config.project.version}", - "-pr:b", - "default", - "-pr:h", - f"./config/conan/profiles/{config.project.target.family}_Release", - ] - conan_upload_cmd = [ - "conan", - "upload", - f"{project_name}", - "-r", - REPO_NAME, - "--all", - "--confirm", + subprocess_commands = [ + [ + "conan", + "export-pkg", + ".", + "-pr", + profile_path, + "-of", + build_path, + ], + [ + "conan", + "test", + "test_package", + f"{project_name}/{config.project.version}", + "-pr", + profile_path, + ], + [ + "conan", + "upload", + f"{project_name}", + "-r", + REPO_NAME, + "--confirm", + ], ] - fp.register(conan_add_remote_1_cmd) - fp.register(conan_add_remote_2_cmd) - fp.register(conan_user_cmd, occurrences=2) - fp.register(conan_source_cmd) - fp.register(conan_export_pkg_cmd) - fp.register(conan_test_cmd) - fp.register(conan_upload_cmd) + for command in subprocess_commands: + fp.register(command) # type: ignore[arg-type] # ACT scargo_publish(REPO_NAME) # ASSERT - assert fp.calls[0] == conan_add_remote_1_cmd - assert fp.calls[1] == conan_user_cmd - assert fp.calls[2] == conan_add_remote_2_cmd - assert fp.calls[3] == conan_user_cmd - assert fp.calls[4] == conan_source_cmd - assert fp.calls[5] == conan_export_pkg_cmd - assert fp.calls[6] == conan_test_cmd - assert fp.calls[7] == conan_upload_cmd - assert len(fp.calls) == 8 + assert list(fp.calls) == CONAN_SETUP_CALLS + subprocess_commands def test_create_package_fail( @@ -105,62 +92,23 @@ def test_create_package_fail( fp: FakeProcess, ) -> None: # ARRANGE - project_name = config.project.name build_path = Path(f"{config.project_root}/build/Release") build_path.mkdir(parents=True, exist_ok=True) - - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) - fp.register( - [ - "conan", - "source", - ".", - ] - ) + profile_path = f"./config/conan/profiles/{config.project.target.family}_Release" fp.register( [ "conan", "export-pkg", ".", + "-pr", + profile_path, "-of", - str(build_path), - "-pr:b", - "default", - "-pr:h", - f"./config/conan/profiles/{config.project.target.family}_Release", - "-f", + f"{build_path}", ], returncode=1, ) - fp.register( - [ - "conan", - "test", - "test_package", - f"{project_name}/{config.project.version}", - "-pr:b", - "default", - "-pr:h", - f"./config/conan/profiles/{config.project.target.family}_Release", - ] - ) - - fp.register( - [ - "conan", - "upload", - f"{project_name}", - "-r", - REPO_NAME, - "--all", - "--confirm", - ] - ) - # ACT with pytest.raises(SystemExit) as error: scargo_publish(REPO_NAME) @@ -179,31 +127,18 @@ def test_upload_package_fail( project_name = config.project.name build_path = Path(f"{config.project_root}/build/Release") build_path.mkdir(parents=True, exist_ok=True) + profile_path = f"./config/conan/profiles/{config.project.target.family}_Release" - fp.register(["conan", "remote", "clean"]) - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL]) - fp.register(["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL]) - fp.register("conan user", occurrences=2) - fp.register( - [ - "conan", - "source", - ".", - ] - ) fp.register( [ "conan", "export-pkg", ".", + "-pr", + profile_path, "-of", - str(build_path), - "-pr:b", - "default", - "-pr:h", - f"./config/conan/profiles/{config.project.target.family}_Release", - "-f", - ] + f"{build_path}", + ], ) fp.register( [ @@ -211,22 +146,12 @@ def test_upload_package_fail( "test", "test_package", f"{project_name}/{config.project.version}", - "-pr:b", - "default", - "-pr:h", - f"./config/conan/profiles/{config.project.target.family}_Release", + "-pr", + profile_path, ] ) fp.register( - [ - "conan", - "upload", - f"{project_name}", - "-r", - REPO_NAME, - "--all", - "--confirm", - ], + ["conan", "upload", f"{project_name}", "-r", REPO_NAME, "--confirm"], returncode=1, ) diff --git a/tests/ut/ut_scargo_test.py b/tests/ut/ut_scargo_test.py index a3481473..9df41a13 100644 --- a/tests/ut/ut_scargo_test.py +++ b/tests/ut/ut_scargo_test.py @@ -9,12 +9,6 @@ from scargo.commands.test import scargo_test from scargo.config import Config -from tests.ut.ut_scargo_publish import ( - EXAMPLE_URL, - REMOTE_REPO_NAME_1, - REMOTE_REPO_NAME_2, - config, -) def test_scargo_test_no_test_dir( # type: ignore[no-any-unimported] @@ -48,75 +42,44 @@ def test_scargo_test( # type: ignore[no-any-unimported] config: Config, fp: FakeProcess, fs: FakeFilesystem, mock_prepare_config: MagicMock ) -> None: config.project_root = Path(".") - conan_add_remote_1_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_1, EXAMPLE_URL] - conan_add_remote_2_cmd = ["conan", "remote", "add", REMOTE_REPO_NAME_2, EXAMPLE_URL] - conan_user_cmd = "conan user" - conan_source_cmd = [ - "conan", - "source", - ".", + subprocess_commands = [ + ["conan", "profile", "list"], + ["conan", "profile", "detect"], + ["conan", "remote", "list-users"], + ["conan", "source", "."], + [ + "conan", + "install", + Path("tests"), + "-of", + Path("build/tests"), + "-sbuild_type=Debug", + "-b", + "missing", + ], + [ + "conan", + "build", + "-of", + Path("build/tests"), + Path("tests"), + "-sbuild_type=Debug", + "-b", + "missing", + ], + ["ctest"], + ["gcovr", "-r", "ut", ".", "-f", Path("src"), "--html=ut-coverage.html"], ] - fp.register(conan_add_remote_1_cmd) - fp.register(conan_add_remote_2_cmd) - fp.register(conan_user_cmd, occurrences=2) - fp.register(conan_source_cmd) - fp.register("conan profile list") - fp.register("conan profile detect") - fp.register("conan install tests -of build/tests -sbuild_type=Debug -b missing") - fp.register("conan build -of build/tests tests -sbuild_type=Debug -b missing") - fp.register("gcovr -r ut . -f src --html=ut-coverage.html") - fp.register("ctest") - os.mkdir("tests") - with open("tests/CMakeLists.txt", "w"): - pass + for command in subprocess_commands: + fp.register(command) # type: ignore[arg-type] + + Path("tests").mkdir() + Path("tests/CMakeLists.txt").touch() scargo_test(False) - assert fp.calls[0] == [ - "conan", - "profile", - "list", - ] - assert fp.calls[1] == [ - "conan", - "profile", - "detect", - ] - assert fp.calls[2] == conan_add_remote_1_cmd - assert fp.calls[3] == conan_user_cmd - assert fp.calls[4] == conan_add_remote_2_cmd - assert fp.calls[5] == conan_user_cmd - assert fp.calls[6] == conan_source_cmd - assert fp.calls[7] == [ - "conan", - "install", - Path("tests"), - "-of", - Path("build/tests"), - "-sbuild_type=Debug", - "-b", - "missing", - ] - assert fp.calls[8] == [ - "conan", - "build", - "-of", - Path("build/tests"), - Path("tests"), - "-sbuild_type=Debug", - "-b", - "missing", - ] - assert fp.calls[9] == ["ctest"] - assert fp.calls[10] == [ - "gcovr", - "-r", - "ut", - ".", - "-f", - Path("src"), - "--html=ut-coverage.html", - ] + # assert correct order + assert list(fp.calls) == subprocess_commands @pytest.fixture From bada3209bad894482d613ee973a39112a4fd5ea8 Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:14:45 +0100 Subject: [PATCH 27/28] Fix Stm32 and atsam toolchains (#400) * stm32 toolchain fix * Add arm toolchain for atsam * Use profile dir for toolchains --- scargo/config_utils.py | 1 + scargo/file_generators/base_gen.py | 4 +- scargo/file_generators/conan_gen.py | 15 ++++-- .../templates/conan/profile_atsam.j2 | 2 +- .../templates/conan/profile_stm32.j2 | 3 +- .../conan/toolchain/arm_gcc_toolchain.cmake | 54 +++++++++++++++++++ .../stm32_gcc_toolchain.cmake.j2} | 3 +- .../file_generators/templates/stm32.cmake.j2 | 1 - 8 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 scargo/file_generators/templates/conan/toolchain/arm_gcc_toolchain.cmake rename scargo/file_generators/templates/conan/{stm32_gcc_toolchain_wrapper.cmake.j2 => toolchain/stm32_gcc_toolchain.cmake.j2} (54%) diff --git a/scargo/config_utils.py b/scargo/config_utils.py index 449c7e5a..a6178cad 100644 --- a/scargo/config_utils.py +++ b/scargo/config_utils.py @@ -16,6 +16,7 @@ def set_up_environment_variables(config: Config) -> None: + os.environ["SCARGO_PROJECT_ROOT"] = str(config.project_root.absolute()) if config.project.in_repo_conan_cache: os.environ["CONAN_HOME"] = f"{config.project_root}/.conan2" diff --git a/scargo/file_generators/base_gen.py b/scargo/file_generators/base_gen.py index d4fbdb6b..7836647a 100644 --- a/scargo/file_generators/base_gen.py +++ b/scargo/file_generators/base_gen.py @@ -13,9 +13,9 @@ logger = get_logger() -_TEMPLATE_ROOT = Path(SCARGO_PKG_PATH, "file_generators", "templates") +TEMPLATE_ROOT = Path(SCARGO_PKG_PATH, "file_generators", "templates") _JINJA_ENV = Environment( - loader=FileSystemLoader(_TEMPLATE_ROOT), + loader=FileSystemLoader(TEMPLATE_ROOT), trim_blocks=True, lstrip_blocks=True, ) diff --git a/scargo/file_generators/conan_gen.py b/scargo/file_generators/conan_gen.py index f97b90e8..406d1e30 100644 --- a/scargo/file_generators/conan_gen.py +++ b/scargo/file_generators/conan_gen.py @@ -2,10 +2,12 @@ # @copyright Copyright (C) 2023 SpyroSoft Solutions S.A. All rights reserved. # # +import shutil import subprocess +from pathlib import Path from scargo.config import Config -from scargo.file_generators.base_gen import create_file_from_template +from scargo.file_generators.base_gen import TEMPLATE_ROOT, create_file_from_template def generate_conanfile(config: Config) -> None: @@ -28,11 +30,18 @@ def generate_conanprofile(config: Config) -> None: if config.project.target.family == "stm32": create_file_from_template( - "conan/stm32_gcc_toolchain_wrapper.cmake.j2", - "config/conan/profiles/stm32_gcc_toolchain_wrapper.cmake", + "conan/toolchain/stm32_gcc_toolchain.cmake.j2", + "config/conan/profiles/stm32_gcc_toolchain.cmake", template_params={"config": config}, config=config, ) + elif config.project.target.family == "atsam": + outpath = Path("config/conan/profiles/arm_gcc_toolchain.cmake") + outpath.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile( + TEMPLATE_ROOT / "conan/toolchain/arm_gcc_toolchain.cmake", + outpath, + ) for profile in profiles: create_file_from_template( diff --git a/scargo/file_generators/templates/conan/profile_atsam.j2 b/scargo/file_generators/templates/conan/profile_atsam.j2 index a366b7cf..0c9a5c38 100644 --- a/scargo/file_generators/templates/conan/profile_atsam.j2 +++ b/scargo/file_generators/templates/conan/profile_atsam.j2 @@ -15,7 +15,7 @@ CXX=arm-none-eabi-g++ [conf] tools.build:cflags=["{{ config.project.cflags if config.project.cflags}} {{config.profiles.get(profile).cflags if config.profiles.get(profile).cflags}}"] tools.build:cxxflags=["{{ config.project.cxxflags if config.project.cxxflags }} {{ config.profiles.get(profile).cxxflags if config.profiles.get(profile).cxxflags }}"] - {% if config.project.max_build_jobs != None %} tools.build:jobs={{config.project.max_build_jobs}} {% endif %} +tools.cmake.cmaketoolchain:user_toolchain=["{{ '{{ os.path.join(profile_dir, "arm_gcc_toolchain.cmake") }}' }}"] diff --git a/scargo/file_generators/templates/conan/profile_stm32.j2 b/scargo/file_generators/templates/conan/profile_stm32.j2 index 9ddbf6dc..46e9c5ab 100644 --- a/scargo/file_generators/templates/conan/profile_stm32.j2 +++ b/scargo/file_generators/templates/conan/profile_stm32.j2 @@ -18,5 +18,4 @@ tools.build:cxxflags=["{{ config.project.cxxflags if config.project.cxxflags }} {% if config.project.max_build_jobs != None %} tools.build:jobs={{config.project.max_build_jobs}} {% endif %} -{% set workspace_dir = config.project_root if config.project.build_env != "docker" else "/workspace" %} -tools.cmake.cmaketoolchain:user_toolchain=["{{ workspace_dir }}/config/conan/profiles/stm32_gcc_toolchain_wrapper.cmake"] +tools.cmake.cmaketoolchain:user_toolchain=["{{ '{{ os.path.join(profile_dir, "stm32_gcc_toolchain.cmake") }}' }}"] diff --git a/scargo/file_generators/templates/conan/toolchain/arm_gcc_toolchain.cmake b/scargo/file_generators/templates/conan/toolchain/arm_gcc_toolchain.cmake new file mode 100644 index 00000000..77d1d569 --- /dev/null +++ b/scargo/file_generators/templates/conan/toolchain/arm_gcc_toolchain.cmake @@ -0,0 +1,54 @@ +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_SYSTEM_PROCESSOR ARM) + +set(TOOLCHAIN_PREFIX arm-none-eabi-) +find_program(BINUTILS_PATH ${TOOLCHAIN_PREFIX}gcc NO_CACHE) + +if (NOT BINUTILS_PATH) + message(FATAL_ERROR "ARM GCC toolchain not found") +endif () + +get_filename_component(ARM_TOOLCHAIN_DIR ${BINUTILS_PATH} DIRECTORY) +# Without that flag CMake is not able to pass test compilation check +if (${CMAKE_VERSION} VERSION_EQUAL "3.6.0" OR ${CMAKE_VERSION} VERSION_GREATER "3.6") + set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) +else () + set(CMAKE_EXE_LINKER_FLAGS_INIT "--specs=nosys.specs") +endif () + +set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}gcc) +set(CMAKE_ASM_COMPILER ${CMAKE_C_COMPILER}) +set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}g++) +set(CMAKE_AR ${TOOLCHAIN_PREFIX}gcc-ar) +set(CMAKE_RANLIB ${TOOLCHAIN_PREFIX}gcc-ranlib) + +execute_process(COMMAND ${CMAKE_C_COMPILER} -print-sysroot + OUTPUT_VARIABLE ARM_GCC_SYSROOT OUTPUT_STRIP_TRAILING_WHITESPACE) + +# Default C compiler flags +set(CMAKE_C_FLAGS_DEBUG_INIT "-g3 -Og -Wall -DDEBUG") +set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG_INIT}" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELEASE_INIT "-O3 -Wall") +set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE_INIT}" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_MINSIZEREL_INIT "-Os -Wall") +set(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL_INIT}" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELWITHDEBINFO_INIT "-O2 -g -Wall") +set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO_INIT}" CACHE STRING "" FORCE) +# Default C++ compiler flags +set(CMAKE_CXX_FLAGS_DEBUG_INIT "-g3 -Og -Wall -DDEBUG") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG_INIT}" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELEASE_INIT "-O3 -Wall") +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE_INIT}" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_MINSIZEREL_INIT "-Os -Wall") +set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL_INIT}" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO_INIT "-O2 -g -Wall") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO_INIT}" CACHE STRING "" FORCE) + +set(CMAKE_OBJCOPY ${ARM_TOOLCHAIN_DIR}/${TOOLCHAIN_PREFIX}objcopy CACHE INTERNAL "objcopy tool") +set(CMAKE_SIZE_UTIL ${ARM_TOOLCHAIN_DIR}/${TOOLCHAIN_PREFIX}size CACHE INTERNAL "size tool") + +set(CMAKE_SYSROOT ${ARM_GCC_SYSROOT}) +set(CMAKE_FIND_ROOT_PATH ${BINUTILS_PATH}) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/scargo/file_generators/templates/conan/stm32_gcc_toolchain_wrapper.cmake.j2 b/scargo/file_generators/templates/conan/toolchain/stm32_gcc_toolchain.cmake.j2 similarity index 54% rename from scargo/file_generators/templates/conan/stm32_gcc_toolchain_wrapper.cmake.j2 rename to scargo/file_generators/templates/conan/toolchain/stm32_gcc_toolchain.cmake.j2 index d0603d67..c5da2cd7 100644 --- a/scargo/file_generators/templates/conan/stm32_gcc_toolchain_wrapper.cmake.j2 +++ b/scargo/file_generators/templates/conan/toolchain/stm32_gcc_toolchain.cmake.j2 @@ -1,5 +1,4 @@ -{% set workspace_dir = config.project_root if config.project.build_env != "docker" else "/workspace" %} -include({{ workspace_dir }}/third-party/stm32-cmake/cmake/stm32_gcc.cmake) +include($ENV{SCARGO_PROJECT_ROOT}/third-party/stm32-cmake/cmake/stm32_gcc.cmake) stm32_get_chip_info($ENV{STM32_CHIP} FAMILY STM32_FAMILY DEVICE STM32_DEVICE TYPE STM32_TYPE) diff --git a/scargo/file_generators/templates/stm32.cmake.j2 b/scargo/file_generators/templates/stm32.cmake.j2 index c9770368..01a88468 100644 --- a/scargo/file_generators/templates/stm32.cmake.j2 +++ b/scargo/file_generators/templates/stm32.cmake.j2 @@ -1,6 +1,5 @@ {% include 'project.cmake.j2' %} -stm32_get_chip_info($ENV{STM32_CHIP} FAMILY STM32_FAMILY DEVICE STM32_DEVICE TYPE STM32_TYPE) message("Fetching stm32 cube for stm32 ${STM32_FAMILY} family") stm32_fetch_cube(${STM32_FAMILY}) From e9f883a670332f553a4b120a5ce9849321daafeb Mon Sep 17 00:00:00 2001 From: fkuatspyro <123727166+fkuatspyro@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:14:57 +0100 Subject: [PATCH 28/28] Version 2.0.1 (#401) --- scargo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scargo/__init__.py b/scargo/__init__.py index 046c304d..d324484f 100644 --- a/scargo/__init__.py +++ b/scargo/__init__.py @@ -1,4 +1,4 @@ # # # @copyright Copyright (C) 2023 SpyroSoft Solutions S.A. All rights reserved. # # -__version__ = "2.0.0" +__version__ = "2.0.1"