diff --git a/example/symlink/.fmf/version b/.distro/.fmf/version similarity index 100% rename from example/symlink/.fmf/version rename to .distro/.fmf/version diff --git a/.distro/fmf-jinja.rpmlintrc b/.distro/fmf-jinja.rpmlintrc new file mode 100644 index 0000000..e69de29 diff --git a/.distro/fmf-jinja.spec b/.distro/fmf-jinja.spec new file mode 100644 index 0000000..2a4a2d7 --- /dev/null +++ b/.distro/fmf-jinja.spec @@ -0,0 +1,47 @@ +Name: fmf-jinja +Version: 0.0.0 +Release: %autorelease +Summary: Jinja template engine using FMF metadata + +License: GPL-3.0-or-later +URL: https://github.com/LecrisUT/fmf-jinja +Source: %{pypi_source tmt_cmake} + +BuildArch: noarch +BuildRequires: python3-devel + +%py_provides python3-fmf-jinja + +%description +Jinja template engine using FMF metadata + + +%prep +%autosetup -n fmf-jinja-%{version} + + +%generate_buildrequires +%pyproject_buildrequires -x test + + +%build +%pyproject_wheel + + +%install +%pyproject_install +%pyproject_save_files fmf_jinja + + +%check +%pytest + + +%files -f %{pyproject_files} +%{_bindir}/fmf-jinja +%license LICENSE.md +%doc README.md + + +%changelog +%autochangelog diff --git a/.distro/plans/full.fmf b/.distro/plans/full.fmf new file mode 100644 index 0000000..31e5ecb --- /dev/null +++ b/.distro/plans/full.fmf @@ -0,0 +1,10 @@ +summary: All tmt tests +discover+: + how: fmf + path: . +execute: + how: tmt +prepare: + how: install + package: + - python3-pytest diff --git a/.distro/plans/main.fmf b/.distro/plans/main.fmf new file mode 100644 index 0000000..48c561b --- /dev/null +++ b/.distro/plans/main.fmf @@ -0,0 +1,4 @@ +adjust: + when: initiator is not defined or initiator != packit + discover+: + dist-git-source: true diff --git a/.distro/plans/rpminspect.fmf b/.distro/plans/rpminspect.fmf new file mode 100644 index 0000000..d545f79 --- /dev/null +++ b/.distro/plans/rpminspect.fmf @@ -0,0 +1,10 @@ +plan: + import: + url: https://github.com/packit/tmt-plans + ref: main + name: /plans/rpminspect +environment: + # upstream is excluded here because it triggers "Unexpected changed source archive content" + # This happens when the released version already contains the package version: + # https://github.com/packit/tmt-plans/issues/13 + RPMINSPECT_EXCLUDE: metadata,upstream diff --git a/.distro/plans/rpmlint.fmf b/.distro/plans/rpmlint.fmf new file mode 100644 index 0000000..aac0154 --- /dev/null +++ b/.distro/plans/rpmlint.fmf @@ -0,0 +1,5 @@ +plan: + import: + url: https://github.com/packit/tmt-plans + ref: main + name: /plans/rpmlint diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/.git_archival.txt b/.git_archival.txt index 8fb235d..7c51009 100644 --- a/.git_archival.txt +++ b/.git_archival.txt @@ -1,4 +1,3 @@ node: $Format:%H$ node-date: $Format:%cI$ describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ -ref-names: $Format:%D$ diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 0000000..cde3f62 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "groupName": "CI and devDependencies", + "matchManagers": ["github-actions", "pre-commit"] + } + ], + "separateMajorMinor": false, + "extends": [ + "config:recommended", + ":dependencyDashboard", + "schedule:weekly", + ":enablePreCommit", + ":semanticCommitTypeAll(chore)" + ] +} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 857ff6b..f95d47b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,20 +1,15 @@ name: CI +run-name: > + CI (${{ github.event_name }}) + ${{ github.event_name == 'pull_request' && format('PR#{0}', github.event.number) || '' }} on: workflow_dispatch: - inputs: - upload-wheel: - type: boolean - required: false - default: false - description: Upload wheel as an artifact - pytest-flags: - type: string - required: false - description: Additional flags to add to pytest pull_request: push: branches: [ main ] + schedule: + - cron: 0 0 * * 3 permissions: contents: read @@ -31,32 +26,39 @@ jobs: needs: [ pre-commit ] uses: ./.github/workflows/step_test.yaml with: - pytest-flags: ${{ inputs.pytest-flags }} + mask-experimental: ${{ github.event_name == 'push' }} - code-analysis: - uses: ./.github/workflows/step_code-analysis.yaml + coverage: + name: ๐Ÿ‘€ coverage needs: [ tests ] + uses: ./.github/workflows/step_coverage.yaml + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + if: github.event_name != 'schedule' + + docs: + name: ๐Ÿ“˜ docs + needs: [ pre-commit ] + uses: ./.github/workflows/step_docs.yaml + + build: + needs: [ pre-commit ] + uses: ./.github/workflows/step_build.yaml + + static-analysis: + needs: [ pre-commit ] + uses: ./.github/workflows/step_static-analysis.yaml secrets: QODANA_TOKEN: ${{ secrets.QODANA_TOKEN }} - permissions: - security-events: write - contents: write - checks: write - pull-requests: write - - build-wheel: - uses: ./.github/workflows/step_build-wheel.yaml - needs: [ tests ] - with: - upload: ${{ inputs.upload-wheel || false }} + if: github.event_name != 'schedule' pass: - name: Pass - needs: [ pre-commit, tests, build-wheel, code-analysis ] + name: โœ… Pass + needs: [ pre-commit, tests, coverage, docs, build, static-analysis ] runs-on: ubuntu-latest steps: - uses: re-actors/alls-green@release/v1 with: - allowed-skips: code-analysis + allowed-skips: coverage, static-analysis jobs: ${{ toJSON(needs) }} if: always() diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d371ef3..57b7278 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,4 +1,7 @@ -name: Prepare release +name: ๐Ÿš€ Release +run-name: > + ๐Ÿš€ Release + ${{ github.ref_name }} on: push: @@ -6,28 +9,24 @@ on: - "v[0-9]+.[0-9]+.[0-9]+" - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" -permissions: - contents: read - jobs: tests: uses: ./.github/workflows/step_test.yaml build-wheel: - needs: [ tests ] - uses: ./.github/workflows/step_build-wheel.yaml + uses: ./.github/workflows/step_build.yaml upload_pypi: name: Upload to PyPI repository needs: [ tests, build-wheel ] runs-on: ubuntu-latest environment: name: pypi - url: https://pypi.org/project/spglib/ + url: https://pypi.org/project/fmf-jinja/ permissions: id-token: write steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: - name: artifact + name: Packages path: dist - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 @@ -37,7 +36,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: softprops/action-gh-release@v1 + - uses: softprops/action-gh-release@v2 with: name: FMF-Jinja ${{ github.ref_name }} draft: true diff --git a/.github/workflows/step_build-wheel.yaml b/.github/workflows/step_build-wheel.yaml deleted file mode 100644 index 03ba7ae..0000000 --- a/.github/workflows/step_build-wheel.yaml +++ /dev/null @@ -1,23 +0,0 @@ -on: - workflow_call: - inputs: - upload: - required: false - type: boolean - default: true - description: Upload wheel as artifact - -permissions: - contents: read - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Build package - run: pipx run build - - uses: actions/upload-artifact@v3 - with: - path: dist/* - if: ${{ inputs.upload }} diff --git a/.github/workflows/step_build.yaml b/.github/workflows/step_build.yaml new file mode 100644 index 0000000..b926d23 --- /dev/null +++ b/.github/workflows/step_build.yaml @@ -0,0 +1,15 @@ +on: + workflow_call: + +permissions: + contents: read + +jobs: + build: + name: ๐Ÿ sdist/wheel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: hynek/build-and-inspect-python-package@v2 diff --git a/.github/workflows/step_code-analysis.yaml b/.github/workflows/step_code-analysis.yaml deleted file mode 100644 index 62ac654..0000000 --- a/.github/workflows/step_code-analysis.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Static code analysis -on: - workflow_call: - secrets: - QODANA_TOKEN: - required: false - description: Qodana token - -permissions: - security-events: write - contents: write - checks: write - pull-requests: write - -jobs: - qodana: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: 3.x - cache: pip - - name: Qodana Scan - uses: JetBrains/qodana-action@v2023.2 - env: - QODANA_TOKEN: ${{ secrets.QODANA_TOKEN }} - - name: Upload to GitHub code scanning - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: ${{ runner.temp }}/qodana/results/qodana.sarif.json - if: false diff --git a/.github/workflows/step_coverage.yaml b/.github/workflows/step_coverage.yaml new file mode 100644 index 0000000..39ad61d --- /dev/null +++ b/.github/workflows/step_coverage.yaml @@ -0,0 +1,28 @@ +name: ๐Ÿ‘€ coverage + +on: + workflow_call: + secrets: + CODECOV_TOKEN: + description: Codecov token of the main repository + required: false + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Install package + run: pip install -e .[test-cov] + - name: Test package + run: pytest --cov --cov-report=xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/step_docs.yaml b/.github/workflows/step_docs.yaml new file mode 100644 index 0000000..d299e9d --- /dev/null +++ b/.github/workflows/step_docs.yaml @@ -0,0 +1,33 @@ +on: + workflow_call: + +permissions: + contents: read + +jobs: + sphinx: + name: Sphinx (${{ matrix.builder }}) + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental || false }} + strategy: + fail-fast: false + matrix: + builder: [ linkcheck, html ] + include: + # Run default html builder with warnings as error + - builder: html + args: -W + # TODO: warnings builder is experimental due to missing sphinx-autodoc support + # https://github.com/sphinx-doc/sphinx/issues/9813 + # https://github.com/sphinx-doc/sphinx/issues/11991 + experimental: true + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + cache: pip + - name: Install the project and docs dependencies + run: pip install -e .[docs] + - name: Run sphinx builder ${{ matrix.builder }} + run: sphinx-build -b ${{ matrix.builder }} ${{ matrix.args }} ./docs ./docs/_build diff --git a/.github/workflows/step_pre-commit.yaml b/.github/workflows/step_pre-commit.yaml index 0831d78..ef891b7 100644 --- a/.github/workflows/step_pre-commit.yaml +++ b/.github/workflows/step_pre-commit.yaml @@ -1,5 +1,3 @@ -name: pre-commit - on: workflow_call: @@ -12,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.x - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/step_static-analysis.yaml b/.github/workflows/step_static-analysis.yaml new file mode 100644 index 0000000..f819cc0 --- /dev/null +++ b/.github/workflows/step_static-analysis.yaml @@ -0,0 +1,19 @@ +name: static-analysis + +on: + workflow_call: + secrets: + QODANA_TOKEN: + required: true + +jobs: + qodana: + name: Qodana + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: JetBrains/qodana-action@v2024.2 + env: + QODANA_TOKEN: ${{ secrets.QODANA_TOKEN }} diff --git a/.github/workflows/step_test.yaml b/.github/workflows/step_test.yaml index 686c3cb..c612755 100644 --- a/.github/workflows/step_test.yaml +++ b/.github/workflows/step_test.yaml @@ -1,33 +1,47 @@ on: workflow_call: inputs: - pytest-flags: - type: string - required: false - description: Additional flags to add to pytest + mask-experimental: + type: boolean + default: true + description: Always report experimental test as successful permissions: contents: read jobs: checks: - name: - Check ๐Ÿ ${{ matrix.python-version }} + name: > + ๐Ÿ ${{ matrix.python-version }} + ๐ŸŒณ ${{ matrix.tmt-version || 'latest' }} + โ›ฉ๏ธ ${{ matrix.jinja-version || 'latest' }} + ${{ matrix.experimental && '[๐Ÿงช Experimental]' || '' }} runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental || false }} strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: [ "3.9", "3.x" ] + tmt-version: [ "" ] + jinja-version: [ "" ] + include: + - python-version: "3.x" + jinja-version: "main" + tmt-version: "main" + experimental: true steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install package - run: pip install -e .[test-cov] + run: pip install -e .[test] + - name: Install fmf ${{ matrix.tmt-version }} + run: pip install fmf@git+https://github.com/teemtee/fmf@${{ matrix.tmt-version }} + if: matrix.tmt-version + - name: Install jinja ${{ matrix.jinja-version }} + run: pip install jinja2@git+https://github.com/pallets/jinja/@${{ matrix.tmt-version }} + if: matrix.jinja-version - name: Test package - run: pytest --cov --cov-report=xml ${{ inputs.pytest-flags }} - - name: Upload coverage report - uses: codecov/codecov-action@v3 - with: - name: python-${{ matrix.python-version }} + run: pytest + continue-on-error: ${{ matrix.experimental && inputs.mask-experimental}} diff --git a/.gitignore b/.gitignore index bfb44c7..d7f2ec0 100644 --- a/.gitignore +++ b/.gitignore @@ -241,3 +241,8 @@ cython_debug/ ### Project specific /src/fmf_jinja/_version.py + +# RPM spec file +!/.distro/*.spec +*.tar.gz +*.rpm diff --git a/.idea/fmf-jinja.iml b/.idea/fmf-jinja.iml deleted file mode 100644 index ce865b5..0000000 --- a/.idea/fmf-jinja.iml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index ca4933c..0000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index cc5462d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 470b4f9..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 295ca3a..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 5ace414..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/.packit.yaml b/.packit.yaml new file mode 100644 index 0000000..1367699 --- /dev/null +++ b/.packit.yaml @@ -0,0 +1,50 @@ +files_to_sync: + - src: .distro/ + dest: ./ + delete: true + filters: + - "protect .git*" + - "protect sources" + - "- plans/rpminspect.fmf" + - "- plans/rpmlint.fmf" + +upstream_package_name: fmf-jinja +downstream_package_name: fmf-jinja +specfile_path: .distro/fmf-jinja.spec + +upstream_tag_template: v{version} +targets: + - fedora-all + +jobs: + - &copr + job: copr_build + trigger: pull_request + - &tests + job: tests + trigger: pull_request + fmf_path: .distro + - <<: *copr + trigger: commit + project: nightly + branch: main + - <<: *tests + trigger: commit + branch: main + - <<: *copr + trigger: release + project: release + - <<: *tests + trigger: release + - job: propose_downstream + trigger: release + dist_git_branches: + - fedora-rawhide + - job: koji_build + trigger: commit + dist_git_branches: + - fedora-all + - job: bodhi_update + trigger: commit + dist_git_branches: + - fedora-branched diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48f60fc..81546f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,10 +25,12 @@ repos: rev: v1.5.1 hooks: - id: mypy + # TODO: Fix mypy stuff + stages: [ manual ] files: ^(src|test) additional_dependencies: - - Click - - fmf @ git+https://github.com/LecrisUT/fmf@fmf-jinja + - click + - fmf - jinja2 - pytest diff --git a/.readthedocs.yml b/.readthedocs.yml index 89b52a0..7318c8d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,9 +1,9 @@ version: 2 build: - os: ubuntu-22.04 + os: ubuntu-lts-latest tools: - python: "3.11" + python: latest sphinx: configuration: docs/conf.py diff --git a/README.md b/README.md index f48c5a3..27fe839 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,64 @@ # FMF-Jinja -[![Documentation Status](https://readthedocs.org/projects/fmf-jinja/badge/?version=latest)](https://fmf-jinja.readthedocs.io/en/latest/?badge=latest) -[![CI](https://github.com/LecrisUT/fmf-jinja/actions/workflows/ci.yaml/badge.svg)](https://github.com/LecrisUT/fmf-jinja/actions/workflows/ci.yaml) -[![codecov](https://codecov.io/github/LecrisUT/fmf-jinja/graph/badge.svg?token=WCTLWU6M2O)](https://codecov.io/github/LecrisUT/fmf-jinja) +[![CI Status][ci-badge]][ci-link] +[![Codecov Status][codecov-badge]][codecov-link] + +[![Documentation Status][rtd-badge]][rtd-link] -Templating engine using [jinja files](https://jinja.palletsprojects.com) and -[fmf metadata](https://fmf.readthedocs.io). +[Jinja templating engine][jinja] using [fmf metadata][fmf]. ## Concept -If you've found this project, chances are you already know about either -the FMF or Jinja project. But for the sake of redundancy: +The scope of this project is to take a templated folder and generate *multiple* output +folders with relation to one another. Consider the following fmf file example in +[`example/minimal`]: -[Jinja](https://jinja.palletsprojects.com) is a popular templating engine that -enables one to substitute text from JSON/YAML/Python like objects into a template -file/string. +```yaml +var1: 42 +var2: Default value -[FMF](https://fmf.readthedocs.io) is an extension to YAML files that incorporates -a file structure format and inheritance of the dictionary variables from the parent -path to the children. +/rootA: +/rootB: + var2: Overwritten +``` -This project combines the two such that you can maintain your metadata in a folder -structure with minimal effort and generate any necessary data folder structure you -desire. +This is interpreted by fmf as: -## Minimum example +```console +$ fmf show --path example/minimal +/rootA +var1: 42 +var2: Default value -See [`example/simple`](/example/simple) for a minimal example. -The output can then be generated using the python function `fmf_jinja.template.generate` -or using the cli interface `fmf-jinja`: +/rootB +var1: 42 +var2: Overwritten +``` + +These variables (`var1`, `var2`) are then used as variables inside a jinja template +creating templated folders under `rootA` and `rootB` with their respective values. Try +it out by running ```console -$ fmf-jinja --root=example/simple generate --output=out -$ tree ./out -./out -โ”œโ”€โ”€ rootA -โ”‚ โ”œโ”€โ”€ file.yaml -โ”‚ โ””โ”€โ”€ some_file.yaml -โ””โ”€โ”€ rootB - โ”œโ”€โ”€ file.yaml - โ””โ”€โ”€ some_file.yaml -$ cat ./out/rootA/template_file.yaml -my_var: A +$ fmf-jinja -r example/minimal generate -o /path/to/some/output/folder ``` + +To appreciate the full capabilities see the [fmf features] and [jinja template guide]. +Also check the [online documentation] for more examples and detailed usage guide. + +[ci-badge]: https://github.com/LecrisUT/fmf-jinja/actions/workflows/ci.yaml/badge.svg?branch=main&event=push +[ci-link]: https://github.com/LecrisUT/fmf-jinja/actions?query=branch%3Amain+event%3Apush +[codecov-badge]: https://codecov.io/gh/LecrisUT/fmf-jinja/graph/badge.svg?token=WCTLWU6M2O +[codecov-link]: https://codecov.io/gh/LecrisUT/fmf-jinja +[fmf]: https://fmf.readthedocs.io +[fmf features]: https://fmf.readthedocs.io/en/stable/features.html +[jinja]: https://jinja.palletsprojects.com +[jinja template guide]: https://jinja.palletsprojects.com/en/stable/templates/ +[online documentation]: https://fmf-jinja.readthedocs.io/ +[rtd-badge]: https://readthedocs.org/projects/fmf-jinja/badge/?version=latest +[rtd-link]: https://fmf-jinja.readthedocs.io/en/latest/?badge=latest +[`example/minimal`]: example/minimal diff --git a/docs/cli_api/fmf-jinja/generate.md b/docs/cli_api/fmf-jinja/generate.md new file mode 100644 index 0000000..fb6e71d --- /dev/null +++ b/docs/cli_api/fmf-jinja/generate.md @@ -0,0 +1,6 @@ +# `fmf-jinja generate` + +```{eval-rst} +.. click:: fmf_jinja.cli:generate + :prog: fmf-jinja generate +``` diff --git a/docs/cli_api/fmf-jinja/index.md b/docs/cli_api/fmf-jinja/index.md new file mode 100644 index 0000000..9eb5b81 --- /dev/null +++ b/docs/cli_api/fmf-jinja/index.md @@ -0,0 +1,12 @@ +# `fmf-jinja` + +```{toctree} +:hidden: true + +generate +``` + +```{eval-rst} +.. click:: fmf_jinja.cli:main + :prog: fmf-jinja +``` diff --git a/docs/cli_api/index.md b/docs/cli_api/index.md new file mode 100644 index 0000000..87fb817 --- /dev/null +++ b/docs/cli_api/index.md @@ -0,0 +1,7 @@ +# CLI reference + +```{toctree} +:maxdepth: 1 + +fmf-jinja/index +``` diff --git a/docs/conf.py b/docs/conf.py index bb96806..78f7e24 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,54 +1,51 @@ # noqa: D100 from __future__ import annotations -# -- Project information ----------------------------------------------------- +import os project = "FMF-Jinja" author = "Cristian Le" - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. extensions = [ "myst_parser", "sphinx.ext.intersphinx", + "sphinx_tippy", + "sphinx.ext.autodoc", + "sphinx.ext.extlinks", + "sphinx_autodoc_typehints", + "sphinx_click", ] - -# Add any paths that contain templates here, relative to this directory. templates_path = [] - source_suffix = [".md"] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [ - "_build", - "**.ipynb_checkpoints", - "Thumbs.db", - ".DS_Store", - ".env", - ".venv", -] - -linkcheck_anchors_ignore = [ - # This seems to be broken on GitHub readmes - "default-versioning-scheme", - "git-archives", -] -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# html_theme = "furo" -# -- Extension configuration ------------------------------------------------- myst_enable_extensions = [ "colon_fence", "substitution", "deflist", + "attrs_block", + "dollarmath", ] +intersphinx_mapping = { + "python": ("https://docs.python.org/3.13/", None), + "tmt": ("https://tmt.readthedocs.io/en/stable", None), + "fmf": ("https://fmf--257.org.readthedocs.build/en/257", None), + "jinja": ("https://jinja.palletsprojects.com/en/stable", None), +} +tippy_rtd_urls = [ + # Only works with RTD hosted intersphinx + "https://tmt.readthedocs.io/en/stable", + "https://fmf--257.org.readthedocs.build/en/257", + "https://jinja.palletsprojects.com/en/stable", +] +autodoc_member_order = "bysource" + +repo_slug = os.getenv("GITHUB_REPOSITORY", "LecrisUT/fmf-jinja") +# Using `GITHUB_REF` is not reliable for the `path` links +git_ref = os.getenv("GITHUB_SHA", "main") + +extlinks = { + "issue": ("https://github.com/LecrisUT/fmf-jinja/issues/%s", "issue %s"), + "path": (f"https://github.com/{repo_slug}/tree/{git_ref}/%s", "%s"), + "user": ("https://github.com/%s", "%s"), +} diff --git a/docs/example/index.md b/docs/example/index.md index ac27c54..f555ff6 100644 --- a/docs/example/index.md +++ b/docs/example/index.md @@ -1,10 +1,9 @@ # Examples -:::{toctree} ---- -maxdepth: 1 -titlesonly: true -glob: true ---- -./* -::: +You can find example templates in the {path}`example` folder + +```{toctree} +minimal +simple +recursive +``` diff --git a/docs/example/minimal.md b/docs/example/minimal.md new file mode 100644 index 0000000..1c07fbb --- /dev/null +++ b/docs/example/minimal.md @@ -0,0 +1,44 @@ +# Minimal + +In the minimal example {term}`vars` is not defined and all keys in the fmf file are +used as Jinja variables. By default, the fmf root (`/`) is used as a template folder +unless {term}`templates` key is specified. + +The example in {path}`example/minimal` contains a minimal template format for +`fmf-jinja`. Inside we have a fmf tree defined by: +```{literalinclude} ../../example/minimal/main.fmf +:language: yaml +:caption: /main.fmf +``` +The template folder contains a file `common_file.txt` that is simply copied over and +a template file `file.yaml.j2` which generates a `file.yaml`. +```{literalinclude} ../../example/minimal/file.yaml.j2 +:language: jinja +:caption: /file.yaml.j2 +``` + +Running `fmf-jinja` on this example we get: +```console +$ fmf-jinja -r example/minimal generate -o output +$ tree -a output +output/ +โ”œโ”€โ”€ .fmf +โ”‚ โ””โ”€โ”€ version +โ”œโ”€โ”€ main.fmf +โ”œโ”€โ”€ rootA +โ”‚ โ”œโ”€โ”€ common_file.txt +โ”‚ โ””โ”€โ”€ file.yaml +โ””โ”€โ”€ rootB + โ”œโ”€โ”€ common_file.txt + โ””โ”€โ”€ file.yaml +$ tail output/root*/file.yaml +==> output/rootA/file.yaml <== +var0: random data +var1: 42 +var2: Default value + +==> output/rootB/file.yaml <== +var0: random data +var1: 42 +var2: Overwritten +``` diff --git a/docs/example/recursive.md b/docs/example/recursive.md new file mode 100644 index 0000000..12c7622 --- /dev/null +++ b/docs/example/recursive.md @@ -0,0 +1,42 @@ +# Recursive + +The example in {path}`example/recursive` showcases how `fmf-jinja` can run recursively +in order to support more complex template outputs. The initial template defines +```{literalinclude} ../../example/recursive/main.fmf +:language: yaml +:caption: /main.fmf +``` +Which is used to generate a new fmf tree from +```{literalinclude} ../../example/recursive/template_fmf/main.fmf.j2 +:language: jinja +:caption: /template_fmf/main.fmf +``` +which overwrites the original `main.fmf` file. + +The second run starts from the generated files: +```{literalinclude} ../../test/data/output/recursive/main.fmf +:language: yaml +:caption: /main.fmf +``` +This overcomes a limitation in `fmf` that glob patterns are not supported that would +otherwise allow cleaner design of the fmf tree without requiring recursive runs. + +Putting all together, running `fmf-jinja` on this example we get: +```console +$ fmf-jinja -r example/recusive generate -o output +$ tree output +output/ +โ”œโ”€โ”€ .fmf +โ”‚ โ””โ”€โ”€ version +โ”œโ”€โ”€ A_2 +โ”‚ โ”œโ”€โ”€ B_x +โ”‚ โ”‚ โ””โ”€โ”€ file.yaml +โ”‚ โ””โ”€โ”€ B_y +โ”‚ โ””โ”€โ”€ file.yaml +โ”œโ”€โ”€ A_3 +โ”‚ โ”œโ”€โ”€ B_x +โ”‚ โ”‚ โ””โ”€โ”€ file.yaml +โ”‚ โ””โ”€โ”€ B_y +โ”‚ โ””โ”€โ”€ file.yaml +โ””โ”€โ”€ main.fmf +``` diff --git a/docs/example/simple.md b/docs/example/simple.md index 74fce9d..13e5ac0 100644 --- a/docs/example/simple.md +++ b/docs/example/simple.md @@ -1,17 +1,40 @@ -# Minimal simple example +# Simple -The minimal example +The more recommended design is to explicitly specify the Jinja variables with +{term}`vars` and template folder with {term}`templates` keys. This allows to use all +other keys in the [Template schema]. -Template: -:::{literalinclude} ../../example/simple/template/file.yaml.j2 ---- -language: yaml ---- -::: +The example in {path}`example/simple` contains a simple template with all supported +functionalities. The fmf tree is defined by: +```{literalinclude} ../../example/simple/main.fmf +:language: yaml +:caption: /main.fmf +:emphasize-lines: 8-16 +``` +This is identical to the [minimal] example, with additional keys used in `/rootA`. The +changes in `/rootA` are: +- create a symbolic link to a generated file in `/rootB` +- copy/rename `common_file.txt` to `renamed_file.txt` +- exclude `common_file.txt` from being copied -FMF metdata -:::{literalinclude} ../../example/simple/main.fmf ---- -language: yaml ---- -::: +Running `fmf-jinja` on this example we get: +```{code-block} console +:emphasize-lines: 8,10 + +$ fmf-jinja -r example/simple generate -o output +$ tree output +output/ +โ”œโ”€โ”€ .fmf +โ”‚ โ””โ”€โ”€ version +โ”œโ”€โ”€ main.fmf +โ”œโ”€โ”€ rootA +โ”‚ โ”œโ”€โ”€ fileB.yaml -> ../rootB/file.yaml +โ”‚ โ”œโ”€โ”€ file.yaml +โ”‚ โ””โ”€โ”€ renamed_file.txt +โ””โ”€โ”€ rootB + โ”œโ”€โ”€ common_file.txt + โ””โ”€โ”€ file.yaml +``` + +[Template schema]: ../usage/index.md#template-schema +[minimal]: minimal.md diff --git a/docs/example/symlink.md b/docs/example/symlink.md deleted file mode 100644 index b2162ef..0000000 --- a/docs/example/symlink.md +++ /dev/null @@ -1,3 +0,0 @@ -# Symlink example - -TBD diff --git a/docs/index.md b/docs/index.md index 6c10f6d..182c208 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,16 +1,23 @@ # FMF-Jinja -:::{toctree} ---- -hidden: true -glob: true ---- +```{toctree} +:hidden: true + +self +usage/index +why example/index -::: +usecase/index +cli_api/index +python_api/index +``` + +```{include} ../README.md +:start-after: +:end-before: +``` + -:::{include} ../README.md ---- -start-after: -end-before: ---- -::: +[jinja]: inv:jinja#index +[fmf]: inv:fmf#index +[`example/minimal`]: example/minimal.md diff --git a/docs/python_api/fmf_jinja/cli/index.md b/docs/python_api/fmf_jinja/cli/index.md new file mode 100644 index 0000000..ef0fd2a --- /dev/null +++ b/docs/python_api/fmf_jinja/cli/index.md @@ -0,0 +1,13 @@ +# `fmf_jinja.cli` + +```{eval-rst} +.. automodule:: fmf_jinja.cli +``` + +## Subpackages + +```{toctree} +:maxdepth: 1 + +main +``` diff --git a/docs/python_api/fmf_jinja/cli/main.md b/docs/python_api/fmf_jinja/cli/main.md new file mode 100644 index 0000000..8ad064b --- /dev/null +++ b/docs/python_api/fmf_jinja/cli/main.md @@ -0,0 +1,9 @@ +# `fmf_jinja.cli.main` + +```{eval-rst} +.. automodule:: fmf_jinja.cli.main + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource +``` diff --git a/docs/python_api/fmf_jinja/generators.md b/docs/python_api/fmf_jinja/generators.md new file mode 100644 index 0000000..15bc23d --- /dev/null +++ b/docs/python_api/fmf_jinja/generators.md @@ -0,0 +1,8 @@ +# `fmf_jinja.generators` + +```{eval-rst} +.. automodule:: fmf_jinja.generators + :members: + :undoc-members: + :show-inheritance: +``` diff --git a/docs/python_api/fmf_jinja/index.md b/docs/python_api/fmf_jinja/index.md new file mode 100644 index 0000000..91f8065 --- /dev/null +++ b/docs/python_api/fmf_jinja/index.md @@ -0,0 +1,16 @@ +# `fmf_jinja` + +```{eval-rst} +.. automodule:: fmf_jinja +``` + +## Subpackages + +```{toctree} +:maxdepth: 1 + +cli/index +generators +template +utils +``` diff --git a/docs/python_api/fmf_jinja/template.md b/docs/python_api/fmf_jinja/template.md new file mode 100644 index 0000000..d9eaaf4 --- /dev/null +++ b/docs/python_api/fmf_jinja/template.md @@ -0,0 +1,8 @@ +# `fmf_jinja.template` + +```{eval-rst} +.. automodule:: fmf_jinja.template + :members: + :undoc-members: + :show-inheritance: +``` diff --git a/docs/python_api/fmf_jinja/utils.md b/docs/python_api/fmf_jinja/utils.md new file mode 100644 index 0000000..d273b07 --- /dev/null +++ b/docs/python_api/fmf_jinja/utils.md @@ -0,0 +1,9 @@ +# `fmf_jinja.utils` + +```{eval-rst} +.. automodule:: fmf_jinja.utils + :members: + :undoc-members: + :private-members: + :show-inheritance: +``` diff --git a/docs/python_api/index.md b/docs/python_api/index.md new file mode 100644 index 0000000..bf51e29 --- /dev/null +++ b/docs/python_api/index.md @@ -0,0 +1,7 @@ +# Python API reference + +```{toctree} +:maxdepth: 1 + +fmf_jinja/index +``` diff --git a/docs/usage/advanced.md b/docs/usage/advanced.md new file mode 100644 index 0000000..b747d3e --- /dev/null +++ b/docs/usage/advanced.md @@ -0,0 +1,85 @@ +# Advanced usage + +Here are some more advanced design patterns that you can take advantage of. + +## Recursive + +By default, the fmf files are copied over in the output, and if there are new `.fmf` +files created, the generation is re-run. This allows to create arbitrarily complex +template structures. The simplest example here is creating 2D array of outputs: + +```{code-block} yaml +:caption: /main.fmf + +vars: + var1: [2, 3] + var2: [x, y] +templates: + path: /template +``` +```{code-block} jinja +:caption: /template/main.fmf.j2 + +{% for A in var1 -%} +/A_{{ A }}: + varA: {{ A }} +{% for B in var2 -%} +/A_{{ A }}/B_{{ B }}: + varB: {{ B }} +{% endfor -%} +{% endfor -%} +``` +Which expands to an intermediate fmf tree of +```{code-block} yaml +:caption: /main.fmf + +/A_2: + varA: 2 +/A_2/B_x: + varB: x +/A_2/B_y: + varB: y +/A_3: + varA: 3 +/A_3/B_x: + varB: x +/A_3/B_y: + varB: y +``` +```console +$ fmf show +/A_2/B_x +varA: 2 +varB: x + +/A_2/B_y +varA: 2 +varB: y + +/A_3/B_x +varA: 3 +varB: x + +/A_3/B_y +varA: 3 +varB: y +``` + +:::{note} +If a templated file like `main.fmf.j2` clashes with an original `main.fmf` file in the +original fmf tree, the generated file overwrites the +::: + +The recursive feature can be turned off with the `--no-recursive` flag. + +### `fmf-jinja` bomb + +If you want an equivalent [fork bomb] while running `fmf-jinja`, here you are: +```console +$ fmf init +$ touch main.fmf +$ ln -s main.fmf main.fmf.j2 +$ fmf-jinja generate -o whatever +``` + +[fork bomb]: https://en.wikipedia.org/wiki/Fork_bomb diff --git a/docs/usage/fmf.md b/docs/usage/fmf.md new file mode 100644 index 0000000..301aa91 --- /dev/null +++ b/docs/usage/fmf.md @@ -0,0 +1,152 @@ +# Fmf metadata + +## Basics + +If you are new to fmf files, these are basically [yaml] files with a file-structure +hierarchy. I.e. a fmf file like: +```{code-block} yaml +:name: basics/main.fmf +:caption: /main.fmf +:emphasize-lines: 4,5 + +var1: 42 +var2: Default value + +/rootA: +/rootB: + var2: Overwritten +``` +is interpreted as if you had 2 fmf/yaml files +```{code-block} yaml +:caption: /rootA/main.fmf + +var1: 42 +var2: Default value +``` +```{code-block} yaml +:caption: /rootB/main.fmf +:emphasize-lines: 2 + +var1: 42 +var2: Overwritten +``` +Notice how the keys starting with `/` are treated as paths describing the tree +structure of the fmf tree as if it was a file structure, and how the variables are +inherited and overwritten. + +:::{caution} +Crucially for a fmf tree to be valid, you have to define where the root of the tree is +located. This is done by adding a `.fmf/version` text file (with the content `1`) at +the root directory of the fmf tree. If there are any other `.fmf/version` somewhere in +the subdirectory hierarchy, the current tree branch will terminate there and a new tree +starts. +::: + +## In the `fmf-jinja` context + +From the fmf tree format we are using two aspects: +1. The paths of the fmf tree nodes (only the final leaves) represent the paths of each + output folders where the templated folder is generated. + + Use `fmf ls` to see the tree structure. + +2. The contents of the fmf tree nodes contain the instructions of how to generate the + current output folder. + + Use `fmf show` to see the contents of the tree. + +## Other useful features + +The reason why fmf format is used for this project is its various [features]. Here are +some of the most common features that you should consider + +### [Inheritance] + +The fmf data is inherited from the top of the tree downwards as shown in the +[Basics example], greatly reducing the deduplication of variable and avoiding the need +of yaml-anchors. + +If needed, the inheritance can be turned off by adding the following to the tree node's +data: +```yaml +/: + inherit: false +``` + +:::{note} +In this case `/` is treated as a special path pointing to the current node where +containing the fmf directives of the current node. Here we are turning off the +inheritance. +::: + +### [Merging] + +By default, when a key is redefined further down the hierarchy, this key-value is +overwritten by the most recent value, as we saw in the [Basics example]. On top of that +fmf supports merging operations defined by appending an operator (`+`, `-`, `+<`, etc.) +to the key and performing an operation like [`dict.update()`]. For example the +following fmf file +```{code-block} yaml +:caption: /main.fmf +:emphasize-lines: 6-8 + +vars: + var1: 42 + var2: Default value + +/merged: + vars+: + var1+: 378 + var3: New one +``` +evaluates to +```{code-block} yaml +:caption: /merged/main.fmf + +vars: + var1: 420 + var2: Default value + var3: New one +``` + +:::{note} +The merging operations take into account the yaml type of the original value. For +example `+` operation can be `dict.update()`, `list.append()` or `str.__add__()`. +::: + +### [Scatter] + +Instead of defining one monolithic `main.fmf` file, you can instead redistribute the +fmf nodes into their equivalent fmf files in the file structure. Specifically, the +following fmf files are equivalent: + + +```{code-block} yaml +:caption: /main.fmf + +/rootA: + var1: 42 + var2: Default value +``` +```{code-block} yaml +:caption: /rootA.fmf + +var1: 42 +var2: Default value +``` +```{code-block} yaml +:caption: /rootA/main.fmf + +var1: 42 +var2: Default value +``` + +Note how `main.fmf` behaves like `index.html` on website or `.` in unix paths. + +[yaml]: https://en.wikipedia.org/wiki/YAML +[features]: inv:fmf#features +[Inheritance]: inv:fmf:std:label#features:inheritance +[Merging]: inv:fmf:std:label#features:merging +[Scatter]: inv:fmf:std:label#features:scatter +[Basics example]: #basics/main.fmf +[`dict.update()`]: inv:python:py:method#dict.update diff --git a/docs/usage/index.md b/docs/usage/index.md new file mode 100644 index 0000000..473974b --- /dev/null +++ b/docs/usage/index.md @@ -0,0 +1,83 @@ +# Usage + +```{toctree} +:hidden: true + +fmf +jinja +advanced +``` + +The simplest design shown on the [top page] and [minimal example] is to simply store +all Jinja variables in the fmf file and use the current directory as the templated +folder that will be generated. + +For more complex structure see the following breakdown of supported variables + +## Template schema + +{.glossary} +`templates` +: *Default*: `"/"` + + *Type*: `Path | Template | list[Path | Template]` + + Define the template source(s) under the current node's output path. Can be either a + list or a single item and the item(s) can be either strings or an object. If the item + is a string it uses the default object with modified {term}`path ` + variable. + +`templates[].path` +: *Required* + + *Type*: `Path` + + Path to the template file or folder to generate under the current node's output. + + If the path is absolute, it is evaluated from the current fmf root directory, + otherwise it is evaluated from the current node's fmf path. + + If the path points to a folder, the whole folder is processed recursively except for + the items in {term}`exclude `. The files are either rendered + using Jinja if the file extension ends in `.j2` (stripping the final `.j2` extension + in the output file), or otherwise copied over. Symbolic links are recreated, + reinterpreting the target's absolute/relative path as described in the previous + paragraph. + +`templates[].exclude` +: *Default*: `[]` + + *Type*: `list[Path]` + + List of paths to exclude from being rendered or copied. + +`vars` +: *Special* + + *Type*: `dict[str,Any]` + + Variables used in the Jinja template files. + + If the key is not specified, keys in the fmf object except for {term}`templates` are + treated as the contents for `vars`. + +`links` +: *Default*: `{}` + + *Type*: `dict[Path,Path]` + + Dictionary of symbolic links to generate, the key being path to the symbolic link + created and the value being the path to the target. The absolute/relative paths are + treated similarly to the {term}`path ` key. + +`copy` +: *Default*: `{}` + + *Type*: `dict[Path,Path]` + + Dictionary of files/folders to copy, the key being the destination file/folder and + the value being the original file/folder. The absolute/relative paths are treated + similarly to the {term}`path ` key. + +[top page]: ../index.md +[minimal example]: ../example/minimal.md diff --git a/docs/usage/jinja.md b/docs/usage/jinja.md new file mode 100644 index 0000000..53beff4 --- /dev/null +++ b/docs/usage/jinja.md @@ -0,0 +1,27 @@ +# Jinja templates + +Jinja is a standard templating engine, and we cannot adequately cover its basics here. +Refer to Jinja's [Template Designer Guide] instead. + +## In the `fmf-jinja` context + +The jinja files are handled as follows: +1. Files ending with a `.j2` extensions are treated as jinja template files with the + generated file having the same path and filename except for the final `.j2` + extension that is stripped. +2. The variables used in the jinja template are determined by the {term}`vars` variable + of the current fmf node. +3. `{{ }}` syntax in the path files are **not** expanded. Use the [recursive feature] + instead. + +## Additional jinja extensions + +Currently `fmf-jinja` does not provide additional filters and functions, but some are +being planned. + +## Extending jinja environment + +Currently, there is no mechanism to extend the {py:class}`jinja2.Environment` used. + +[Template Designer Guide]: inv:jinja#templates +[recursive feature]: advanced.md#recursive diff --git a/docs/usecase/index.md b/docs/usecase/index.md new file mode 100644 index 0000000..00421c4 --- /dev/null +++ b/docs/usecase/index.md @@ -0,0 +1,12 @@ +# Use cases + +Here you can find a few practical use-cases of `fmf-jinja`: + +```{toctree} +:maxdepth: 1 + +scientific +tmt +``` + +If you have any further suggestions and ideas, feel free to extend this. diff --git a/docs/usecase/scientific.md b/docs/usecase/scientific.md new file mode 100644 index 0000000..e2671fe --- /dev/null +++ b/docs/usecase/scientific.md @@ -0,0 +1,198 @@ +# Scientific calculation + +The majority of scientific programs have highly specific workflows with custom input +file formats, specific file structure formats and little workflow support for chaining +calculations. This project can help bridge the limitations of the scientific program. + +In this example we will take `quantum-espresso` as an example. + +## Use-case: Running an array of calculations + +A simple example to explore here is running convergence calculations e.g. varying the +$k$-point grid: +```{code-block} jinja +:caption: /template/pw.scf.in.j2 + +&CONTROL + calculation = 'scf', + prefix = 'silicon', + outdir = './tmp/' +/ + +&SYSTEM + ibrav = 2, + celldm(1) = 10.26, + nat = 2, + ntyp = 1, + ecutwfc = {{ ecutwfc }}, + nbnd = 8 +/ + +&ELECTRONS +/ + +ATOMIC_SPECIES + Si 28.086 Si.pz-vbc.UPF + +ATOMIC_POSITIONS (alat) + Si 0.0 0.0 0.0 + Si 0.25 0.25 0.25 + +K_POINTS (automatic) + {{ n_k }} {{ n_k }} {{ n_k }} 0 0 0 +``` +```{code-block} yaml +:caption: main.fmf + +templates: /template +vars: + ecutwfc: 30 +/k_2: + vars+: + n_k: 2 +/k_4: + vars+: + n_k: 4 +/k_6: + vars+: + n_k: 6 +``` + +Since fmf structure has a filestructure, it is easy to loop over all variants and +perform the calculations, e.g. a full workflow can look like: +```console +$ fmf-jinja generate -o qe-run +$ pushd qe-run +$ for path in $(fmf ls); do +> pushd "$(pwd)$path" +> pw.x < pw.scf.in > pw.scf.out +> popd +> done +$ popd +``` + +## Use-case: Linking calculations + +Traditionally most scientific programs assume you run each step of the calculation in +the same directory, but this can easily explode the complexity of the folders making it +hard to track what files come from where. Here we recommend using symlinks to the +necessary files of the previous step. + +Here we will give an example of calculating the band structure and here we will show +how you can take full advantage of the Jinja templating engine. + +Here we will show how you can take full advantage of the Jinja templating engine. We +start from a base template containing the common general data +```{code-block} jinja +:caption: /template/base.j2 + +{% block Control %} +&CONTROL + calculation = '{% block Calculation required%}{% endblock +%}', + prefix = 'silicon', + outdir = './tmp/' +/ +{% endblock %} + +{% block System %} +&SYSTEM + ibrav = 2, + celldm(1) = 10.26, + nat = 2, + ntyp = 1, + ecutwfc = {{ ecutwfc|default(30) }}, + nbnd = 8 +/ +{% endblock %} + +{% block Electrons %} +&ELECTRONS +/ +{% endblock %} + +{% block Atomic %} +ATOMIC_SPECIES + Si 28.086 Si.pz-vbc.UPF + +ATOMIC_POSITIONS (alat) + Si 0.0 0.0 0.0 + Si 0.25 0.25 0.25 +{% endblock %} + +{% block KPoints %} +K_POINTS (automatic) + {{ n_k }} {{ n_k }} {{ n_k }} 0 0 0 +{% endblock %} +``` +from which we inherit to define the `scf` and the `bands` input files, overriding the +[`block`] as needed. +```{code-block} jinja +:caption: /template/scf/pw.in.j2 + +{% extends "base.inp.j2" %} +{% block CalcMode %}scf{% endblock %} +``` +```{code-block} jinja +:caption: /template/bands/pw.in.j2 + +{% extends "base.inp.j2" %} +{% block CalcMode %}bands{% endblock %} +{% block KPoints %} +5 + 0.0000 0.5000 0.0000 {{ 2 * n_k }} !L + 0.0000 0.0000 0.0000 {{ 3 * n_k }} !G + -0.500 0.0000 -0.500 {{ n_k }} !X + -0.375 0.2500 -0.375 {{ 3 * n_k }} !U + 0.0000 0.0000 0.0000 {{ 2 * n_k }} !G +{% endblock %} +``` + +From which we use the following fmf tree: + +```{code-block} yaml +:caption: main.fmf + +templates: + exclude: + - base.j2 +vars: {} +/scf: + templates+: + path: /template/scf + vars+: + n_k: 6 +/bands: + templates+: + path: /template/bands + vars+: + n_k: 10 + links: + tmp: /scf/tmp +``` + +:::{caution} +Currently `fmf-jinja` does not support relative paths in the [`extends`] directive, +therefore you will have to make symbolic links to the `base.j2` and add it to the +{term}`exclude ` key in order to avoid generating this +intermediate file. +::: + +And a simple workflow could look like: +```console +$ fmf-jinja generate -o qe-run +$ pushd qe-run +$ for calc in scf bands; do +> pushd "$calc" +> pw.x < pw.in > pw.out +> popd +> done +$ popd +``` + +The example here is not ideal because when running the `/bands` calculation the `tmp` +folder is overwritten, however there are other workflows like the Wannier calculation +which can be designed to not overwrite the previous calculation's files. This example +is simply used to illustrate the workflow design using `fmf-jinja.` + +[`extends`]: inv:jinja#extends +[`block`]: inv:jinja#blocks diff --git a/docs/usecase/tmt.md b/docs/usecase/tmt.md new file mode 100644 index 0000000..733607f --- /dev/null +++ b/docs/usecase/tmt.md @@ -0,0 +1,73 @@ +# tmt tests + +`.fmf` files are primarily used as [tmt] test files. Currently, tmt does not integrate +with this project, but here are some workflows that can be generated from this. + +## Matrix plans + +Consider the case of running a matrix of tests in various python environments and +running different sets of tests. Using the [recursive feature] we can design this as +follows: +```{code-block} jinja +:caption: /template_tmt/main.fmf.j2 + +prepare: + - how: shell + script: | + source venv/bin/activate + pip install -e . +execute: + how: tmt +{% for p_ver in python_versions %} +/python_{{ p_ver }}: + prepare+<: + - how: install + package: python-{{ p_ver }} + - how: shell + script: | + python{{ p_ver }} -m venv venv +{% for tier in tier_filters %} +/python_{{ p_ver }}/tier_{{ tier }}: + discover: + how: fmf + filter: 'tier: {{ tier }}' +{% endfor %} +{% endfor %} +``` +```{code-block} yaml +:caption: /main.fmf + +templates: /template_tmt +vars: + python_versions: [ "3.9", "3.13" ] + tier_filters: [ 0, 1 ] +``` + +This produces the tmt tree: +```console +$ fmf-jinja generate -o output --no-recursive +$ tmt -r output plans ls +/python_3.13/tier_0 +/python_3.13/tier_1 +/python_3.9/tier_0 +/python_3.9/tier_1 +$ tmt -r output plans show /python_3.9/tier_0 +/python_3.9/tier_0 + discover + how fmf + filter tier: 0 + prepare + how install + package python-3.9 + prepare + how shell + script python3.9 -m venv venv + prepare + how shell + script source venv/bin/activate + pip install -e . + enabled true +``` + +[tmt]: inv:tmt#index +[recursive feature]: ../usage/advanced.md#recursive diff --git a/docs/why.md b/docs/why.md new file mode 100644 index 0000000..f0a29ae --- /dev/null +++ b/docs/why.md @@ -0,0 +1,6 @@ +# Why? + +There are multiple motivations for creating this project: +- fmf metadata allows for elegant de-duplication of yaml files +- easy maintenance of file generations +- create a relation of the generated outputs, e.g. via symlinks diff --git a/example/minimal/.fmf/version b/example/minimal/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/example/minimal/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/example/minimal/common_file.txt b/example/minimal/common_file.txt new file mode 100644 index 0000000..c9cdaf3 --- /dev/null +++ b/example/minimal/common_file.txt @@ -0,0 +1 @@ +This is a file that is simply copied over. diff --git a/example/minimal/file.yaml.j2 b/example/minimal/file.yaml.j2 new file mode 100644 index 0000000..8f47162 --- /dev/null +++ b/example/minimal/file.yaml.j2 @@ -0,0 +1,3 @@ +var0: random data +var1: {{ var1 }} +var2: {{ var2 }} diff --git a/example/minimal/main.fmf b/example/minimal/main.fmf new file mode 100644 index 0000000..ba45cf2 --- /dev/null +++ b/example/minimal/main.fmf @@ -0,0 +1,9 @@ +# By default, all keys are treated as variables for jinja unless a `vars` variable is defined +# or the key starts with a `/`. +var1: 42 +var2: Default value + +# The path-like keys are the output (root) paths generated +/rootA: +/rootB: + var2: Overwritten diff --git a/example/recursive/.fmf/version b/example/recursive/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/example/recursive/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/example/recursive/main.fmf b/example/recursive/main.fmf new file mode 100644 index 0000000..e8e1d30 --- /dev/null +++ b/example/recursive/main.fmf @@ -0,0 +1,9 @@ +vars: + var1: [2, 3] + var2: [x, y] +templates: + # Use the template folder for the fmf metadata + path: /template_fmf +copy: + # Copy the template folder to be used in the final round + /template: /template diff --git a/example/recursive/template/file.yaml.j2 b/example/recursive/template/file.yaml.j2 new file mode 100644 index 0000000..b0746af --- /dev/null +++ b/example/recursive/template/file.yaml.j2 @@ -0,0 +1,3 @@ +var0: random data +varA: {{ varA }} +varB: {{ varB }} diff --git a/example/recursive/template_fmf/main.fmf.j2 b/example/recursive/template_fmf/main.fmf.j2 new file mode 100644 index 0000000..2066a86 --- /dev/null +++ b/example/recursive/template_fmf/main.fmf.j2 @@ -0,0 +1,12 @@ +templates: /template +vars: {} +{% for A in var1 -%} +/A_{{ A }}: + vars+: + varA: {{ A }} +{% for B in var2 -%} +/A_{{ A }}/B_{{ B }}: + vars+: + varB: {{ B }} +{% endfor -%} +{% endfor -%} diff --git a/example/simple/main.fmf b/example/simple/main.fmf index a9f4ef1..be5d6ff 100644 --- a/example/simple/main.fmf +++ b/example/simple/main.fmf @@ -1,8 +1,19 @@ -templates: /template +templates: + path: /template +vars: + var1: 42 + var2: Default value /rootA: - vars: - my_var: A + links: + # Create a symlink to the file in rootB + fileB.yaml: /rootB/file.yaml + copy: + # Copy and rename `common_file.txt` + renamed_file.txt: /template/common_file.txt + templates+: + # Do not copy `common_file.txt` because we are renaming instead above + exclude: [ common_file.txt ] /rootB: - vars: - my_var: B + vars+: + var2: Overwritten diff --git a/example/simple/template/common_file.txt b/example/simple/template/common_file.txt new file mode 100644 index 0000000..c9cdaf3 --- /dev/null +++ b/example/simple/template/common_file.txt @@ -0,0 +1 @@ +This is a file that is simply copied over. diff --git a/example/simple/template/file.yaml.j2 b/example/simple/template/file.yaml.j2 index 1b08d91..8f47162 100644 --- a/example/simple/template/file.yaml.j2 +++ b/example/simple/template/file.yaml.j2 @@ -1 +1,3 @@ -my_var: {{ my_var }} +var0: random data +var1: {{ var1 }} +var2: {{ var2 }} diff --git a/example/simple/template/some_file.yaml b/example/simple/template/some_file.yaml deleted file mode 100644 index 8076812..0000000 --- a/example/simple/template/some_file.yaml +++ /dev/null @@ -1 +0,0 @@ -type: normal_file diff --git a/example/symlink/main.fmf b/example/symlink/main.fmf deleted file mode 100644 index 9c2fd7f..0000000 --- a/example/symlink/main.fmf +++ /dev/null @@ -1,12 +0,0 @@ -templates: /template - -/rootA: - links: - link_tpl.yaml: file.yaml - vars: - my_var: A -/rootB: - links: - link_to_A.yaml: /rootA/file.yaml - vars: - my_var: B diff --git a/example/symlink/template/file.yaml.j2 b/example/symlink/template/file.yaml.j2 deleted file mode 100644 index 1b08d91..0000000 --- a/example/symlink/template/file.yaml.j2 +++ /dev/null @@ -1 +0,0 @@ -my_var: {{ my_var }} diff --git a/example/symlink/template/link_orig.yaml b/example/symlink/template/link_orig.yaml deleted file mode 120000 index f15fc9b..0000000 --- a/example/symlink/template/link_orig.yaml +++ /dev/null @@ -1 +0,0 @@ -file.yaml \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d36da24..e004c5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,72 +1,72 @@ [build-system] -requires = ['hatchling', 'hatch-vcs'] -build-backend = 'hatchling.build' +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" [project] -name = 'fmf_jinja' +name = "fmf-jinja" authors = [ - { name = 'Cristian Le', email = 'git@lecris.dev' }, + { name = "Cristian Le", email = "git@lecris.dev" }, ] maintainers = [ - { name = 'Cristian Le', email = 'git@lecris.dev' }, + { name = "Cristian Le", email = "git@lecris.dev" }, ] -description = 'Jinja-style templater using fmf metadata' -readme = 'README.md' -license = 'GPL-3.0-or-later' -license-files = { paths = ['LICENSE.md'] } -requires-python = '>=3.9' +description = "Jinja-style templater using fmf metadata" +readme = "README.md" +license = "GPL-3.0-or-later" +requires-python = ">=3.9" classifiers = [ - 'Natural Language :: English', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Topic :: Utilities', + "Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Utilities", ] keywords = [ - 'metadata', - 'template', + "metadata", + "template", ] dependencies = [ - 'fmf @ git+https://github.com/LecrisUT/fmf@fmf-jinja', - 'Jinja2', - 'click', + "fmf", + "attrs>=23.2.0", # cached_property is broken otherwise + "Jinja2", + "click", ] -dynamic = ['version'] +dynamic = ["version"] [project.urls] -Homepage = 'https://github.com/psss/fmf' +Homepage = "https://github.com/LecrisUT/fmf-jinja" [project.optional-dependencies] test = [ - 'pytest', + "pytest", ] test-cov = [ - 'fmf_jinja[test]', - 'pytest-cov', -] -dev = [ - 'fmf_jinja[test]', - 'pre-commit', + "fmf-jinja[test]", + "pytest-cov", ] docs = [ - 'sphinx', - 'furo', - 'myst_parser', + "sphinx", + "furo", + "myst-parser", + "sphinx-tippy", + "sphinx-autodoc-typehints", + "sphinx-click", ] [project.scripts] -fmf-jinja = 'fmf_jinja.cli:main' +fmf-jinja = "fmf_jinja.cli:main" [tool.hatch] -version.source = 'vcs' -version.raw-options.version_scheme = 'post-release' +version.source = "vcs" +version.raw-options.version_scheme = "post-release" build.hooks.vcs.version-file = "src/fmf_jinja/_version.py" metadata.allow-direct-references = true [tool.pytest.ini_options] testpaths = [ - 'test', + "test", ] [tool.ruff] @@ -125,6 +125,8 @@ ignore = [ "S101", # Use of assert detected "TD002", # Missing author in TODO "TD003", # Missing issue link on the line following this TODO + "D200", # One-line docstring should fit on one line + "TID252", # Prefer absolute imports over relative imports from parent modules ] [tool.ruff.lint.per-file-ignores] @@ -145,3 +147,7 @@ disallow_incomplete_defs = false module = ["fmf_jinja.*"] disallow_untyped_defs = true disallow_incomplete_defs = true + +[[tool.mypy.overrides]] +module = ["fmf.*"] +ignore_missing_imports = true diff --git a/qodana.yaml b/qodana.yaml index 176fdd9..4e8a8fe 100644 --- a/qodana.yaml +++ b/qodana.yaml @@ -1,32 +1,6 @@ -#-------------------------------------------------------------------------------# -# Qodana analysis is configured by qodana.yaml file # -# https://www.jetbrains.com/help/qodana/qodana-yaml.html # -#-------------------------------------------------------------------------------# version: "1.0" - -#Specify inspection profile for code analysis profile: name: qodana.starter - -#Enable inspections -#include: -# - name: - -#Disable inspections -#exclude: -# - name: -# paths: -# - - -#Execute shell command before Qodana execution (Applied in CI/CD pipeline) bootstrap: | - # https://youtrack.jetbrains.com/issue/QD-2706 - rm -rf .idea - pip install -e .[test] - -#Install IDE plugins before Qodana execution (Applied in CI/CD pipeline) -#plugins: -# - id: #(plugin id can be found at https://plugins.jetbrains.com) - -#Specify Qodana linter for analysis (Applied in CI/CD pipeline) -linter: jetbrains/qodana-python:latest + pip install -e . +linter: jetbrains/qodana-python-community:latest diff --git a/src/fmf_jinja/__init__.py b/src/fmf_jinja/__init__.py index f6803e5..85368d7 100644 --- a/src/fmf_jinja/__init__.py +++ b/src/fmf_jinja/__init__.py @@ -3,11 +3,9 @@ from __future__ import annotations from ._version import __version__ -from .cli import main -from .template import generate +from .template import TemplateContext __all__ = [ "__version__", - "main", - "generate", + "TemplateContext", ] diff --git a/src/fmf_jinja/__main__.py b/src/fmf_jinja/__main__.py index a13e3bc..87b0bb3 100644 --- a/src/fmf_jinja/__main__.py +++ b/src/fmf_jinja/__main__.py @@ -1,5 +1,7 @@ """Equivalent to `fmf-jinja` CLI command.""" +from __future__ import annotations + from .cli.main import main main() diff --git a/src/fmf_jinja/_compat/__init__.py b/src/fmf_jinja/_compat/__init__.py new file mode 100644 index 0000000..201fe3d --- /dev/null +++ b/src/fmf_jinja/_compat/__init__.py @@ -0,0 +1,63 @@ +""" +Compatibility modules. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Any + + +def walk( + self: Path, + top_down: bool = True, # noqa: FBT001, FBT002 + on_error: Any = None, # noqa: ANN401 + follow_symlinks: bool = False, # noqa: FBT001, FBT002 +) -> Iterator[tuple[Path, list[str], list[str]]]: + if sys.version_info < (3, 12): + import os + + for path_str, dirs, files in os.walk( + self, + topdown=top_down, + onerror=on_error, + followlinks=follow_symlinks, + ): + yield Path(path_str), dirs, files + else: + yield from self.walk( + top_down=top_down, + on_error=on_error, + follow_symlinks=follow_symlinks, + ) + + +def relative_to(self: Path, other: Path, walk_up: bool = False) -> Path: # noqa: FBT001, FBT002 + if sys.version_info < (3, 12): + if not walk_up: + return self.relative_to(other) + self_parts = self.parts + other_parts = other.parts + anchor0, parts0 = self_parts[0], list(reversed(self_parts[1:])) + anchor1, parts1 = other_parts[0], list(reversed(other_parts[1:])) + if anchor0 != anchor1: + msg = f"{self!r} and {other!r} have different anchors" + raise ValueError(msg) + while parts0 and parts1 and parts0[-1] == parts1[-1]: + parts0.pop() + parts1.pop() + for part in parts1: + if not part or part == ".": + pass + elif part == "..": + msg = f"'..' segment in {other!r} cannot be walked" + raise ValueError(msg) + else: + parts0.append("..") + return Path(*reversed(parts0)) + return self.relative_to(other, walk_up=walk_up) diff --git a/src/fmf_jinja/cli/__init__.py b/src/fmf_jinja/cli/__init__.py index f464141..bad0da5 100644 --- a/src/fmf_jinja/cli/__init__.py +++ b/src/fmf_jinja/cli/__init__.py @@ -1,7 +1,10 @@ """CLI interface wrappers.""" -from .main import main +from __future__ import annotations + +from .main import generate, main __all__ = [ "main", + "generate", ] diff --git a/src/fmf_jinja/cli/main.py b/src/fmf_jinja/cli/main.py index 6157f1f..10c78a3 100644 --- a/src/fmf_jinja/cli/main.py +++ b/src/fmf_jinja/cli/main.py @@ -1,13 +1,14 @@ """Main CLI interface.""" +from __future__ import annotations + from pathlib import Path import click -from click import Context +from fmf import Tree -from fmf_jinja import __version__ -from fmf_jinja.fmf import Tree -from fmf_jinja.template import generate as _generate +from .. import __version__ +from ..template import TemplateContext @click.group("fmf-jinja") @@ -19,11 +20,18 @@ type=Path, default=".", show_default=True, - help="Path to the metadata tree root.", + help="Path to the fmf tree root", ) @click.pass_context -def main(ctx: Context, root: Path) -> None: - """FMF-Jinja template generator.""" +def main(ctx: click.Context, root: Path) -> None: + """ + FMF-Jinja template generator. + + \f + + :param ctx: click context + :param root: fmf tree root path + """ # noqa: D301 ctx.ensure_object(dict) ctx.obj["tree"] = Tree(root) @@ -38,15 +46,24 @@ def main(ctx: Context, root: Path) -> None: show_default=True, help="Path to generated output directory", ) +@click.option( + "--recursive/--no-recursive", + default=True, + help="Re-run the generator if any fmf file is generated", +) @click.pass_context -def generate(ctx: Context, output: Path) -> None: - r""" +def generate(ctx: click.Context, output: Path, recursive: bool) -> None: # noqa: FBT001 + """ Generate template output. \f - :param ctx: Click context - :param output: Output path - :return: - """ - _generate(ctx.obj["tree"], output) + :param ctx: click context + :param output: output path + :param recursive: run recursively + """ # noqa: D301 + template_ctx = TemplateContext( + tree_root=ctx.obj["tree"], + recursive=recursive, + ) + template_ctx.generate(output) diff --git a/src/fmf_jinja/cli/utils.py b/src/fmf_jinja/cli/utils.py deleted file mode 100644 index 1a7d2e7..0000000 --- a/src/fmf_jinja/cli/utils.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Various helper constructs.""" - -from __future__ import annotations - -import os -from contextlib import contextmanager -from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Iterator - - -@contextmanager -def cd(target: str | Path) -> Iterator[None]: - """ - Manage cd in a pushd/popd fashion. - - Usage: - - with cd(tmpdir): - do something in tmpdir - """ - curdir = Path.cwd() - os.chdir(target) - try: - yield - finally: - os.chdir(curdir) diff --git a/src/fmf_jinja/fmf/__init__.py b/src/fmf_jinja/fmf/__init__.py deleted file mode 100644 index b8ea163..0000000 --- a/src/fmf_jinja/fmf/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Wrappers over :ref:`fmf` types.""" - -from .tree import Tree - -__all__ = [ - "Tree", -] diff --git a/src/fmf_jinja/fmf/tree.py b/src/fmf_jinja/fmf/tree.py deleted file mode 100644 index a63fd77..0000000 --- a/src/fmf_jinja/fmf/tree.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Module defining :py:class:`Tree`.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from fmf import Tree as FMFTree - -if TYPE_CHECKING: - from pathlib import Path - - -class Tree(FMFTree): - """Wrapper of the :py:class:`fmf.Tree`.""" - - def __init__(self, path: Path, **kwargs) -> None: # type: ignore[no-untyped-def] # noqa: D107, ANN003 - super().__init__(str(path), **kwargs) diff --git a/src/fmf_jinja/generators.py b/src/fmf_jinja/generators.py new file mode 100644 index 0000000..9a7c453 --- /dev/null +++ b/src/fmf_jinja/generators.py @@ -0,0 +1,350 @@ +""" +Generators that consume the template and its data. +""" +from __future__ import annotations + +import abc +import functools +import shutil +from abc import abstractmethod +from pathlib import Path +from typing import TYPE_CHECKING + +import attrs +from jinja2 import Environment, FileSystemLoader + +from ._compat import relative_to, walk + +if TYPE_CHECKING: + from typing import Self, TypeAlias + + from fmf import Tree + + from .template import TemplateContext + + RawDataType: TypeAlias = None | int | float | str | bool + ListDataType: TypeAlias = list[RawDataType | "ListDataType" | "DictDataType"] + DictDataType: TypeAlias = dict[str, RawDataType | ListDataType | "DictDataType"] + DataType: TypeAlias = RawDataType | ListDataType | DictDataType + + +@attrs.define +class FullGenerator: + """ + Main generator. + + Processes the fmf tree and generates all output files + """ + + ctx: TemplateContext + """Current run context.""" + _sub_generators: list[SubGenerator] | None = attrs.field(init=False, default=None) + """See :py:attrs:`sub_generators`""" + + @property + def curr_path(self) -> str: + """Current path string that is being processed.""" + return self.ctx.curr_path + + @property + def tree(self) -> Tree: + """Current fmf tree node.""" + return self.ctx.tree + + @functools.cached_property + def vars(self) -> dict[str, DataType]: # noqa: A003 + """ + Data variables used in the jinja template generation. + + The dict is passed as-is to the jinja template renderer. + """ + if "vars" not in self.tree.data: + # If vars was not defined use the whole fmf tree content as the vars dict + vars = self.tree.data # noqa: A001 + # Handle templates + vars.pop("templates", None) + return vars + # Otherwise use the vars node data + return self.tree.get("vars") + + @property + def sub_generators(self) -> list[SubGenerator]: + """Actual generators to execute.""" + if self._sub_generators is None: + self._sub_generators = [] + # Get the template generators + raw_templates_data = self.tree.get("templates", ["/"]) + if not isinstance(raw_templates_data, list): + raw_templates_data = [raw_templates_data] + for template_data in raw_templates_data: + self._sub_generators.append( + TemplateGenerator.from_fmf_data(self, template_data), + ) + # Get the other generators only if vars was defined + # Otherwise it is ambiguous if the data is a variable or generator data + if "vars" in self.tree.data: + symlink_data = self.tree.get("links", {}) + self._sub_generators.append( + SymlinkGenerator.from_fmf_data(self, symlink_data), + ) + copy_data = self.tree.get("copy", {}) + self._sub_generators.append( + CopyGenerator.from_fmf_data(self, copy_data), + ) + return self._sub_generators + + def generate(self) -> None: + """ + Run the sub generators. + """ + # Make the output path if it doesn't exist + (self.ctx.tmp_path / self.curr_path).mkdir(parents=True, exist_ok=True) + for generator in self.sub_generators: + generator.generate() + + +@attrs.define +class SubGenerator(abc.ABC): + """ + Generators that do the main work. + """ + + parent: FullGenerator + """Main generator run.""" + + @property + def ctx(self) -> TemplateContext: + """Current run context.""" + return self.parent.ctx + + @classmethod + @abstractmethod + def from_fmf_data(cls, parent: FullGenerator, data: DataType) -> Self: + """ + Construct from the fmf data. + """ + + @abstractmethod + def generate(self) -> None: + """ + Run the generator. + """ + + +@attrs.define +class TemplateGenerator(SubGenerator): + """ + Jinja template generator. + + Renders a jinja template or a whole folder. + """ + + path: Path + """Relative path to the template folder.""" + exclude: list[Path] = attrs.field(factory=list) + """Paths to be excluded from the template generation.""" + include_empty_folder: bool = False + """Whether to include empty folders in the generated output.""" + + @property + def template_path(self) -> Path: + """Resolved path of the template.""" + return self.ctx.join_path(self.path) + + @property + def template_dir(self) -> Path: + """Directory of the template: either `path` or its parent.""" + if self.template_path.is_dir(): + return self.template_path + return self.template_path.parent + + @classmethod + def from_fmf_data( # noqa: D102 + cls, + parent: FullGenerator, + data: str | DictDataType, + ) -> TemplateGenerator: + if isinstance(data, str): + return cls(parent=parent, path=Path(data)) + try: + path = Path(data.get("path")) + exclude = [Path(path) for path in data.get("exclude", [])] + # TODO: expose `include_empty_folder` to fmf parsing + return cls(parent=parent, path=path, exclude=exclude) + except Exception as err: + msg = f"Unsupported input data [{type(data)}]: {data}" + raise TypeError(msg) from err + + def _render_or_copy( + self, + input_file: Path, + output_dir: Path, + env: Environment, + ) -> None: + """ + Render or copy the file. + + If the file has a final `.j2`suffix it is treated as a template and rendered + otherwise it simply copies the file. + + :param input_file: original file to be copied or rendered + :param output_dir: output directory where to create the files + :param env: current jinja environment + """ + if ".j2" in input_file.suffix: + # Render the jinja template file + # TODO: Ignore if it's a template input + tpl = env.get_template(str(input_file.relative_to(self.template_dir))) + output_file_name = input_file.name.removesuffix(".j2") + output_file = output_dir / output_file_name + with output_file.open("w") as f: + f.write(tpl.render(self.parent.vars)) + if ".fmf" in output_file.suffixes: + self.ctx.generated_fmf = True + return + # If it's not a template simply copy the file + output_file = output_dir / input_file.name + if input_file.is_symlink(): + # If it's a symlink generate an equivalent symlink + output_file.unlink(missing_ok=True) + symlink_target = input_file.resolve() + # Check if symlink target points to a file in the template directory + if symlink_target.is_relative_to(self.template_dir.resolve()): + # Maintain the relative target paths if it's within the template_dir + target_path = symlink_target.relative_to(self.template_dir.resolve()) + else: + # Otherwise point to the resolved absolute path + # TODO: log warning + target_path = symlink_target + output_file.symlink_to(target_path) + return + # Otherwise copy the contents of the file + shutil.copy(input_file, output_file) + + def generate(self) -> None: # noqa: D102 + if not self.template_path.exists(): + msg = f"Template path not found: {self.path}" + raise FileNotFoundError(msg) + # Use the template path as the root for the jinja templates + env = self.ctx.jinja_env.overlay(loader=FileSystemLoader(self.template_dir)) + if not self.template_path.is_dir(): + # If path points to a file render or copy that file + self._render_or_copy(self.template_path, self.ctx.output_path, env=env) + else: + # Otherwise generate the whole directory rendering or copying the files + for curr_path, dirs_str, files_str in walk(self.template_path): + # Generate the output path and make sure it exists + rel_path = curr_path.relative_to(self.template_path) + output_path = self.ctx.output_path / rel_path + # If we don't need the empty folders, the parent directories will be + # created as needed. `.fmf` folder is a special case where we don't + # include it since it might not be part of the templated folder. + if self.include_empty_folder and output_path.name != ".fmf": + output_path.mkdir(parents=True, exist_ok=True) + # Exclude all directories from recursive walk + for dir in dirs_str: # noqa: A001 + if rel_path / dir in self.exclude: + dirs_str.remove(dir) + # Loop over all the files copying or rendering the templates + for file in files_str: + file_path = rel_path / file + # Skip fmf source files because they were handled previously + if file_path in self.ctx.fmf_files: + continue + # Skip any files that were requested to be skipped + if file_path in self.exclude: + continue + output_path.mkdir(parents=True, exist_ok=True) + # Pass the resolved absolute path of the file + self._render_or_copy(curr_path / file, output_path, env=env) + + +@attrs.define +class SymlinkGenerator(SubGenerator): + """ + Symbolic link generator. + + Creates symbolic links with the context of the fmf tree. + """ + + symlinks: dict[str, Path] + """ + Symbolic links to generate + + The dict structure is: + - key: path to the symbolic link generated + - value: target where the symbolic link points to + """ + + @classmethod + def from_fmf_data( # noqa: D102 + cls, + parent: FullGenerator, + data: dict[str, str], + ) -> SymlinkGenerator: + return cls( + parent=parent, + symlinks={ + output_symlink: Path(symlink_target) + for output_symlink, symlink_target in data.items() + }, + ) + + def generate(self) -> None: # noqa: D102 + for output_symlink_str, symlink_target in self.symlinks.items(): + output_symlink = self.ctx.output_path / output_symlink_str + if symlink_target.is_absolute(): + # If the symlink target is absolute, then it starts from the output_path + target_path = self.ctx.output_path / str(symlink_target).removeprefix( + "/", + ) + else: + # Otherwise it starts from the current output path + target_path = self.ctx.output_path / symlink_target + # Make sure the symlink is created with relative path structure + relative_target_path = relative_to( + target_path, + output_symlink, + walk_up=True, + ) + # Makes sure parent directory is created + output_symlink.parent.mkdir(exist_ok=True) + # Remove any pre-existing symlinks or files + output_symlink.unlink(missing_ok=True) + # Create the symlink + output_symlink.symlink_to(relative_target_path) + + +@attrs.define +class CopyGenerator(SubGenerator): + """ + Copy generator. + + Copies files with the context of the fmf tree. + """ + + files: dict[str, Path] + """Files to copy.""" + + @classmethod + def from_fmf_data( # noqa: D102 + cls, + parent: FullGenerator, + data: dict[str, str], + ) -> CopyGenerator: + return cls( + parent=parent, + files={dest: Path(src) for dest, src in data.items()}, + ) + + def generate(self) -> None: # noqa: D102 + for dest_str, src in self.files.items(): + dest_path = self.ctx.join_path(Path(dest_str), self.ctx.tmp_path) + actual_src = self.ctx.join_path(src) + if not actual_src.exists(): + msg = f"File not found: {actual_src}" + raise FileNotFoundError(msg) + if actual_src.is_dir(): + shutil.copytree(actual_src, dest_path) + else: + shutil.copy(actual_src, dest_path) diff --git a/src/fmf_jinja/schema/copy.yaml b/src/fmf_jinja/schema/copy.yaml new file mode 100644 index 0000000..1e69130 --- /dev/null +++ b/src/fmf_jinja/schema/copy.yaml @@ -0,0 +1,20 @@ +$schema: https://json-schema.org/draft/2020-12/schema +$id: /copy +title: Copy generator inputs +description: | + Files or folders to be copied. The keys are the destination paths where the files are copied to + and the values are the original files that are being copied. + +type: object +items: { $ref: "#/defs/path" } + +$defs: + path: + type: string + oneOf: + - description: | + Path to the symbolic link target relative to the fmf root + pattern: '^/.*$' + - description: | + Path to the symbolic link target relative to the current fmf path + pattern: '^[^/].*$' diff --git a/src/fmf_jinja/schema/link.yaml b/src/fmf_jinja/schema/link.yaml new file mode 100644 index 0000000..e7571e5 --- /dev/null +++ b/src/fmf_jinja/schema/link.yaml @@ -0,0 +1,20 @@ +$schema: https://json-schema.org/draft/2020-12/schema +$id: /link +title: Symlink generator inputs +description: | + Symbolic links to be created. The keys are the paths to the symbolic link to be generated + and the values are the targets where there symbolic links point to. + +type: object +items: { $ref: "#/defs/path" } + +$defs: + path: + type: string + oneOf: + - description: | + Path to the original file/folder being copied relative to the fmf root + pattern: '^/.*$' + - description: | + Path to the original file/folder being copied relative to the current fmf path + pattern: '^[^/].*$' diff --git a/src/fmf_jinja/schema/main.yaml b/src/fmf_jinja/schema/main.yaml new file mode 100644 index 0000000..c8c0a2c --- /dev/null +++ b/src/fmf_jinja/schema/main.yaml @@ -0,0 +1,36 @@ +$schema: https://json-schema.org/draft/2020-12/schema +$id: /main +title: fmf-jinja input +description: | + Input file for + +type: object +oneOf: + - properties: + templates: + oneOf: + - $ref: "/template" + - type: array + items: { $ref: "/template" } + vars: { $ref: "#/$defs/vars" } + links: { $ref: "/link" } + copy: { $ref: "/copy" } + required: [ vars ] + additionalProperties: false + - properties: + templates: + oneOf: + - $ref: "/template" + - type: array + items: { $ref: "/template" } + additionalProperties: { $ref: "#/$defs/vars" } + required: [ templates ] + - patternProperties: + ^(?!^(vars|templates)$).*$: { $ref: "#/$defs/vars" } + additionalProperties: false + +$defs: + vars: + type: object + description: | + Jinja variables used in the template files diff --git a/src/fmf_jinja/schema/template.yaml b/src/fmf_jinja/schema/template.yaml new file mode 100644 index 0000000..d5a1890 --- /dev/null +++ b/src/fmf_jinja/schema/template.yaml @@ -0,0 +1,31 @@ +$schema: https://json-schema.org/draft/2020-12/schema +$id: /template +title: Template generator inputs +description: | + This describes the template folders that need to be generated by either copying or rendering using jinja + +oneOf: + - $ref: "#/$defs/path" + - type: object + properties: + path: { $ref: "#/$defs/path" } + exclude: { $ref: "#/$defs/exclude" } + required: [ "path" ] + additionalProperties: false + +$defs: + path: + type: string + oneOf: + - description: | + Path to the templated file/folder relative to the fmf root + pattern: '^/.*$' + - description: | + Path to the templated file/folder relative to the current fmf path + pattern: '^[^/].*$' + exclude: + type: array + description: | + Paths to exclude from being copied or rendered. Paths are treated the same as `path` key + items: + $ref: "#/$defs/path" diff --git a/src/fmf_jinja/template.py b/src/fmf_jinja/template.py index e74442c..0c6a5f0 100644 --- a/src/fmf_jinja/template.py +++ b/src/fmf_jinja/template.py @@ -2,186 +2,160 @@ from __future__ import annotations -import os +import functools import shutil +import tempfile from pathlib import Path -from typing import TYPE_CHECKING -from jinja2 import Environment, FileSystemLoader +import attrs +from fmf import Tree +from jinja2 import Environment -if TYPE_CHECKING: - from fmf.typing import DataType +from .generators import FullGenerator +from .utils import copy_fmf_tree, fmf_tree_walk, get_fmf_files -from fmf import Tree -from fmf.normalize import normalize +DEFAULT_JINJA_ENV = Environment( + keep_trailing_newline=True, + trim_blocks=True, + autoescape=True, +) +"""Default settings for the jinja environment.""" -class Template: +@attrs.define +class TemplateContext: """ - Template fmf data type. - - Contains the template folder to be generated by `fmf-jinja` + Generator's run context. """ - path: Path - """Path to the template folder.""" - exclude: list[Path] - """Paths to be excluded from the template generation.""" - - def __init__(self, data: DataType, root: str) -> None: # noqa: D107 - self.exclude = [] - if isinstance(data, str): - self.path = normalize(Path, data, root, normalizer=self.normalize_path) - return - if isinstance(data, dict): - path = data["path"] - if not isinstance(path, str): - msg = f"Unexpected type for item path: expected str, got {type(path)}" - raise TypeError(msg) - try: - self.path = normalize(Path, path, root, normalizer=self.normalize_path) - if "exclude" in data: - self.exclude = normalize(list[Path], data["exclude"]) - except TypeError as err: - msg = f"Unsupported input data: {data}" - raise TypeError(msg) from err - else: - msg = f"Unsupported input data: [{type(data)}] {data}" - raise TypeError(msg) - - @staticmethod - def normalize_path(path_str: str, root: str) -> Path: + tree_root: Tree + """Full fmf tree being processed.""" + jinja_env: Environment = attrs.field(init=False, default=DEFAULT_JINJA_ENV) + """The base jinja environment.""" + previous_ctx: TemplateContext | None = None + """The context of the previous run.""" + recursive: bool = True + """Whether to recursively generate the templated files.""" + generated_fmf: bool = attrs.field(init=False, default=False) + """Whether a fmf file was generated in the current run.""" + _tmp_path: Path = attrs.field(init=False, default=None) + """See :py:attr:`tmp_path`.""" + _curr_path: str | None = attrs.field(init=False, default=None) + """See :py:attr:`curr_path`.""" + + def __enter__(self) -> FullGenerator: # noqa: D105 + assert self._curr_path is not None + assert self.tmp_path.exists() + return FullGenerator(self) + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: # noqa: D105, ANN001 + self._curr_path = None + + @property + def curr_path(self) -> str: + """Current path string that is being processed.""" + if self._curr_path is None: + msg = "Context is used without __enter__" + raise RuntimeError(msg) + return self._curr_path + + @property + def tmp_path(self) -> Path: + """Temporary path where the current run's output is placed.""" + if self._tmp_path is None: + msg = "Temporary folder of Context was not created" + raise RuntimeError(msg) + return self._tmp_path + + @property + def output_path(self) -> Path: + """Output path of the current node being processed.""" + return self.tmp_path / self.curr_path + + @property + def tree(self) -> Tree: + """Current fmf tree node.""" + # TODO: fmf fails if node has no children but we are trying to navigate self + if not self.tree_root.children: + assert self.curr_path == "." + return self.tree_root + path_parts = self.curr_path.removeprefix(".").split("/") + tree = self.tree_root + for part in path_parts: + tree = tree[f"/{part}"] + return tree + + @property + def tree_path(self) -> Path: + """Path to the current tree node.""" + return self.tree_root_path / self.curr_path + + @functools.cached_property + def tree_root_path(self) -> Path: + """Path to the tree root directory.""" + return Path(self.tree_root.root) + + @functools.cached_property + def fmf_files(self) -> set[Path]: + """Fmf source files (cache).""" + return set(get_fmf_files(self.tree_root)) + + def join_path(self, path: Path, output_root: Path | None = None) -> Path: """ - Normalize the path to absolute path. + Join a path to the current tree context. - :param path_str: path string to normalize - :param root: path root - :return: normalized absolute Path + If the path is absolute, it navigates from the tree root or output_root, + otherwise it navigates from the current node's path. + + :param path: path object to join + :param output_root: root path where to construct the path. + Defaults to `tree_root_path` + :return: joined path """ - path = Path(path_str) - # Drop root ('/') to make it relative to tree.root + if not output_root: + output_root = self.tree_root_path if path.is_absolute(): - path = Path(*path.parts[1:]) - # Return the absolute path including the tree.root - return root / path - - -# TODO: Resolve PLR issues -def _generate(tree: Tree, output: Path, output_root: Path) -> None: # noqa: PLR0912, PLR0915 - # Get the vars used in the template - tree_vars = tree.get("vars", {}) - assert isinstance(tree_vars, dict) - # Get the symlinks to generate - links = tree.get("links", {}) - assert isinstance(links, dict) - # Make the root output path if it doesn't exist - output.mkdir(parents=True, exist_ok=True) - assert tree.root is not None - # Loop over all templates files/folders - templates = tree.normalize(list[Template], "templates", tree.root) - assert isinstance(templates, list) - for template in templates: - if not template.path.exists(): - msg = f"Template path not found: {template.path}" - raise FileNotFoundError(msg) - # Create the jinja environment that will load the - loader = FileSystemLoader( - template.path if template.path.is_dir() else template.path.parent, - ) - env = Environment( - loader=loader, - keep_trailing_newline=True, - trim_blocks=True, - autoescape=True, - ) - if template.path.is_dir(): - # If it's a directory, loop over the file structure - for path_str, dirs_str, files_str in os.walk(template.path): - # Generate the output path and make sure it exists - rel_path = Path(path_str).relative_to(template.path) - output_path = output / rel_path - output_path.mkdir(parents=True, exist_ok=True) - # Exclude all directories from recursive walk - for d in dirs_str: - if rel_path / d in template.exclude: - dirs_str.remove(d) - # Loop over all the files copying or rendering the templates - for fil in [ - Path(f) for f in files_str if rel_path / f not in template.exclude - ]: - if ".j2" not in fil.suffixes: - # If it's not a template simply copy the file - input_file = template.path / rel_path / fil - output_file = output_path / fil - if input_file.is_symlink(): - # If it's a symlink copy as is - if output_file.is_symlink(): - # Remove existing symlink if it points to somewhere else - if output_file.readlink() == input_file.readlink(): - continue - output_file.unlink() - elif output_file.exists(): - # If it's a file always remove it - output_file.unlink() - output_file.symlink_to(input_file.readlink()) - else: - # Otherwise copy the contents of the file - shutil.copy(input_file, output_file) - else: - # Otherwise render the file - # TODO: Ignore if it's a template input - tpl = env.get_template(str(rel_path / fil)) - output_file = output_path / ".".join( - [part for part in fil.name.split(".") if part != "j2"], - ) - with output_file.open("w") as f: - f.write(tpl.render(**tree_vars)) - else: - # If it's a file treat it as a template and output it to the output root - tpl = env.get_template(template.path.name) - # Strip the .j2 suffix if it exists - fil_name = ".".join( - [part for part in template.path.name.split(".") if part != "j2"], - ) - output_file = output / fil_name - with output_file.open("w") as f: - f.write(tpl.render(**tree_vars)) - # Create the symlinks - for link_name, link_path_str in links.items(): - assert isinstance(link_name, str) - assert isinstance(link_path_str, str) - link_path = Path(link_path_str) - # If the link path is absolute treat it as relative to the output root - if link_path.is_absolute(): - link_path = output_root / Path(*link_path.parts[1:]) - output_link = output / link_name - # Makes sure paren directory is created - output_link.parent.mkdir(exist_ok=True) - if output_link.is_symlink(): - # Remove existing symlink if it points to somewhere else - if output_link.readlink() == link_path: - continue - output_link.unlink() - elif output_link.exists(): - # If it's a file always remove it - output_link.unlink() - output_link.symlink_to(link_path) - - -def generate(tree: Tree, output: Path) -> None: - """ - Generate a template from fmf metadata. + return output_root / str(path).removeprefix("/") + return output_root / self.curr_path / path - :param tree: FMF metadata tree - :param output: Output path - """ - # Currently only generating on leaves - - for curr_tree, _, leaves in tree.walk(): - for leaf_tree in [curr_tree[f"/{leaf}"] for leaf in leaves]: - assert isinstance(leaf_tree, Tree) - _generate( - leaf_tree, - output / leaf_tree.name.removeprefix("/"), - output.absolute(), - ) + def generate(self, output: Path) -> None: + """ + Generate the templated files from the tree's data. + + :param output: output path where the rendered fmf tree is placed + """ + with tempfile.TemporaryDirectory(prefix="fmf-jinja-") as tmp_path: + self._tmp_path = Path(tmp_path) + copy_fmf_tree(self.tree_root, self.tmp_path) + # If tree has a single node then we cannot use leaves from walk + if not self.tree_root.children: + self._curr_path = "." + with self as generator: + generator.generate() + else: + # Walk through the tree and generate the template for each leaf + for curr_tree, _, leaves in fmf_tree_walk(self.tree_root): + for leaf in leaves: + # Construct the full leaf path, dropping the initial `/` + full_leaf_path = str(Path(curr_tree.name) / leaf) + full_leaf_path = full_leaf_path.removeprefix("/") + self._curr_path = full_leaf_path + with self as generator: + generator.generate() + # Check if we need to run the generator again + if self.recursive and self.generated_fmf: + next_tree = Tree(self.tmp_path) + next_ctx = TemplateContext( + tree_root=next_tree, + previous_ctx=self, + recursive=self.recursive, + ) + next_ctx.generate(output) + else: + # Otherwise move the final + shutil.copytree( + self.tmp_path, + output, + symlinks=True, + dirs_exist_ok=True, + ) diff --git a/src/fmf_jinja/utils.py b/src/fmf_jinja/utils.py new file mode 100644 index 0000000..8c88e9e --- /dev/null +++ b/src/fmf_jinja/utils.py @@ -0,0 +1,123 @@ +"""Utility module.""" + +from __future__ import annotations + +import shutil +from pathlib import Path +from typing import TYPE_CHECKING + +from fmf import Tree + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import TypeAlias + + WalkTuple: TypeAlias = tuple[Tree, list[str], list[str]] + """Output of :py:meth:`fmf_tree_walk`""" + + +def _fmf_node_name(tree: Tree) -> str: + """ + Equivalent of :py:attr:`pathlib.PurePath.name` for :py:class:`fmf.Tree`. + + :param tree: fmf tree + :return: the path-like ``name`` attribute + """ + if tree.parent is None: + assert tree.name == "/" + return "/" + parent_name: str = tree.parent.name + if not parent_name.endswith("/"): + parent_name = f"{parent_name}/" + assert parent_name in tree.name + return tree.name.removeprefix(parent_name) + + +def get_fmf_files(tree: Tree, *, relative: bool = True) -> Iterator[Path]: + """ + Get all fmf tree source files. + + :param tree: fmf tree + :param relative: whether to output the source files relative to the root + :return: all fmf source files including the ``.fmf`` folder files + """ + + def _return_path(path: Path) -> Path: + if not relative: + return path.absolute() + return path.relative_to(tree_root) + + assert tree.parent is None + tree_root = Path(tree.root) + # First return the .fmf/version file + # Including the `.fmf` folder itself so that it can be excluded + yield _return_path(tree_root / ".fmf/version") + # Next navigate the full fmf tree returning every source we find at most once + sources_so_far = set() + for curr_tree, _, _ in fmf_tree_walk(tree): + for source in curr_tree.sources: + if source in sources_so_far: + continue + sources_so_far.add(source) + yield _return_path(Path(source)) + + +def copy_fmf_tree(tree: Tree, output: Path) -> None: + """ + Copy a fmf tree with all its sources. + + :param tree: fmf tree + :param output: destination path + """ + assert tree.parent is None + tree_root = Path(tree.root) + for source in get_fmf_files(tree): + (output / source).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(tree_root / source, output / source) + + +def fmf_tree_walk(tree: Tree, *, top_down: bool = True) -> Iterator[WalkTuple]: + """ + Equivalent of :py:meth:`pathlib.Path.walk` for :py:class:`fmf.Tree`. + + :param tree: fmf tree + :param top_down: whether to output paths as it goes (from root ``/`` down to the + leaves) + :return: equivalent output of :py:meth:`pathlib.Path.walk` + """ + # Based on the implementation from os.walk + + # Save the path list that need to be navigated + # The navigation is always from top down, and depending on the top_down flag: + # - top down: output the tuple as we go down + # - bottom up: append to the navigation list until a leaf is reached and the + # results are popped in reverse order + paths_or_output: list[Tree | WalkTuple] = [tree] + while paths_or_output: + curr_path = paths_or_output.pop() + if isinstance(curr_path, tuple): + # We are outputting from buttom up and reached a leaf + # start yielding the results + assert not top_down + yield curr_path + continue + # Otherwise continue navigating + assert isinstance(curr_path, Tree) + + # Construct the output + branches: list[str] = [] + leaves: list[str] = [] + for child in curr_path.children.values(): + if child.children: + branches.append(_fmf_node_name(child)) + else: + leaves.append(_fmf_node_name(child)) + + if top_down: + # Output the result as we go + yield curr_path, branches, leaves + else: + # Save the result for later + paths_or_output.append((curr_path, branches, leaves)) + # Continue to navigate through the branches + paths_or_output += [curr_path.children[branch] for branch in reversed(branches)] diff --git a/test/unit/conftest.py b/test/conftest.py similarity index 86% rename from test/unit/conftest.py rename to test/conftest.py index 76a279c..9c4a783 100644 --- a/test/unit/conftest.py +++ b/test/conftest.py @@ -7,8 +7,7 @@ from typing import TYPE_CHECKING import pytest - -from fmf_jinja.fmf import Tree +from fmf import Tree if TYPE_CHECKING: from _pytest.fixtures import SubRequest @@ -142,12 +141,29 @@ class TreeFixture: @pytest.fixture() def fmf_tree(tmp_path: Path, request: SubRequest) -> TreeFixture: path = Path(request.param) - tree_path = DIR / "data" / "input" / "trees" / path - expected_path = DIR / "data" / "output" / "trees" / path - assert tree_path.exists() - assert tree_path.is_dir() - assert tree_path.joinpath(".fmf", "version").exists() - assert expected_path.exists() - assert expected_path.is_dir() + tree_path = DIR / "data/input" / path + expected_path = DIR / "data/output" / path tree = Tree(tree_path) return TreeFixture(tree, PathComp(tmp_path), PathComp(expected_path)) + + +def set_test_type_marker( + config: pytest.Config, + items: list[pytest.Item], + test_type: str, +) -> None: + rootdir = config.rootpath + test_path = rootdir / "test" / test_type + for item in items: + if not item.path.is_relative_to(test_path): + continue + item.add_marker(test_type) + + +def pytest_collection_modifyitems( + session: pytest.Session, # noqa: ARG001 + config: pytest.Config, + items: list[pytest.Item], +) -> None: + set_test_type_marker(config, items, "smoke") + set_test_type_marker(config, items, "functional") diff --git a/test/data/input/minimal b/test/data/input/minimal new file mode 120000 index 0000000..77c829f --- /dev/null +++ b/test/data/input/minimal @@ -0,0 +1 @@ +../../../example/minimal \ No newline at end of file diff --git a/test/data/input/recursive b/test/data/input/recursive new file mode 120000 index 0000000..4895bcc --- /dev/null +++ b/test/data/input/recursive @@ -0,0 +1 @@ +../../../example/recursive \ No newline at end of file diff --git a/test/data/input/simple b/test/data/input/simple new file mode 120000 index 0000000..42f57a8 --- /dev/null +++ b/test/data/input/simple @@ -0,0 +1 @@ +../../../example/simple \ No newline at end of file diff --git a/test/data/output/minimal/.fmf/version b/test/data/output/minimal/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/test/data/output/minimal/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/test/data/output/minimal/main.fmf b/test/data/output/minimal/main.fmf new file mode 100644 index 0000000..ba45cf2 --- /dev/null +++ b/test/data/output/minimal/main.fmf @@ -0,0 +1,9 @@ +# By default, all keys are treated as variables for jinja unless a `vars` variable is defined +# or the key starts with a `/`. +var1: 42 +var2: Default value + +# The path-like keys are the output (root) paths generated +/rootA: +/rootB: + var2: Overwritten diff --git a/test/data/output/minimal/rootA/common_file.txt b/test/data/output/minimal/rootA/common_file.txt new file mode 100644 index 0000000..c9cdaf3 --- /dev/null +++ b/test/data/output/minimal/rootA/common_file.txt @@ -0,0 +1 @@ +This is a file that is simply copied over. diff --git a/test/data/output/minimal/rootA/file.yaml b/test/data/output/minimal/rootA/file.yaml new file mode 100644 index 0000000..9cfc3c1 --- /dev/null +++ b/test/data/output/minimal/rootA/file.yaml @@ -0,0 +1,3 @@ +var0: random data +var1: 42 +var2: Default value diff --git a/test/data/output/minimal/rootB/common_file.txt b/test/data/output/minimal/rootB/common_file.txt new file mode 100644 index 0000000..c9cdaf3 --- /dev/null +++ b/test/data/output/minimal/rootB/common_file.txt @@ -0,0 +1 @@ +This is a file that is simply copied over. diff --git a/test/data/output/minimal/rootB/file.yaml b/test/data/output/minimal/rootB/file.yaml new file mode 100644 index 0000000..5401e9d --- /dev/null +++ b/test/data/output/minimal/rootB/file.yaml @@ -0,0 +1,3 @@ +var0: random data +var1: 42 +var2: Overwritten diff --git a/test/data/output/recursive/.fmf/version b/test/data/output/recursive/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/test/data/output/recursive/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/test/data/output/recursive/A_2/B_x/file.yaml b/test/data/output/recursive/A_2/B_x/file.yaml new file mode 100644 index 0000000..8ec0e36 --- /dev/null +++ b/test/data/output/recursive/A_2/B_x/file.yaml @@ -0,0 +1,3 @@ +var0: random data +varA: 2 +varB: x diff --git a/test/data/output/recursive/A_2/B_y/file.yaml b/test/data/output/recursive/A_2/B_y/file.yaml new file mode 100644 index 0000000..c400d4f --- /dev/null +++ b/test/data/output/recursive/A_2/B_y/file.yaml @@ -0,0 +1,3 @@ +var0: random data +varA: 2 +varB: y diff --git a/test/data/output/recursive/A_3/B_x/file.yaml b/test/data/output/recursive/A_3/B_x/file.yaml new file mode 100644 index 0000000..a54cee3 --- /dev/null +++ b/test/data/output/recursive/A_3/B_x/file.yaml @@ -0,0 +1,3 @@ +var0: random data +varA: 3 +varB: x diff --git a/test/data/output/recursive/A_3/B_y/file.yaml b/test/data/output/recursive/A_3/B_y/file.yaml new file mode 100644 index 0000000..1034541 --- /dev/null +++ b/test/data/output/recursive/A_3/B_y/file.yaml @@ -0,0 +1,3 @@ +var0: random data +varA: 3 +varB: y diff --git a/test/data/output/recursive/main.fmf b/test/data/output/recursive/main.fmf new file mode 100644 index 0000000..db001fd --- /dev/null +++ b/test/data/output/recursive/main.fmf @@ -0,0 +1,20 @@ +templates: /template +vars: {} +/A_2: + vars+: + varA: 2 +/A_2/B_x: + vars+: + varB: x +/A_2/B_y: + vars+: + varB: y +/A_3: + vars+: + varA: 3 +/A_3/B_x: + vars+: + varB: x +/A_3/B_y: + vars+: + varB: y diff --git a/test/data/output/simple/.fmf/version b/test/data/output/simple/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/test/data/output/simple/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/test/data/output/simple/main.fmf b/test/data/output/simple/main.fmf new file mode 100644 index 0000000..be5d6ff --- /dev/null +++ b/test/data/output/simple/main.fmf @@ -0,0 +1,19 @@ +templates: + path: /template +vars: + var1: 42 + var2: Default value + +/rootA: + links: + # Create a symlink to the file in rootB + fileB.yaml: /rootB/file.yaml + copy: + # Copy and rename `common_file.txt` + renamed_file.txt: /template/common_file.txt + templates+: + # Do not copy `common_file.txt` because we are renaming instead above + exclude: [ common_file.txt ] +/rootB: + vars+: + var2: Overwritten diff --git a/test/data/output/simple/rootA/file.yaml b/test/data/output/simple/rootA/file.yaml new file mode 100644 index 0000000..9cfc3c1 --- /dev/null +++ b/test/data/output/simple/rootA/file.yaml @@ -0,0 +1,3 @@ +var0: random data +var1: 42 +var2: Default value diff --git a/test/data/output/simple/rootA/fileB.yaml b/test/data/output/simple/rootA/fileB.yaml new file mode 120000 index 0000000..d00eea8 --- /dev/null +++ b/test/data/output/simple/rootA/fileB.yaml @@ -0,0 +1 @@ +../rootB/file.yaml \ No newline at end of file diff --git a/test/data/output/simple/rootA/renamed_file.txt b/test/data/output/simple/rootA/renamed_file.txt new file mode 100644 index 0000000..c9cdaf3 --- /dev/null +++ b/test/data/output/simple/rootA/renamed_file.txt @@ -0,0 +1 @@ +This is a file that is simply copied over. diff --git a/test/data/output/simple/rootB/common_file.txt b/test/data/output/simple/rootB/common_file.txt new file mode 100644 index 0000000..c9cdaf3 --- /dev/null +++ b/test/data/output/simple/rootB/common_file.txt @@ -0,0 +1 @@ +This is a file that is simply copied over. diff --git a/test/data/output/simple/rootB/file.yaml b/test/data/output/simple/rootB/file.yaml new file mode 100644 index 0000000..5401e9d --- /dev/null +++ b/test/data/output/simple/rootB/file.yaml @@ -0,0 +1,3 @@ +var0: random data +var1: 42 +var2: Overwritten diff --git a/test/functional/test_template.py b/test/functional/test_template.py new file mode 100644 index 0000000..465458f --- /dev/null +++ b/test/functional/test_template.py @@ -0,0 +1,14 @@ +import pytest + +from fmf_jinja.template import TemplateContext + + +@pytest.mark.parametrize( + "fmf_tree", + ["simple", "minimal", "recursive"], + indirect=True, +) +def test_generate(fmf_tree): + ctx = TemplateContext(fmf_tree.tree) + ctx.generate(fmf_tree.out_path.path) + assert fmf_tree.out_path == fmf_tree.expected_path diff --git a/test/smoke/test_cli.py b/test/smoke/test_cli.py new file mode 100644 index 0000000..09dbd5e --- /dev/null +++ b/test/smoke/test_cli.py @@ -0,0 +1,19 @@ +from importlib.metadata import version + +from click.testing import CliRunner + +from fmf_jinja.cli import main + + +def test_help() -> None: + runner = CliRunner() + result = runner.invoke(main, ["--help"]) + assert result.exit_code == 0 + assert "Usage: fmf-jinja [OPTIONS] COMMAND [ARGS]..." in result.output + + +def test_version() -> None: + runner = CliRunner() + result = runner.invoke(main, ["--version"]) + assert result.exit_code == 0 + assert result.output.strip() == version("fmf-jinja") diff --git a/test/unit/data/input/trees/simple b/test/unit/data/input/trees/simple deleted file mode 120000 index f3debb5..0000000 --- a/test/unit/data/input/trees/simple +++ /dev/null @@ -1 +0,0 @@ -../../../../../example/simple \ No newline at end of file diff --git a/test/unit/data/input/trees/symlink b/test/unit/data/input/trees/symlink deleted file mode 120000 index 4841a1f..0000000 --- a/test/unit/data/input/trees/symlink +++ /dev/null @@ -1 +0,0 @@ -../../../../../example/symlink \ No newline at end of file diff --git a/test/unit/data/output/trees/simple/rootA/file.yaml b/test/unit/data/output/trees/simple/rootA/file.yaml deleted file mode 100644 index 8c6054d..0000000 --- a/test/unit/data/output/trees/simple/rootA/file.yaml +++ /dev/null @@ -1 +0,0 @@ -my_var: A diff --git a/test/unit/data/output/trees/simple/rootA/some_file.yaml b/test/unit/data/output/trees/simple/rootA/some_file.yaml deleted file mode 100644 index 8076812..0000000 --- a/test/unit/data/output/trees/simple/rootA/some_file.yaml +++ /dev/null @@ -1 +0,0 @@ -type: normal_file diff --git a/test/unit/data/output/trees/simple/rootB/file.yaml b/test/unit/data/output/trees/simple/rootB/file.yaml deleted file mode 100644 index f31c5d3..0000000 --- a/test/unit/data/output/trees/simple/rootB/file.yaml +++ /dev/null @@ -1 +0,0 @@ -my_var: B diff --git a/test/unit/data/output/trees/simple/rootB/some_file.yaml b/test/unit/data/output/trees/simple/rootB/some_file.yaml deleted file mode 100644 index 8076812..0000000 --- a/test/unit/data/output/trees/simple/rootB/some_file.yaml +++ /dev/null @@ -1 +0,0 @@ -type: normal_file diff --git a/test/unit/data/output/trees/symlink/rootA/file.yaml b/test/unit/data/output/trees/symlink/rootA/file.yaml deleted file mode 100644 index 8c6054d..0000000 --- a/test/unit/data/output/trees/symlink/rootA/file.yaml +++ /dev/null @@ -1 +0,0 @@ -my_var: A diff --git a/test/unit/data/output/trees/symlink/rootA/link_orig.yaml b/test/unit/data/output/trees/symlink/rootA/link_orig.yaml deleted file mode 120000 index f15fc9b..0000000 --- a/test/unit/data/output/trees/symlink/rootA/link_orig.yaml +++ /dev/null @@ -1 +0,0 @@ -file.yaml \ No newline at end of file diff --git a/test/unit/data/output/trees/symlink/rootA/link_tpl.yaml b/test/unit/data/output/trees/symlink/rootA/link_tpl.yaml deleted file mode 120000 index f15fc9b..0000000 --- a/test/unit/data/output/trees/symlink/rootA/link_tpl.yaml +++ /dev/null @@ -1 +0,0 @@ -file.yaml \ No newline at end of file diff --git a/test/unit/data/output/trees/symlink/rootB/file.yaml b/test/unit/data/output/trees/symlink/rootB/file.yaml deleted file mode 100644 index f31c5d3..0000000 --- a/test/unit/data/output/trees/symlink/rootB/file.yaml +++ /dev/null @@ -1 +0,0 @@ -my_var: B diff --git a/test/unit/data/output/trees/symlink/rootB/link_orig.yaml b/test/unit/data/output/trees/symlink/rootB/link_orig.yaml deleted file mode 120000 index f15fc9b..0000000 --- a/test/unit/data/output/trees/symlink/rootB/link_orig.yaml +++ /dev/null @@ -1 +0,0 @@ -file.yaml \ No newline at end of file diff --git a/test/unit/data/output/trees/symlink/rootB/link_to_A.yaml b/test/unit/data/output/trees/symlink/rootB/link_to_A.yaml deleted file mode 120000 index 39bf82a..0000000 --- a/test/unit/data/output/trees/symlink/rootB/link_to_A.yaml +++ /dev/null @@ -1 +0,0 @@ -../rootA/file.yaml \ No newline at end of file diff --git a/test/unit/test_cli.py b/test/unit/test_cli.py deleted file mode 100644 index 7b464ed..0000000 --- a/test/unit/test_cli.py +++ /dev/null @@ -1,16 +0,0 @@ -from importlib.metadata import version - -from utils import call_cli - - -def test_base_help() -> None: - result = call_cli(parameters=["--help"]) - assert result.exit_code == 0 - assert "Usage: fmf-jinja [OPTIONS] COMMAND [ARGS]..." in result.output - - -def test_base_version() -> None: - # This test requires packit on pythonpath - result = call_cli(parameters=["--version"]) - assert result.exit_code == 0 - assert result.output.strip() == version("fmf-jinja") diff --git a/test/unit/test_template.py b/test/unit/test_template.py deleted file mode 100644 index 2731db3..0000000 --- a/test/unit/test_template.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest - -from fmf_jinja.template import generate - - -@pytest.mark.parametrize("fmf_tree", ["simple", "symlink"], indirect=True) -def test_generate(fmf_tree): - generate(fmf_tree.tree, fmf_tree.out_path.path) - assert fmf_tree.out_path == fmf_tree.expected_path diff --git a/test/unit/utils.py b/test/unit/utils.py deleted file mode 100644 index 72c88e7..0000000 --- a/test/unit/utils.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from click.testing import CliRunner, Result - -from fmf_jinja.cli.main import main -from fmf_jinja.cli.utils import cd - -if TYPE_CHECKING: - from pathlib import Path - - from click.core import BaseCommand - - -def call_cli( - fnc: BaseCommand = main, - parameters: list[str] | None = None, - envs: dict[str, str | None] | None = None, - working_dir: str | Path = ".", -) -> Result: - runner = CliRunner() - envs = envs or {} - parameters = parameters or [] - # catch exceptions enables debugger - with cd(working_dir): - return runner.invoke(fnc, args=parameters, env=envs, catch_exceptions=False)