diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..ed64e63 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @anevis diff --git a/.github/workflows/ci-cd.yaml b/.github/workflows/ci-cd.yaml new file mode 100644 index 0000000..b0d8b01 --- /dev/null +++ b/.github/workflows/ci-cd.yaml @@ -0,0 +1,19 @@ +name: CI/CD Pipeline +run-name: CI/CD ๐ +on: [push] +jobs: + CI: + runs-on: ubuntu-latest + steps: + - run: echo "๐ The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "๐ง This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "๐ The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + - name: Check out repository code + uses: actions/checkout@v4 + - run: echo "๐ก The ${{ github.repository }} repository has been cloned to the runner." + - run: echo "๐ฅ๏ธ The workflow is now ready to test your code on the runner." + - name: Installing Devbox โ๏ธ + run: curl -fsSL https://get.jetpack.io/devbox | bash + - name: ๐งน Linting & Formatting + run: devbox run lint && devbox run format + - run: echo "๐ This job's status is ${{ job.status }}." diff --git a/.gitignore b/.gitignore index 68bc17f..d58e0b8 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,5 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ +*.iml diff --git a/README.md b/README.md index 591875e..3fad707 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,52 @@ -# json-to-markdown -A utility to take a JSON / YAML file or a python dict / list and create a Markdown file. +# JSON to Markdown Converter + +A Python utility to take a JSON / YAML file or a python dict / list and create a Markdown file. + +## Usage + +### In Python Code example: + +#### Convert a Pyton dictionary to Markdown: +```python +from json_to_markdown.md_converter import MDConverter + +data = { + "name": "John Doe", + "age": 30, + "city": "Sydney", + "hobbies": ["reading", "swimming"], +} +converter = MDConverter() +with open("output.md", "w") as f: + converter.convert(data, f) +``` +Content of `output.md` file will be: +```markdown +## Name +John Doe +## Age +30 +## City +Sydney +## Hobbies +* reading +* swimming +``` + +### From the Command Line + +You can also use the command line interface to convert a JSON or YAML file to Markdown. Here's an example: + +#### Convert a JSON file to Markdown: +```bash +python json_to_markdown/convert.py --output-file output.md --json-file test.json +``` + +#### Convert a YAML file to Markdown: +```bash +python json_to_markdown/convert.py --output-file output.md --yaml-file test.yaml +``` + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/devbox.json b/devbox.json new file mode 100644 index 0000000..9d536eb --- /dev/null +++ b/devbox.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://raw.githubusercontent.com/jetpack-io/devbox/0.10.3/.schema/devbox.schema.json", + "packages": ["python@3.12.2"], + "shell": { + "env": { + "VENV_DIR": "$HOME/MyFiles/programming/OpenSource/json-to-markdown/.devbox/virtenv/python/.venv" + }, + "init_hook": [ + ". $VENV_DIR/bin/activate" + ], + "scripts": { + "install": [ + "pip install -r requirements.txt" + ], + "install-dev": [ + "pip install -r requirements.txt -r requirements-dev.txt" + ], + "test": [ + "pytest src/" + ], + "lint": [ + "flake8 src/" + ], + "format-check": [ + "black --check src/" + ], + "format": [ + "black src/" + ] + } + } +} diff --git a/devbox.lock b/devbox.lock new file mode 100644 index 0000000..a97650a --- /dev/null +++ b/devbox.lock @@ -0,0 +1,62 @@ +{ + "lockfile_version": "1", + "packages": { + "python@3.12.2": { + "last_modified": "2024-03-22T11:26:23Z", + "plugin_version": "0.0.3", + "resolved": "github:NixOS/nixpkgs/a3ed7406349a9335cb4c2a71369b697cecd9d351#python312", + "source": "devbox-search", + "version": "3.12.2", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/jc5jlynlx561ibqxd6sy12hcqc8p39c9-python3-3.12.2", + "default": true + } + ], + "store_path": "/nix/store/jc5jlynlx561ibqxd6sy12hcqc8p39c9-python3-3.12.2" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/n5yvl08kxz5llrdiwwxfxyy6wiq2g6lc-python3-3.12.2", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/bihg62nz0vqqski18cpyppwgqz62blrq-python3-3.12.2-debug" + } + ], + "store_path": "/nix/store/n5yvl08kxz5llrdiwwxfxyy6wiq2g6lc-python3-3.12.2" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/41yqb3sxsx22drhza74icn4x1gfh3h8m-python3-3.12.2", + "default": true + } + ], + "store_path": "/nix/store/41yqb3sxsx22drhza74icn4x1gfh3h8m-python3-3.12.2" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/7yh2ax34jd7fgf17mjfd3c6niw1h2hsj-python3-3.12.2", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/mq8jh0sl1lcpk592whzw96n52grhq8wl-python3-3.12.2-debug" + } + ], + "store_path": "/nix/store/7yh2ax34jd7fgf17mjfd3c6niw1h2hsj-python3-3.12.2" + } + } + } + } +} diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md new file mode 100644 index 0000000..9a8b6f2 --- /dev/null +++ b/docs/DEVELOPER.md @@ -0,0 +1,99 @@ +# Developer Documentation + +## Getting Started + +### Prerequisites + +- devbox +- Python (3.12.2) +- pip + +#### Setting up the Development Environment +Install the devbox CLI tool if you haven't already. You can install it using the following command: + +```bash +curl -fsSL https://get.jetpack.io/devbox | bash +```` + +[![Built with Devbox](https://jetpack.io/img/devbox/shield_galaxy.svg)](https://jetpack.io/devbox/docs/contributor-quickstart/) +### Setting up Development Environment + +#### Clone the repository: + +```bash +git clone git@github.com:anevis/json-to-markdown.git + +cd json-to-markdown-converter +``` +#### Start the development environment: + +```bash +devbox shell +``` + +#### Install the required packages: + +From within devbox shell +```bash +pip install -r requirements.txt -r requirements-dev.txt +``` + +From outside devbox shell +```bash +devbox run install-dev +``` + +## Code Structure + +The functionality can be used in the command line or in Python code. + +To use the functionality in Python code, you can import the `MDConverter` class from the `json_to_markdown.md_converter` module. +The `md_converter.py` file contains the `MDConverter` class, which is used by the `convert` function to perform the conversion. + +To use the functionality from the command line, you can run the `convert.py` script in the `json_to_markdown` directory. +This file contains the `convert` function, which takes a JSON or YAML file and converts it to Markdown. + +## Testing +Tests for the project are located in the `*_test.py` files. +These tests use the Pytest and Mock libraries to test the functionality of the `convert` function and the `MDConverter` class. + +### Running the Tests + +You can run the tests with the following command: + +From within devbox shell +```bash +pytest src/ +``` + +From outside devbox shell +```bash +devbox run test +``` + +## Linting & Formatting + +The project uses the Black and Flake8 libraries for code formatting and linting. +You can run the following commands to format and lint the code: + +From within devbox shell +```bash +black src/ +flake8 src/ +``` + +From outside devbox shell +```bash +devbox run format +devbox run lint +``` + +You can use `devbox run format-check` to check if the code is formatted correctly without making any changes. + +## Contributing + +We welcome contributions to this project. Please feel free to submit a pull request or open an issue on GitHub. + +## License + +This project is licensed under the MIT License - see the [LICENSE](../LICENSE) file for details. diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..021f871 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,23 @@ +# Dev dependencies +coverage==7.4.4 +mock==5.1.0 +pytest==8.1.1 +pytest-cov==5.0.0 + +bandit==1.7.8 +black==24.3.0 +flake8==7.0.0 +flake8-bandit==4.1.1 +flake8-black==0.3.6 +flake8-bugbear==24.2.6 +flake8-functions==0.0.8 +isort==5.13.2 +mypy==1.9.0 +pep8-naming==0.13.3 +safety + +# Typing +types-mock==5.1.0.20240311 +types-orjson==3.6.2 +types-PyYAML==6.0.12.20240311 +types-jsonschema==4.21.0.20240331 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..52635c8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +click==8.1.7 +jsonschema[format]==4.21.1 +pyyaml==6.0.1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..207e20d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 120 +per-file-ignores = *_test.py:S101 +max-returns-amount = 4 diff --git a/src/json_to_markdown/__init__.py b/src/json_to_markdown/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/json_to_markdown/convert.py b/src/json_to_markdown/convert.py new file mode 100644 index 0000000..4583087 --- /dev/null +++ b/src/json_to_markdown/convert.py @@ -0,0 +1,54 @@ +import io +import json +from typing import Dict, Any, Optional + +import click +import yaml + +from json_to_markdown.md_converter import MDConverter + + +def _get_json_data(json_file: str) -> Dict[str, Any]: + with io.open(json_file, "r", encoding="utf-8") as j_file: + return json.load(j_file) + + +def _get_yaml_data(yaml_file: str) -> Dict[str, Any]: + with io.open(yaml_file, "r", encoding="utf-8") as y_file: + return yaml.safe_load(y_file) + + +@click.command() +@click.option("-o", "--output-file", "output_file", type=str) +@click.option("-y", "--yaml-file", "yaml_file", type=str, default=None) +@click.option("-j", "--json-file", "json_file", type=str, default=None) +def main(output_file: str, yaml_file: Optional[str], json_file: Optional[str]) -> None: + convert(output_file=output_file, yaml_file=yaml_file, json_file=json_file) + + +def convert( + output_file: str, yaml_file: Optional[str] = None, json_file: Optional[str] = None +) -> None: + if yaml_file is None and json_file is None: + raise RuntimeError("One of yaml_file or json_file is required.") + + data = _get_json_data(json_file) if json_file else _get_yaml_data(yaml_file) + with io.open(output_file, "w", encoding="utf-8") as md_file: + converter = MDConverter() + converter.set_selected_sections( + sections=[ + "assessment-date", + "assessors", + "description", + "diagram", + "external-dependencies", + "roles", + "entry-points", + "threats", + ] + ) + converter.convert(data=data, output_writer=md_file) + + +if __name__ == "__main__": + main() diff --git a/src/json_to_markdown/convert_test.py b/src/json_to_markdown/convert_test.py new file mode 100644 index 0000000..0c43c2f --- /dev/null +++ b/src/json_to_markdown/convert_test.py @@ -0,0 +1,43 @@ +from io import StringIO +from unittest.mock import mock_open, patch, Mock + +import pytest + +from json_to_markdown.convert import convert + +_JSON_DATA = '{"key": "value"}' + + +def test_convert_with_no_file() -> None: + # Execute + with pytest.raises( + RuntimeError, match="One of yaml_file or json_file is required." + ): + convert(output_file="some.md") + + +@patch("io.open", new_callable=mock_open(read_data=_JSON_DATA)) +def test_convert_with_json_data(mock_open_file: Mock) -> None: + # Prepare + mock_open_file.return_value.__enter__.return_value = StringIO(_JSON_DATA) + + # Execute + convert(output_file="output.md", json_file="test.json") + + # Assert + mock_open_file.assert_any_call("test.json", "r", encoding="utf-8") + mock_open_file.assert_any_call("output.md", "w", encoding="utf-8") + + +@patch("io.open", new_callable=mock_open()) +def test_convert_with_yaml_data(mock_open_file: Mock) -> None: + # Prepare + data = "key: value" + mock_open_file.return_value.__enter__.return_value = StringIO(data) + + # Execute + convert(output_file="output.md", yaml_file="test.yaml") + + # Assert + mock_open_file.assert_any_call("test.yaml", "r", encoding="utf-8") + mock_open_file.assert_any_call("output.md", "w", encoding="utf-8") diff --git a/src/json_to_markdown/md_converter.py b/src/json_to_markdown/md_converter.py new file mode 100644 index 0000000..1ee16f9 --- /dev/null +++ b/src/json_to_markdown/md_converter.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from typing import Any, Dict, IO, List, Optional, Union, Callable + +from json_to_markdown.utils import convert_to_title_case + + +class MDConverter: + def __init__(self) -> None: + """ + Converter to convert a JSON object into Markdown. + """ + self._sections: Optional[List[str]] = None + self._custom_processors: Optional[ + Dict[str, Callable[[MDConverter, str, Any, int], str]] + ] = None + + def set_selected_sections(self, sections: List[str]) -> None: + """ + Set the sections (JSON keys) to include in the Markdown. + By default, all sections will be included. + + Args: + sections (List[str]): A list of section titles. + """ + self._sections = sections + + def set_custom_section_processors( + self, + custom_processors: Dict[str, Callable[[MDConverter, str, Any, int], str]], + ): + """ + Set custom section processors, the key must match a section name/key + and the processor must take 4 arguments and return a Markdown string: + converter (MDConverter): The current converter object. + section (str): The section key + data (Union[List[Any], Dict[str, Any], str]): The data for the section + level (int): The section level + + Args: + custom_processors ([Dict[Callable[[MDConverter, str, Any, int], str]]]) + """ + self._custom_processors = custom_processors + + def convert( + self, + data: Dict[str, Union[List[Any], Dict[str, Any], str]], + output_writer: IO[str], + ) -> None: + """ + Convert the given JSON object into Markdown. + + Args: + data (Dict[str, Union[List[Any], Dict[str, Any], str]]): + output_writer (IO[str]): + The output stream object to write the Markdown to. + """ + for section in self._sections if self._sections is not None else data.keys(): + if section in data: + output_writer.write(self.process_section(section, data.get(section))) + + def process_section( + self, section: str, data: Union[List[Any], Dict[str, Any], str], level: int = 2 + ) -> str: + if self._custom_processors and section in self._custom_processors: + section_str = self._custom_processors[section](self, section, data, level) + elif isinstance(data, list): + section_str = f"{'#' * level} {convert_to_title_case(section)}\n{self._process_list(data=data)}" + elif isinstance(data, dict): + section_str = f"{'#' * level} {convert_to_title_case(section)}\n" + for section in data.keys(): + section_str += self.process_section( + section, data.get(section), level=level + 1 + ) + else: + section_str = self._get_str(section, data, level) + return f"{section_str}\n" + + def _process_list(self, data: List[Any]) -> str: + if isinstance(data[0], dict): + return self._process_table(data) + elif isinstance(data[0], list): + list_str = "" + for item in data: + list_str += f"{self._process_list(item)}\n" + return list_str + else: + return "\n".join([f"* {item}" for item in data]) + + def _process_table(self, data: List[Dict[str, str]]) -> str: + columns = self._get_columns(data) + table_str = self._process_columns(columns) + for row in data: + cell_data = [self._get_str(col, row.get(col, ""), -1) for col in columns] + row_data = " | ".join(cell_data) + table_str += f"\n| {row_data} |" + return table_str + + @staticmethod + def _process_columns(columns: List[str]) -> str: + column_titles = " | ".join([convert_to_title_case(col) for col in columns]) + col_sep = " | ".join(["---" for _ in columns]) + return f"| {column_titles} |\n| {col_sep} |" + + @staticmethod + def _get_columns(data: List[Dict[str, Any]]) -> List[str]: + columns: List[str] = [] + for row in data: + for col in row.keys(): + if col not in columns: + columns.append(col) + return columns + + def _get_str(self, text: str, data: Any, level: int) -> str: + str_data = str(data) + prefix = "\n" if level > 0 else "" + if isinstance(data, list): + lst_str = "".join([f"