feat: add YAML support (#66)

* feat: add support for YAML config files

* test: ensure no overlap between cargo and python profraw files

* test: add tests for new code

* style: run cargo fmt

* ci: install new integration test deps

* ci: install new integration test deps

* ci: install python deps via pip

* ci: make sure integration tests produce coverage

* doc: update book with references to YAML

.github/workflows/check.yml

@@ -132,13 +132,17 @@ jobs:
           CARGO_INCREMENTAL: '0'
           RUSTFLAGS: '-Zinstrument-coverage'
           RUSTDOCFLAGS: '-Zinstrument-coverage'
-          LLVM_PROFILE_FILE: 'profraw/hoart-tests-%p-%m.profraw'
+          LLVM_PROFILE_FILE: 'profraw/hoard-cargo-tests-%p-%m.profraw'
+
+      - name: Install dependencies (Linux)
+        run: pip install pyyaml toml
 
       - name: Run Integration Tests
         run: python ci-tests/tests all
         env:
           # Creates a lot of output but also makes sure all logging receives coverage
           HOARD_LOG: trace
+          LLVM_PROFILE_FILE: 'profraw/hoard-python-tests-%p-%m.profraw'
 
       - name: Run grcov
         run: |
@@ -206,6 +210,9 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: choco install tree
 
+      - name: Install Python dependencies
+        run: pip install pyyaml toml
+
       - name: Test Last Paths
         run: python ci-tests/tests last_paths
 

Cargo.toml

@@ -9,6 +9,9 @@ homepage = "https://github.com/Shadow53/hoard"
 repository = "https://github.com/Shadow53/hoard"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[features]
+default = ["yaml"]
+yaml = ["serde_yaml"]
 
 [dependencies]
 directories = "3.0.1"
@@ -20,6 +23,7 @@ petgraph = "0.5"
 regex = "1.5"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+serde_yaml = { version = "0.8", optional = true }
 structopt = "0.3.21"
 thiserror = "1.0.24"
 time = { version = "0.3", default-features = false, features = ["formatting", "macros", "serde", "std"] }

Dockerfile

@@ -1,26 +1,11 @@
-FROM rust:alpine AS build
+FROM rust:alpine
+
 ENV RUSTFLAGS="-Zinstrument-coverage"
-ENV LLVM_PROFILE_FILE="profraw/hoard-test-%p-%m.profraw"
-ENV CI=true GITHUB_ACTIONS=true HOARD_LOG=trace
+ENV LLVM_PROFILE_FILE="profraw/hoard-python-test-%p-%m.profraw"
+ENV CI=true GITHUB_ACTIONS=true HOARD_LOG=debug
 WORKDIR /hoard-tests
 
-RUN apk add python3 tree
-#RUN apk add build-base python3 tree
-#RUN rustup toolchain add nightly --component llvm-tools-preview
-#COPY Cargo.toml Cargo.toml
-#COPY Cargo.lock Cargo.lock
-#COPY src src
-#RUN cargo +nightly build
-
-#FROM ubuntu:latest
-#ENV RUSTFLAGS="-Zinstrument-coverage"
-#ENV LLVM_PROFILE_FILE="/hoard-tests/profraw/hoard-test-%p-%m.profraw"
-
-#VOLUME /hoard-tests/profraw
-
-#RUN apt-get update && apt-get install -y tree python3
-
-#COPY --from=build target/debug/hoard target/debug/hoard
+RUN apk add python3 tree py3-yaml py3-toml
 COPY target/x86_64-unknown-linux-musl/debug/hoard target/debug/hoard
 COPY ci-tests ci-tests
 RUN echo $WORKDIR

Makefile.toml

@@ -22,7 +22,7 @@
     args = ["+nightly", "test", "--target", "x86_64-unknown-linux-musl"]
     [tasks.test-nightly.env]
         RUSTFLAGS="-Zinstrument-coverage"
-        LLVM_PROFILE_FILE="profraw/hoard-test-%p-%m.profraw"
+        LLVM_PROFILE_FILE="profraw/hoard-cargo-test-%p-%m.profraw"
 
 [tasks.integration-tests]
     dependencies = ["build-nightly"]
@@ -47,7 +47,6 @@
     """
     [tasks.grcov.env]
         RUSTFLAGS="-Zinstrument-coverage"
-        LLVM_PROFILE_FILE="default.profraw"
         RUSTUP_TOOLCHAIN="nightly"
         HOARD_LOG="trace"
         EXCLUDE_DERIVE="#\\[derive\\("

book/src/config/README.md

@@ -1,3 +1,9 @@
 # Configuration File
 
-This section describes the configuration file format, with examples.
+This section describes the configuration file structure, with examples in TOML.
+
+> **NOTE:** Hoard supports YAML as well as TOML, for those who prefer the former format. This guide 
+> assumes that those using YAML already know the format and expects that said users are able to 
+> translate from TOML examples to YAML for their own configurations.
+>
+> All other users should use TOML, as it is the default format and the one used by the author.

book/src/file-locations.md

@@ -7,8 +7,8 @@ using the `config_dir` and the `data_dir` of the `ProjectDirs` struct.
 
 ## Config Directory
 
-The configuration directory holds the configuration file (`config.toml`) as well as other local-only
-configuration data.
+The configuration directory holds the configuration file (`config.toml`, `config.yaml`, or `config.yml`) as well as 
+other local-only configuration data.
 
 - Linux/BSD: `$XDG_CONFIG_HOME/hoard/` or `$HOME/.config/hoard/`
 - macos: `$HOME/Library/Application Support/com.shadow53.hoard/`

book/src/getting-started/README.md

@@ -1,3 +1,6 @@
 # Getting Started
 
-This section will guide you through installing Hoard and setting up your first hoards.
+This section will guide you through installing Hoard and setting up your first hoards.
+
+> **NOTE:** The examples use TOML as the config file format. Users looking to use YAML should be able to translate 
+> the configuration from TOML. See also [this other note](../config/).

book/src/getting-started/create-config/README.md

@@ -15,6 +15,9 @@ in hidden folders, so you may have to do some digging to find them.
 
 For the sake of this guide, we will consider three different programs:
 
+> **NOTE:** The examples use TOML as the config file format. Users looking to use YAML should be able to translate 
+> the configuration from TOML. See also [this other note](../../config/).
+
 1. [Hoard itself](./hoard.md)
 2. [Vim and Neovim](./vim.md)
 3. [*Mindustry* and *Death and Taxes*](./games.md)

ci-tests/config.toml

@@ -1,19 +1,29 @@
+# Using weird table-array syntax to make converting from TOML->YAML for tests easier.
+# Using inline {} tables uses a custom TOML type that does not translate correctly
+# NOTE: this is only for testing. Inline tables work fine with Hoard
+
 exclusivity = [
     ["first", "second"],
     ["unix", "windows"]
 ]
 
 [envs]
 [envs.first]
-    env = [{ var = "USE_ENV", expected = "1" }]
+[[envs.first.env]]
+    var = "USE_ENV"
+    expected = "1"
 [envs.second]
-    env = [{ var = "USE_ENV", expected = "2" }]
+[[envs.second.env]]
+    var = "USE_ENV"
+    expected = "2"
 [envs.windows]
     os = ["windows"]
-    env = [{ var = "HOMEPATH" }]
+[[envs.windows.env]]
+    var = "HOMEPATH"
 [envs.unix]
     os = ["linux", "macos"]
-    env = [{ var = "HOME" }]
+[[envs.unix.env]]
+    var = "HOME"
 
 [config]
     ignore = ["global*"]
@@ -38,11 +48,12 @@ exclusivity = [
         "windows|first"  = "C:/${HOMEPATH}/first_named_file"
         "windows|second" = "C:/${HOMEPATH}/second_named_file"
     [hoards.named.dir1]
-        config = { ignore = ["*pile*", ".hidden"] }
         "unix|first"  = "${HOME}/first_named_dir1"
         "unix|second" = "${HOME}/second_named_dir1"
         "windows|first"  = "C:/${HOMEPATH}/first_named_dir1"
         "windows|second" = "C:/${HOMEPATH}/second_named_dir1"
+    [hoards.named.dir1.config]
+        ignore = ["*pile*", ".hidden"]
     [hoards.named.dir2]
         "unix|first"  = "${HOME}/first_named_dir2"
         "unix|second" = "${HOME}/second_named_dir2"

ci-tests/tests/__main__.py

@@ -10,6 +10,8 @@
 from testers.hoard_tester import HoardFile, Environment
 from testers.correct_errors import CorrectErrorsTester
 from testers.no_config_dir import MissingConfigDirTester
+from testers.yaml_support import YAMLSupportTester
+
 
 for var in ["CI", "GITHUB_ACTIONS"]:
     val = os.environ.get(var)
@@ -63,8 +65,12 @@ def print_checksums():
         elif sys.argv[1] == "missing_config":
             print("Running missing config dir test")
             MissingConfigDirTester().run_test()
+        elif sys.argv[1] == "yaml":
+            print("Running YAML compat tests")
+            YAMLSupportTester().run_test()
         elif sys.argv[1] == "all":
             print("Running all tests")
+            YAMLSupportTester().run_test()
             MissingConfigDirTester().run_test()
             CorrectErrorsTester().run_test()
             LastPathsTester().run_test()
@@ -83,4 +89,12 @@ def print_checksums():
         subprocess.run(["tree", "-aL", "3", str(Path.home())])
         print_checksums()
         print_logs()
+        print("\n### Configs:")
+        config_dir = LastPathsTester.config_file_path().parent
+        for file in os.listdir(config_dir):
+            file_path = config_dir.joinpath(file)
+            if file_path.is_file():
+                with open(file_path, "r", encoding="utf-8") as opened:
+                    print(f"##### {file_path}\n")
+                    print(opened.read())
         raise

ci-tests/tests/testers/correct_errors.py

@@ -1,3 +1,5 @@
+import os
+
 from .hoard_tester import HoardTester
 
 
@@ -30,11 +32,39 @@ def _run_warn_on_invalid_uuid(self):
         assert self.uuid != invalid_id
         assert b'failed to parse uuid in file' in result.stdout
 
+    def _run_invalid_config_extension(self):
+        self.reset()
+        expected_text = b"configuration file must have file extension \""
+
+        # Missing extension
+        old_config_file = self.config_file_path()
+        new_config_file = old_config_file.parent.joinpath(old_config_file.stem)
+        os.rename(old_config_file, new_config_file)
+        self.args = ["--config-file", str(new_config_file)]
+        result = self.run_hoard("validate", allow_failure=True, capture_output=True)
+        assert expected_text in result.stdout
+
+        # Bad extension
+        old_config_file = new_config_file
+        new_config_file = old_config_file.parent.joinpath(f"{old_config_file.stem}.conf")
+        os.rename(old_config_file, new_config_file)
+        self.args = ["--config-file", str(new_config_file)]
+        result = self.run_hoard("validate", allow_failure=True, capture_output=True)
+        self.args = []
+        assert expected_text in result.stdout
+
+    def _run_missing_config(self):
+        self.reset()
+        os.remove(self.config_file_path())
+        result = self.run_hoard("validate", allow_failure=True, capture_output=True)
+        assert b"could not find any of config." in result.stdout
+
     def run_test(self):
         self._run_missing_parent_test()
         self._run_pile_named_config_test()
         self._run_env_string_named_config_test()
         self._run_hoard_named_config_test()
         self._run_warn_on_invalid_uuid()
-
+        self._run_invalid_config_extension()
+        self._run_missing_config()
 

ci-tests/tests/testers/hoard_tester.py

@@ -206,8 +206,6 @@ def run_hoard(self, command, allow_failure=False, capture_output=False):
             os.environ[key] = val
 
         args = ["target/debug/hoard"]
-        if "--config-file" not in self.args:
-            args += ["--config-file", self.config_file_path()]
         if self.force:
             args.append("--force")
         args += self.args