Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(anta): add test atomic results #937

Open
wants to merge 38 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
3ba5563
feat(anta): add anta.result_manager.models.AtomicTestResult
mtache Nov 26, 2024
88d0fe9
feat(anta.tests): add atomic results to VerifyReachability
mtache Nov 26, 2024
bb2392d
feat(anta): now set the parent TestResult status from the AtomicTestR…
mtache Nov 26, 2024
830b128
feat(anta.tests): update VerifyReachability
mtache Nov 26, 2024
174c89e
fix: unit tests
mtache Nov 26, 2024
0c3ef47
fix: syntax error py<311
mtache Nov 26, 2024
cfd7e1d
feat: add expanded results to table output
mtache Nov 26, 2024
2c22eb6
test: add reporter unit test
mtache Nov 27, 2024
63dfbf1
fix: test performance fix
mtache Nov 28, 2024
b5b03de
fix: test another performance fix
mtache Nov 28, 2024
1b0d705
fix: try again
mtache Nov 28, 2024
917ae07
damn
mtache Nov 28, 2024
139708b
Added inputs serialization
carl-baillargeon Dec 9, 2024
3109d85
Remove exclude_unset
carl-baillargeon Dec 9, 2024
a0de036
Forgot to exclude result_overwrite
carl-baillargeon Dec 9, 2024
fa09d30
test(anta): fix ReportTable benchmark
mtache Dec 19, 2024
282181d
Merge branch 'main' into issue-427
gmuloc Dec 23, 2024
a9a0061
Merge branch 'main' into issue-427
mtache Dec 26, 2024
76d9e3f
feat(anta.constants): add missing power supply as known EOS error.
mtache Dec 26, 2024
75e612c
ci: remove release-coverage needs
mtache Dec 26, 2024
49a8f8d
refactor: review ResultManager serialization
mtache Dec 26, 2024
8f3c496
feat(anta): implement atomic results for text output
mtache Dec 26, 2024
bb3bb29
refactor: fix some TODO
mtache Dec 26, 2024
44ac993
refactor: typing
mtache Dec 26, 2024
4d4919a
feat(anta.tests): add optinal description to Host input model
mtache Dec 26, 2024
7f9b0a8
fix: mistake in text output
mtache Dec 26, 2024
e10ac11
feat(anta): do not show parent test inputs in table output
mtache Dec 26, 2024
a982e02
feat(anta): clear all cached_properties in ResultManager when a new r…
mtache Dec 26, 2024
2b62d02
feat(anta): redefine fields in TestResult and AtomicTestResult to con…
mtache Dec 26, 2024
1e5e918
test(anta.tests): add AntaUnitTest TypedDict
mtache Dec 27, 2024
f97c6ca
test(anta.tests): update VerifyReachability unit test
mtache Dec 27, 2024
91cfeda
chore: fix TODO
mtache Dec 27, 2024
6c26a44
fix: do not always exclude result_overwrite
mtache Dec 27, 2024
ad4e5e7
feat(reporter): dump inputs as YAML instead of JSON
mtache Dec 27, 2024
754d1f9
fix: use typing_extensions.NotRequired for Pyhton < 3.11
mtache Dec 29, 2024
7dd36f0
chore: add note
mtache Dec 29, 2024
c0cfda9
test: fix benchmark
mtache Dec 29, 2024
e3c35af
Merge branch 'main' into issue-427
gmuloc Jan 3, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ jobs:
release-doc:
name: "Publish documentation for release ${{github.ref_name}}"
runs-on: ubuntu-latest
needs: [release-coverage]
steps:
- uses: actions/checkout@v4
with:
Expand Down
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ repos:
name: Check typing with mypy
args:
- --config-file=pyproject.toml
- --explicit-package-bases
additional_dependencies:
- anta[cli]
- types-PyYAML
Expand Down
8 changes: 3 additions & 5 deletions anta/cli/debug/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,11 @@ def debug_options(f: Callable[..., Any]) -> Callable[..., Any]:
@functools.wraps(f)
def wrapper(
ctx: click.Context,
*args: tuple[Any],
*args: Any, # noqa: ANN401
inventory: AntaInventory,
device: str,
**kwargs: Any,
) -> Any:
# TODO: @gmuloc - tags come from context https://github.com/aristanetworks/anta/issues/584
# ruff: noqa: ARG001
**kwargs: Any, # noqa: ANN401
) -> Callable[..., Any]:
if (d := inventory.get(device)) is None:
logger.error("Device '%s' does not exist in Inventory", device)
ctx.exit(ExitCode.USAGE_ERROR)
Expand Down
15 changes: 6 additions & 9 deletions anta/cli/get/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@
from __future__ import annotations

import asyncio
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING

import click
import requests
Expand Down Expand Up @@ -115,8 +114,6 @@ def from_ansible(ctx: click.Context, output: Path, ansible_group: str, ansible_i
@click.option("--connected/--not-connected", help="Display inventory after connection has been created", default=False, required=False)
def inventory(inventory: AntaInventory, tags: set[str] | None, *, connected: bool) -> None:
"""Show inventory loaded in ANTA."""
# TODO: @gmuloc - tags come from context - we cannot have everything..
# ruff: noqa: ARG001
logger.debug("Requesting devices for tags: %s", tags)
console.print("Current inventory content is:", style="white on blue")

Expand All @@ -129,13 +126,13 @@ def inventory(inventory: AntaInventory, tags: set[str] | None, *, connected: boo

@click.command
@inventory_options
def tags(inventory: AntaInventory, **kwargs: Any) -> None:
def tags(inventory: AntaInventory, tags: set[str] | None) -> None: # noqa: ARG001
"""Get list of configured tags in user inventory."""
tags: set[str] = set()
t: set[str] = set()
for device in inventory.values():
tags.update(device.tags)
console.print("Tags found:")
console.print_json(json.dumps(sorted(tags), indent=2))
t.update(device.tags)
console.print("Tags defined in inventory:")
console.print_json(data=sorted(t), indent=2)


@click.command
Expand Down
6 changes: 3 additions & 3 deletions anta/cli/get/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,11 @@ def inventory_output_options(f: Callable[..., Any]) -> Callable[..., Any]:
@functools.wraps(f)
def wrapper(
ctx: click.Context,
*args: tuple[Any],
*args: Any, # noqa: ANN401
output: Path,
overwrite: bool,
**kwargs: dict[str, Any],
) -> Any:
**kwargs: Any, # noqa: ANN401
) -> Callable[..., Any]:
# Boolean to check if the file is empty
output_is_not_empty = output.exists() and output.stat().st_size != 0
# Check overwrite when file is not empty
Expand Down
26 changes: 22 additions & 4 deletions anta/cli/nrfu/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,19 @@
help="Group result by test or device.",
required=False,
)
def table(ctx: click.Context, group_by: Literal["device", "test"] | None) -> None:
@click.option(
"--expand-atomic",
"-x",
default=False,
show_envvar=True,
is_flag=True,
show_default=True,
help="Flag to indicate if atomic results should be rendered",
)
def table(ctx: click.Context, group_by: Literal["device", "test"] | None, expand_atomic: bool) -> None:
"""ANTA command to check network state with table results."""
run_tests(ctx)
print_table(ctx, group_by=group_by)
print_table(ctx, expand_atomic, group_by)
exit_with_code(ctx)


Expand All @@ -53,10 +62,19 @@ def json(ctx: click.Context, output: pathlib.Path | None) -> None:

@click.command()
@click.pass_context
def text(ctx: click.Context) -> None:
@click.option(
"--expand-atomic",
"-x",
default=False,
show_envvar=True,
is_flag=True,
show_default=True,
help="Flag to indicate if atomic results should be rendered",
)
def text(ctx: click.Context, expand_atomic: bool) -> None:
"""ANTA command to check network state with text results."""
run_tests(ctx)
print_text(ctx)
print_text(ctx, expand_atomic)
exit_with_code(ctx)


Expand Down
24 changes: 14 additions & 10 deletions anta/cli/nrfu/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def print_settings(
console.print()


def print_table(ctx: click.Context, group_by: Literal["device", "test"] | None = None) -> None:
def print_table(ctx: click.Context, expand_atomic: bool, group_by: Literal["device", "test"] | None) -> None:
"""Print result in a table."""
reporter = ReportTable()
console.print()
Expand All @@ -90,8 +90,10 @@ def print_table(ctx: click.Context, group_by: Literal["device", "test"] | None =
console.print(reporter.report_summary_devices(results))
elif group_by == "test":
console.print(reporter.report_summary_tests(results))
elif expand_atomic:
console.print(reporter.report_expanded(results))
else:
console.print(reporter.report_all(results))
console.print(reporter.report(results))


def print_json(ctx: click.Context, output: pathlib.Path | None = None) -> None:
Expand All @@ -112,16 +114,18 @@ def print_json(ctx: click.Context, output: pathlib.Path | None = None) -> None:
ctx.exit(ExitCode.USAGE_ERROR)


def print_text(ctx: click.Context) -> None:
def print_text(ctx: click.Context, expand_atomic: bool) -> None:
"""Print results as simple text."""
console.print()
for test in _get_result_manager(ctx).results:
if len(test.messages) <= 1:
message = test.messages[0] if len(test.messages) == 1 else ""
console.print(f"{test.name} :: {test.test} :: [{test.result}]{test.result.upper()}[/{test.result}]({message})", highlight=False)
else: # len(test.messages) > 1
console.print(f"{test.name} :: {test.test} :: [{test.result}]{test.result.upper()}[/{test.result}]", highlight=False)
console.print("\n".join(f" {message}" for message in test.messages), highlight=False)
for result in _get_result_manager(ctx).results:
console.print(f"{result.name} :: {result.test} :: [{result.result}]{result.result.upper()}[/{result.result}]", highlight=False)
if result.messages and not expand_atomic:
console.print("\n".join(f" {message}" for message in result.messages), highlight=False)
if expand_atomic:
for r in result.atomic_results:
console.print(f" {r.description} :: [{r.result}]{r.result.upper()}[/{r.result}]", highlight=False)
if r.messages:
console.print("\n".join(f" {message}" for message in r.messages), highlight=False)


def print_jinja(results: ResultManager, template: pathlib.Path, output: pathlib.Path | None = None) -> None:
Expand Down
29 changes: 14 additions & 15 deletions anta/cli/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@ class AliasedGroup(click.Group):
From Click documentation.
"""

def get_command(self, ctx: click.Context, cmd_name: str) -> Any:
"""Todo: document code."""
def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command | None:
"""Try to find a command name based on a prefix."""
rv = click.Group.get_command(self, ctx, cmd_name)
if rv is not None:
return rv
Expand All @@ -103,12 +103,11 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Any:
ctx.fail(f"Too many matches: {', '.join(sorted(matches))}")
return None

def resolve_command(self, ctx: click.Context, args: Any) -> Any:
"""Todo: document code."""
# always return the full command name
def resolve_command(self, ctx: click.Context, args: list[str]) -> tuple[str | None, click.Command | None, list[str]]:
"""Return the full command name as first tuple element."""
_, cmd, args = super().resolve_command(ctx, args)
if not cmd:
return None, None, None
return None, None, []
return cmd.name, cmd, args


Expand Down Expand Up @@ -194,7 +193,7 @@ def core_options(f: Callable[..., Any]) -> Callable[..., Any]:
@functools.wraps(f)
def wrapper(
ctx: click.Context,
*args: tuple[Any],
*args: Any, # noqa: ANN401
inventory: Path,
username: str,
password: str | None,
Expand All @@ -204,8 +203,8 @@ def wrapper(
timeout: float,
insecure: bool,
disable_cache: bool,
**kwargs: dict[str, Any],
) -> Any:
**kwargs: Any, # noqa: ANN401
) -> Callable[..., Any]:
# If help is invoke somewhere, do not parse inventory
if ctx.obj.get("_anta_help"):
return f(*args, inventory=None, **kwargs)
Expand Down Expand Up @@ -266,10 +265,10 @@ def inventory_options(f: Callable[..., Any]) -> Callable[..., Any]:
@functools.wraps(f)
def wrapper(
ctx: click.Context,
*args: tuple[Any],
*args: Any, # noqa: ANN401
tags: set[str] | None,
**kwargs: dict[str, Any],
) -> Any:
**kwargs: Any, # noqa: ANN401
) -> Callable[..., Any]:
# If help is invoke somewhere, do not parse inventory
if ctx.obj.get("_anta_help"):
return f(*args, tags=tags, **kwargs)
Expand Down Expand Up @@ -308,11 +307,11 @@ def catalog_options(f: Callable[..., Any]) -> Callable[..., Any]:
@functools.wraps(f)
def wrapper(
ctx: click.Context,
*args: tuple[Any],
*args: Any, # noqa: ANN401
catalog: Path,
catalog_format: str,
**kwargs: dict[str, Any],
) -> Any:
**kwargs: Any, # noqa: ANN401
) -> Callable[..., Any]:
# If help is invoke somewhere, do not parse catalog
if ctx.obj.get("_anta_help"):
return f(*args, catalog=None, **kwargs)
Expand Down
1 change: 1 addition & 0 deletions anta/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,5 +43,6 @@
r".* does not support IP",
r"IS-IS (.*) is disabled because: .*",
r"No source interface .*",
r"There seem to be no power supplies connected.",
]
"""List of known EOS errors that should set a test status to 'failure' with the error message."""
13 changes: 4 additions & 9 deletions anta/input_models/connectivity.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ class Host(BaseModel):
"""Model for a remote host to ping."""

model_config = ConfigDict(extra="forbid")
description: str | None = None
"""Description of the remote destination."""
destination: IPv4Address
"""IPv4 address to ping."""
source: IPv4Address | Interface
Expand All @@ -32,15 +34,8 @@ class Host(BaseModel):
"""Enable do not fragment bit in IP header. Defaults to False."""

def __str__(self) -> str:
"""Return a human-readable string representation of the Host for reporting.

Examples
--------
Host 10.1.1.1 (src: 10.2.2.2, vrf: mgmt, size: 100B, repeat: 2)

"""
df_status = ", df-bit: enabled" if self.df_bit else ""
return f"Host {self.destination} (src: {self.source}, vrf: {self.vrf}, size: {self.size}B, repeat: {self.repeat}{df_status})"
"""Return a human-readable string representation of the Host for reporting."""
return f"Destination {self.destination}{f' ({self.description})' if self.description is not None else ''} from {self.source} in VRF {self.vrf}"


class LLDPNeighbor(BaseModel):
Expand Down
6 changes: 3 additions & 3 deletions anta/inventory/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def _update_disable_cache(kwargs: dict[str, Any], *, inventory_disable_cache: bo
def _parse_hosts(
inventory_input: AntaInventoryInput,
inventory: AntaInventory,
**kwargs: dict[str, Any],
**kwargs: Any, # noqa: ANN401
) -> None:
"""Parse the host section of an AntaInventoryInput and add the devices to the inventory.

Expand Down Expand Up @@ -92,7 +92,7 @@ def _parse_hosts(
def _parse_networks(
inventory_input: AntaInventoryInput,
inventory: AntaInventory,
**kwargs: dict[str, Any],
**kwargs: Any, # noqa: ANN401
) -> None:
"""Parse the network section of an AntaInventoryInput and add the devices to the inventory.

Expand Down Expand Up @@ -129,7 +129,7 @@ def _parse_networks(
def _parse_ranges(
inventory_input: AntaInventoryInput,
inventory: AntaInventory,
**kwargs: dict[str, Any],
**kwargs: Any, # noqa: ANN401
) -> None:
"""Parse the range section of an AntaInventoryInput and add the devices to the inventory.

Expand Down
3 changes: 2 additions & 1 deletion anta/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,7 @@ def _init_inputs(self, inputs: dict[str, Any] | AntaTest.Input | None) -> None:
if res_ow.description:
self.result.description = res_ow.description
self.result.custom_field = res_ow.custom_field
self.result.inputs = self.inputs

def _init_commands(self, eos_data: list[dict[Any, Any] | str] | None) -> None:
"""Instantiate the `instance_commands` instance attribute from the `commands` class attribute.
Expand Down Expand Up @@ -615,7 +616,7 @@ def anta_test(function: F) -> Callable[..., Coroutine[Any, Any, TestResult]]:
async def wrapper(
self: AntaTest,
eos_data: list[dict[Any, Any] | str] | None = None,
**kwargs: dict[str, Any],
**kwargs: Any, # noqa: ANN401
) -> TestResult:
"""Inner function for the anta_test decorator.

Expand Down
Loading
Loading