From 56d3db167e2194b91a6f22c509c0808ebff9e116 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Wed, 11 Dec 2024 14:40:16 +0100 Subject: [PATCH 01/21] Add: standalone plugin for evaluating dependencies with a graph --- poetry.lock | 29 ++- pyproject.toml | 2 + .../standalone_plugins/dependency_graph.py | 238 ++++++++++++++++++ 3 files changed, 264 insertions(+), 5 deletions(-) create mode 100644 troubadix/standalone_plugins/dependency_graph.py diff --git a/poetry.lock b/poetry.lock index fc1719cc..726992cb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "anyio" @@ -648,6 +648,25 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "networkx" +version = "3.4.2" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.10" +files = [ + {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, + {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, +] + +[package.extras] +default = ["matplotlib (>=3.7)", "numpy (>=1.24)", "pandas (>=2.0)", "scipy (>=1.10,!=1.11.0,!=1.11.1)"] +developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] +doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.15)", "sphinx (>=7.3)", "sphinx-gallery (>=0.16)", "texext (>=0.6.7)"] +example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=1.9)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] +extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] +test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] + [[package]] name = "packaging" version = "24.2" @@ -688,13 +707,13 @@ type = ["mypy (>=1.11.2)"] [[package]] name = "pontos" -version = "25.1.0" +version = "25.3.0" description = "Common utilities and tools maintained by Greenbone Networks" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "pontos-25.1.0-py3-none-any.whl", hash = "sha256:952764e1b6aa9c1b7436e7fba71e64e6726d04a0a84925d51940998d9c5af667"}, - {file = "pontos-25.1.0.tar.gz", hash = "sha256:3a2a4267521e316fbac18aa21e7bb779358b1cb47e364ce7869ac4c630621594"}, + {file = "pontos-25.3.0-py3-none-any.whl", hash = "sha256:3799fee071a2b770487f17ce80d54ec92384f8aeedec9c7ed09fc51c4cb5daa4"}, + {file = "pontos-25.3.0.tar.gz", hash = "sha256:f5481adb8a1b6333f72b473fcdf09bac732289d795fd248b8b6d468dbe28c46c"}, ] [package.dependencies] @@ -931,4 +950,4 @@ crypto-eth-addresses = ["eth-hash[pycryptodome] (>=0.7.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "dbfa396fe4a8c9cc42d100c0804d9c5e9457677a02ce6e420a33e765eb2c05b6" +content-hash = "e2e68a665cb5cdbfc4eb449f1dd4505734499bca3e540de27eecbfb979b291a1" diff --git a/pyproject.toml b/pyproject.toml index cd76555e..a81528c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ chardet = ">=4,<6" validators = "^0.34.0" gitpython = "^3.1.31" charset-normalizer = "^3.2.0" +networkx = "^3.4.2" [tool.poetry.group.dev.dependencies] autohooks = ">=21.7.0" @@ -81,6 +82,7 @@ troubadix-changed-cves = 'troubadix.standalone_plugins.changed_cves:main' troubadix-allowed-rev-diff = 'troubadix.standalone_plugins.allowed_rev_diff:main' troubadix-file-extensions = 'troubadix.standalone_plugins.file_extensions:main' troubadix-deprecate-vts = 'troubadix.standalone_plugins.deprecate_vts:main' +troubadix-dependency-graph = 'troubadix.standalone_plugins.dependency_graph:main' [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py new file mode 100644 index 00000000..8ab3857a --- /dev/null +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -0,0 +1,238 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2024 Greenbone AG +import logging +import os +import re +import sys +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path + +import networkx as nx + +from troubadix.helper import CURRENT_ENCODING +from troubadix.helper.helper import is_enterprise_folder +from troubadix.helper.patterns import _get_special_script_tag_pattern + +EXTENSIONS = (".nasl",) +DEPENDENCY_REGEX = r"script_dependencies\((.*?)\);" +DEPENDENCY_PATTERN = _get_special_script_tag_pattern( + "dependencies", flags=re.DOTALL | re.MULTILINE +) + + +@dataclass +class Script: + name: str + path: Path + feed: str + dependencies: list[str] + + +def directory_type(string: str) -> Path: + directory_path = Path(string) + if not directory_path.is_dir(): + raise ValueError(f"{string} is not a directory.") + return directory_path + + +def parse_args() -> Namespace: + parser = ArgumentParser( + description="Check for files with unwanted file extensions", + ) + parser.add_argument( + "root", + type=directory_type, + help="directory that should be linted", + ) + parser.add_argument( + "--feed", + choices=["21.04", "22.04", "common", "full"], + default="full", + help="feed", + ) + parser.add_argument( + "--log", + default="WARNING", + help="Set the logging level (INFO, WARNING, ERROR)", + ) + return parser.parse_args() + + +def get_feed(root, feed) -> list[Script]: + match feed: + case "21.04": + return get_scripts(root / "common") + get_scripts(root / "21.04") + case "22.04": + return get_scripts(root / "common") + get_scripts(root / "22.04") + case "common": + return get_scripts(root / "common") + case "full": + return ( + get_scripts(root / "common") + + get_scripts(root / "21.04") + + get_scripts(root / "22.04") + ) + case _: + return [] + + +def get_scripts(directory) -> list[Script]: + scripts = [] + for root, _, files in os.walk(directory): + root_path = Path(root) + for file in files: + if file.endswith(EXTENSIONS): + path = root_path / file + relative_path = path.relative_to(directory) + name = str(relative_path) + feed = determine_feed(relative_path) + dependencies = extract_dependencies(path) + scripts.append(Script(name, path, feed, dependencies)) + return scripts + + +def determine_feed(script_relative_path: Path) -> str: + parts = script_relative_path.parts + if is_enterprise_folder(parts[0]): + return "enterprise" + else: + return "community" + + +# works but not used, skips gsf folder +# could be used to only determine a scripts feed +# by only fetching from enterprise folder in a seperate call +def community_files(directory): + enterprise_dir = directory / "gsf" + for root, dirs, files in os.walk(directory): + root_path = Path(root) + # durch edit in place wird gsf folder ausgelassen + dirs[:] = [d for d in dirs if root_path / d != enterprise_dir] + for file in files: + if file.endswith(EXTENSIONS): + yield root_path / file + + +def extract_dependencies(file_path: Path) -> list[str]: + deps = [] + + try: + with file_path.open("r", encoding=CURRENT_ENCODING) as file: + content = file.read() + + matches = DEPENDENCY_PATTERN.finditer(content) + for match in matches: + for line in match.group("value").splitlines(): + subject = line[: line.index("#")] if "#" in line else line + _dependencies = re.sub(r'[\'"\s]', "", subject).split(",") + deps.extend([dep for dep in _dependencies if dep != ""]) + + except Exception as e: + logging.error(f"Error processing {file_path}: {e}") + + return deps + + +def create_graph(scripts: list[Script]): + graph = nx.DiGraph() + + # Add nodes and edges based on dependencies + for script in scripts: + # explicit add incase the script has no dependencies + graph.add_node(script.name, feed=script.feed) + for dep in script.dependencies: + graph.add_edge(script.name, dep) + return graph + + +def check_duplicates(scripts: list[Script]): + """ + checks for a script depending on a script multiple times + """ + for script in scripts: + duplicates = { + dep + for dep in script.dependencies + if script.dependencies.count(dep) > 1 + } + if duplicates: + logging.warning( + f"Duplicate dependencies in {script.name}: {', '.join(duplicates)}" + ) + + +def check_missing_dependencies(scripts: list[Script], graph: nx.DiGraph) -> int: + """ + checks if any scripts that are depended on are missing from the list of scripts + + also logs the scripts dependending on the missing script + """ + dependencies = {dep for script in scripts for dep in script.dependencies} + script_names = {script.name for script in scripts} + missing_dependencies = dependencies - script_names + if not missing_dependencies: + return 0 + + for missing in missing_dependencies: + depending_scripts = graph.predecessors(missing) + logging.error(f"missing dependency file: {missing}:") + for script in depending_scripts: + logging.info(f" - used by: {script}") + + return 1 + + +def check_cycles(graph) -> int: + """ + checks for cyclic dependencies + """ + if nx.is_directed_acyclic_graph(graph): + return 0 + + cyles = nx.simple_cycles(graph) + for cycle in cyles: + logging.error(f"cyclic dependency: {cycle}") + + return 1 + + +def cross_feed_dependencies(graph): + """ + checks if scripts in community depend on scripts in enterprise folders + """ + cross_feed_dependencies = [ + (u, v) + for u, v in graph.edges + if graph.nodes[u]["feed"] == "community" + and graph.nodes[v].get("feed", "unknown") == "enterprise" + ] + for u, v in cross_feed_dependencies: + logging.info(f"cross-feed-dependency: {u} depends on {v}") + + +def main(): + args = parse_args() + logging.basicConfig( + level=args.log.upper(), format="%(levelname)s: %(message)s" + ) + logging.info("starting troubadix dependency analysis") + + scripts = get_feed(args.root, args.feed) + graph = create_graph(scripts) + + logging.info(f"nodes (scripts) in graph: {graph.number_of_nodes()}") + logging.info(f"edges (dependencies) in graph: {graph.number_of_edges()}") + + failed = 0 + + check_duplicates(scripts) + failed += check_missing_dependencies(scripts, graph) + failed += check_cycles(graph) + cross_feed_dependencies(graph) + + return failed + + +if __name__ == "__main__": + sys.exit(main()) From 13fb6a7411d3ee8a9f2fab9d1dc9ebd159ed083c Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Wed, 11 Dec 2024 15:06:25 +0100 Subject: [PATCH 02/21] Change: dependency_graph codeql mixed returns fix --- troubadix/standalone_plugins/dependency_graph.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index 8ab3857a..4eeb795c 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -15,7 +15,6 @@ from troubadix.helper.patterns import _get_special_script_tag_pattern EXTENSIONS = (".nasl",) -DEPENDENCY_REGEX = r"script_dependencies\((.*?)\);" DEPENDENCY_PATTERN = _get_special_script_tag_pattern( "dependencies", flags=re.DOTALL | re.MULTILINE ) @@ -75,6 +74,9 @@ def get_feed(root, feed) -> list[Script]: ) case _: return [] + # should be unreachable + # only here for codeql + return [] def get_scripts(directory) -> list[Script]: From c33b6e3b6db796a8a934a91adedaaed470f6ecf3 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Thu, 12 Dec 2024 15:17:47 +0100 Subject: [PATCH 03/21] Change: Revert "Change: dependency_graph codeql mixed returns fix" This reverts commit 2d9a86fc1674c46e9b64b846a8b330714c508956. --- troubadix/standalone_plugins/dependency_graph.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index 4eeb795c..8ab3857a 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -15,6 +15,7 @@ from troubadix.helper.patterns import _get_special_script_tag_pattern EXTENSIONS = (".nasl",) +DEPENDENCY_REGEX = r"script_dependencies\((.*?)\);" DEPENDENCY_PATTERN = _get_special_script_tag_pattern( "dependencies", flags=re.DOTALL | re.MULTILINE ) @@ -74,9 +75,6 @@ def get_feed(root, feed) -> list[Script]: ) case _: return [] - # should be unreachable - # only here for codeql - return [] def get_scripts(directory) -> list[Script]: From 25aeecd57ca6980017d468d5e6c452c499ef60f9 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Thu, 12 Dec 2024 16:59:59 +0100 Subject: [PATCH 04/21] Add: standalone dependency_graph unittests --- .gitignore | 1 + .../nasl/21.04/21_script.nasl | 5 ++ .../nasl/22.04/22_script.nasl | 5 ++ tests/standalone_plugins/nasl/common/bar.nasl | 9 +++ tests/standalone_plugins/nasl/common/foo.nasl | 7 ++ .../nasl/common/foobar.nasl | 5 ++ .../nasl/common/gsf/enterprise_script.nasl | 0 .../test_dependency_graph.py | 71 +++++++++++++++++++ 8 files changed, 103 insertions(+) create mode 100644 tests/standalone_plugins/nasl/21.04/21_script.nasl create mode 100644 tests/standalone_plugins/nasl/22.04/22_script.nasl create mode 100644 tests/standalone_plugins/nasl/common/bar.nasl create mode 100644 tests/standalone_plugins/nasl/common/foo.nasl create mode 100644 tests/standalone_plugins/nasl/common/foobar.nasl create mode 100644 tests/standalone_plugins/nasl/common/gsf/enterprise_script.nasl create mode 100644 tests/standalone_plugins/test_dependency_graph.py diff --git a/.gitignore b/.gitignore index b8a71240..39b99de7 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,4 @@ link.sh # vts/nasl folder used for testing nasl +!tests/standalone_plugins/nasl/ diff --git a/tests/standalone_plugins/nasl/21.04/21_script.nasl b/tests/standalone_plugins/nasl/21.04/21_script.nasl new file mode 100644 index 00000000..afa304f0 --- /dev/null +++ b/tests/standalone_plugins/nasl/21.04/21_script.nasl @@ -0,0 +1,5 @@ +if(description) +{ + script_dependencies( "foo.nasl" ); + exit(0); +} diff --git a/tests/standalone_plugins/nasl/22.04/22_script.nasl b/tests/standalone_plugins/nasl/22.04/22_script.nasl new file mode 100644 index 00000000..afa304f0 --- /dev/null +++ b/tests/standalone_plugins/nasl/22.04/22_script.nasl @@ -0,0 +1,5 @@ +if(description) +{ + script_dependencies( "foo.nasl" ); + exit(0); +} diff --git a/tests/standalone_plugins/nasl/common/bar.nasl b/tests/standalone_plugins/nasl/common/bar.nasl new file mode 100644 index 00000000..b0cbf4c5 --- /dev/null +++ b/tests/standalone_plugins/nasl/common/bar.nasl @@ -0,0 +1,9 @@ +if(description) +{ + script_dependencies( "foo.nasl", "foo.nasl" ); + + if(FEED_NAME == "GSF" || FEED_NAME == "GEF" || FEED_NAME == "SCM") + script_dependencies("gsf/enterprise_script.nasl"); + + exit(0); +} diff --git a/tests/standalone_plugins/nasl/common/foo.nasl b/tests/standalone_plugins/nasl/common/foo.nasl new file mode 100644 index 00000000..e3e7f715 --- /dev/null +++ b/tests/standalone_plugins/nasl/common/foo.nasl @@ -0,0 +1,7 @@ +if(description) +{ + script_dependencies( "foobar.nasl" ); + exit(0); +} + +script_dependencies( "missing.nasl" ); diff --git a/tests/standalone_plugins/nasl/common/foobar.nasl b/tests/standalone_plugins/nasl/common/foobar.nasl new file mode 100644 index 00000000..46e57bcb --- /dev/null +++ b/tests/standalone_plugins/nasl/common/foobar.nasl @@ -0,0 +1,5 @@ +if(description) +{ + script_dependencies( "bar.nasl" ); + exit(0); +} diff --git a/tests/standalone_plugins/nasl/common/gsf/enterprise_script.nasl b/tests/standalone_plugins/nasl/common/gsf/enterprise_script.nasl new file mode 100644 index 00000000..e69de29b diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py new file mode 100644 index 00000000..85dee7c9 --- /dev/null +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2024 Greenbone AG +import sys +import unittest +from contextlib import redirect_stderr, redirect_stdout +from io import StringIO +from pathlib import Path +from unittest.mock import patch + +from troubadix.standalone_plugins.dependency_graph import ( + Script, + create_graph, + get_feed, + main, + parse_args, +) + +NASL_DIR = "tests/standalone_plugins/nasl" + + +class TestDependencyGraph(unittest.TestCase): + + def test_parse_args_ok(self): + test_args = [ + "prog", + NASL_DIR, + "--feed", + "22.04", + "--log", + "info", + ] + with patch.object(sys, "argv", test_args): + args = parse_args() + self.assertTrue(args) + self.assertEqual(args.root, Path(NASL_DIR)) + self.assertEqual(args.feed, "22.04") + self.assertEqual(args.log, "info") + + @patch("sys.stderr", new_callable=StringIO) + def test_parse_args_no_dir(self, mock_stderr): + test_args = ["prog", "not_real_dir"] + with patch.object(sys, "argv", test_args): + with self.assertRaises(SystemExit): + parse_args() + self.assertRegex(mock_stderr.getvalue(), "invalid directory_type") + + def test_get_feed(self): + feed = "full" + scripts = get_feed(Path(NASL_DIR), feed) + self.assertEqual(len(scripts), 6) + + def test_create_graph(self): + scripts = [ + Script("foo.nasl", None, "community", ["bar.nasl"]), + Script("bar.nasl", None, "enterprise", []), + ] + graph = create_graph(scripts) + self.assertEqual(len(list(graph.nodes)), 2) + + def test_full_run(self): + test_args = [ + "prog", + NASL_DIR, + ] + with ( + redirect_stdout(StringIO()), + redirect_stderr(StringIO()), + patch.object(sys, "argv", test_args), + ): + return_code = main() + self.assertEqual(return_code, 2) From 3be9e46d177dce701397f9bdaa851a2bd4b65602 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Tue, 31 Dec 2024 12:32:12 +0100 Subject: [PATCH 05/21] Add: include when a dependency is gated in the dependency graph --- .../standalone_plugins/dependency_graph.py | 141 +++++++++++++----- 1 file changed, 103 insertions(+), 38 deletions(-) diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index 8ab3857a..3ba4385b 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -14,11 +14,14 @@ from troubadix.helper.helper import is_enterprise_folder from troubadix.helper.patterns import _get_special_script_tag_pattern -EXTENSIONS = (".nasl",) -DEPENDENCY_REGEX = r"script_dependencies\((.*?)\);" +EXTENSIONS = (".nasl",) # not sure if inc files can also have dependencies DEPENDENCY_PATTERN = _get_special_script_tag_pattern( "dependencies", flags=re.DOTALL | re.MULTILINE ) +IF_BLOCK_PATTERN = re.compile( + r'if\s*\(FEED_NAME\s*==\s*"GSF"\s*\|\|\s*FEED_NAME\s*==\s*"GEF"\s*\|\|\s*FEED_NAME\s*==\s*"SCM"\)\s*' + r"(?:\{[^}]*\}\s*|[^\{;]*;)" +) # Matches specific if blocks used to gate code to run only for enterprise feeds @dataclass @@ -26,7 +29,12 @@ class Script: name: str path: Path feed: str - dependencies: list[str] + ungated_dependencies: list[str] # not in a enterprise gate + gated_dependencies: list[str] # inside a enterprise gate + + @property + def dependencies(self) -> list[str]: + return self.ungated_dependencies + self.gated_dependencies def directory_type(string: str) -> Path: @@ -59,6 +67,7 @@ def parse_args() -> Namespace: return parser.parse_args() +# Usefull? Or is full only ever used and can therfore be removed? def get_feed(root, feed) -> list[Script]: match feed: case "21.04": @@ -83,12 +92,24 @@ def get_scripts(directory) -> list[Script]: root_path = Path(root) for file in files: if file.endswith(EXTENSIONS): - path = root_path / file - relative_path = path.relative_to(directory) + path = root_path / file # absolute path for file access + relative_path = path.relative_to( + directory + ) # relative path to \nasl will be used a identifier name = str(relative_path) feed = determine_feed(relative_path) - dependencies = extract_dependencies(path) - scripts.append(Script(name, path, feed, dependencies)) + ungated_dependencies, gated_dependencies = extract_dependencies( + path + ) + scripts.append( + Script( + name, + path, + feed, + ungated_dependencies, + gated_dependencies, + ) + ) return scripts @@ -100,38 +121,46 @@ def determine_feed(script_relative_path: Path) -> str: return "community" -# works but not used, skips gsf folder -# could be used to only determine a scripts feed -# by only fetching from enterprise folder in a seperate call -def community_files(directory): - enterprise_dir = directory / "gsf" - for root, dirs, files in os.walk(directory): - root_path = Path(root) - # durch edit in place wird gsf folder ausgelassen - dirs[:] = [d for d in dirs if root_path / d != enterprise_dir] - for file in files: - if file.endswith(EXTENSIONS): - yield root_path / file +def split_dependencies(value: str) -> list[str]: + """ + removes blank lines, strips comments, cleans dependencies, + splits them by commas, and excludes empty strings. + """ + return [ + dep + for line in value.splitlines() + if line.strip() # Ignore blank or whitespace-only lines + # ignore comment, clean line of unwanted chars, split by ',' + for dep in re.sub(r'[\'"\s]', "", line.split("#", 1)[0]).split(",") + if dep # Include only non-empty + ] -def extract_dependencies(file_path: Path) -> list[str]: - deps = [] +def extract_dependencies(file_path: Path) -> tuple[list[str], list[str]]: + ungated_deps = [] + gated_deps = [] try: with file_path.open("r", encoding=CURRENT_ENCODING) as file: content = file.read() - matches = DEPENDENCY_PATTERN.finditer(content) - for match in matches: - for line in match.group("value").splitlines(): - subject = line[: line.index("#")] if "#" in line else line - _dependencies = re.sub(r'[\'"\s]', "", subject).split(",") - deps.extend([dep for dep in _dependencies if dep != ""]) + if_blocks = [ + (m.start(), m.end()) for m in IF_BLOCK_PATTERN.finditer(content) + ] + + for match in DEPENDENCY_PATTERN.finditer(content): + start, end = match.span() + is_gated = any( + start >= block_start and end <= block_end + for block_start, block_end in if_blocks + ) + dependencies = split_dependencies(match.group("value")) + (gated_deps if is_gated else ungated_deps).extend(dependencies) except Exception as e: logging.error(f"Error processing {file_path}: {e}") - return deps + return (ungated_deps, gated_deps) def create_graph(scripts: list[Script]): @@ -141,8 +170,10 @@ def create_graph(scripts: list[Script]): for script in scripts: # explicit add incase the script has no dependencies graph.add_node(script.name, feed=script.feed) - for dep in script.dependencies: - graph.add_edge(script.name, dep) + for dep in script.ungated_dependencies: + graph.add_edge(script.name, dep, is_gated=False) + for dep in script.gated_dependencies: + graph.add_edge(script.name, dep, is_gated=True) return graph @@ -164,9 +195,9 @@ def check_duplicates(scripts: list[Script]): def check_missing_dependencies(scripts: list[Script], graph: nx.DiGraph) -> int: """ - checks if any scripts that are depended on are missing from the list of scripts - - also logs the scripts dependending on the missing script + Checks if any scripts that are depended on are missing from + the list of scripts created from the local file system, + logs the scripts dependending on the missing script """ dependencies = {dep for script in scripts for dep in script.dependencies} script_names = {script.name for script in scripts} @@ -199,16 +230,50 @@ def check_cycles(graph) -> int: def cross_feed_dependencies(graph): """ - checks if scripts in community depend on scripts in enterprise folders + creates a list of script and dep for scripts + in community feed that depend on scripts in enterprise folders """ - cross_feed_dependencies = [ + return [ (u, v) for u, v in graph.edges if graph.nodes[u]["feed"] == "community" and graph.nodes[v].get("feed", "unknown") == "enterprise" ] - for u, v in cross_feed_dependencies: - logging.info(f"cross-feed-dependency: {u} depends on {v}") + + +def ungated_cross_feed_dependencies(graph): + """ + Checks if scripts in the community feed have dependencies to enterprise scripts, + but are not contained within a gate. + """ + cross_feed_dependencies = [ + (u, v) + for u, v, is_gated in graph.edges.data("is_gated") + if graph.nodes[u]["feed"] == "community" + and graph.nodes[v].get("feed", "unknown") == "enterprise" + and not is_gated + ] + + return cross_feed_dependencies + + +def check_cross_feed_dependecies(graph): + cfd = cross_feed_dependencies(graph) + logging.info(f" {len(cfd)} cross-feed-dependencies were found:") + for u, v in cfd: + logging.warning(f"ungated cross-feed-dependency: {u} depends on {v}") + + ungated_cfd = ungated_cross_feed_dependencies(graph) + logging.info( + f" {len(ungated_cfd)} ungated cross-feed-dependencies were found:" + ) + for u, v in ungated_cfd: + logging.error(f"ungated cross-feed-dependency: {u} depends on {v}") + + if ungated_cfd: + return 1 + else: + return 0 def main(): @@ -229,7 +294,7 @@ def main(): check_duplicates(scripts) failed += check_missing_dependencies(scripts, graph) failed += check_cycles(graph) - cross_feed_dependencies(graph) + failed += check_cross_feed_dependecies(graph) return failed From 109ea7ba147e039fe0cf2464c79beb36076e090a Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Mon, 6 Jan 2025 11:58:20 +0100 Subject: [PATCH 06/21] Change: dependency_graph minor restructure and changes --- .../test_dependency_graph.py | 4 +- tests/test_naslinter.py | 1 + .../standalone_plugins/dependency_graph.py | 38 +++++++------------ 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py index 85dee7c9..d935bec3 100644 --- a/tests/standalone_plugins/test_dependency_graph.py +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -51,8 +51,8 @@ def test_get_feed(self): def test_create_graph(self): scripts = [ - Script("foo.nasl", None, "community", ["bar.nasl"]), - Script("bar.nasl", None, "enterprise", []), + Script("foo.nasl", None, "community", ["bar.nasl"], []), + Script("bar.nasl", None, "enterprise", [], []), ] graph = create_graph(scripts) self.assertEqual(len(list(graph.nodes)), 2) diff --git a/tests/test_naslinter.py b/tests/test_naslinter.py index 50e1f2ea..735b85e7 100644 --- a/tests/test_naslinter.py +++ b/tests/test_naslinter.py @@ -44,6 +44,7 @@ def test_generate_file_list_with_exclude_patterns(self): "**/templates/*/*.nasl", "**/test_files/*", "**/test_files/**/*.nasl", + "**/tests/standalone_plugins/**/*.nasl", ], include_patterns=["**/*.nasl", "**/*.inc"], ) diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index 3ba4385b..5d55ebbc 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -95,7 +95,7 @@ def get_scripts(directory) -> list[Script]: path = root_path / file # absolute path for file access relative_path = path.relative_to( directory - ) # relative path to \nasl will be used a identifier + ) # relative path to \nasl will be used as identifier name = str(relative_path) feed = determine_feed(relative_path) ungated_dependencies, gated_dependencies = extract_dependencies( @@ -228,42 +228,32 @@ def check_cycles(graph) -> int: return 1 -def cross_feed_dependencies(graph): +def cross_feed_dependencies(graph, gated_status: bool): """ - creates a list of script and dep for scripts + creates a list of script and dependency for scripts in community feed that depend on scripts in enterprise folders """ - return [ - (u, v) - for u, v in graph.edges - if graph.nodes[u]["feed"] == "community" - and graph.nodes[v].get("feed", "unknown") == "enterprise" - ] - - -def ungated_cross_feed_dependencies(graph): - """ - Checks if scripts in the community feed have dependencies to enterprise scripts, - but are not contained within a gate. - """ cross_feed_dependencies = [ (u, v) for u, v, is_gated in graph.edges.data("is_gated") if graph.nodes[u]["feed"] == "community" and graph.nodes[v].get("feed", "unknown") == "enterprise" - and not is_gated - ] - + and is_gated == gated_status + ] # unknown as standard value due to non existend nodes not having a feed value return cross_feed_dependencies def check_cross_feed_dependecies(graph): - cfd = cross_feed_dependencies(graph) - logging.info(f" {len(cfd)} cross-feed-dependencies were found:") - for u, v in cfd: - logging.warning(f"ungated cross-feed-dependency: {u} depends on {v}") + """ + Checks if scripts in the community feed have dependencies to enterprise scripts, + and if they are contained within a gate. + """ + gated_cfd = cross_feed_dependencies(graph, gated_status=True) + logging.info(f" {len(gated_cfd)} gated cross-feed-dependencies were found:") + for u, v in gated_cfd: + logging.info(f"gated cross-feed-dependency: {u} depends on {v}") - ungated_cfd = ungated_cross_feed_dependencies(graph) + ungated_cfd = cross_feed_dependencies(graph, gated_status=False) logging.info( f" {len(ungated_cfd)} ungated cross-feed-dependencies were found:" ) From de70954f8c8d70a265899b1da34123dd87f36e0d Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Thu, 16 Jan 2025 17:13:41 +0100 Subject: [PATCH 07/21] Add: category order, deprecated dependencies to graph standalone plugin --- .../nasl/21.04/21_script.nasl | 1 + .../nasl/22.04/22_script.nasl | 1 + tests/standalone_plugins/nasl/common/bar.nasl | 1 + tests/standalone_plugins/nasl/common/foo.nasl | 1 + .../nasl/common/foobar.nasl | 2 + .../nasl/common/gsf/enterprise_script.nasl | 5 + .../test_dependency_graph.py | 6 +- .../standalone_plugins/dependency_graph.py | 194 +++++++++++------- 8 files changed, 137 insertions(+), 74 deletions(-) diff --git a/tests/standalone_plugins/nasl/21.04/21_script.nasl b/tests/standalone_plugins/nasl/21.04/21_script.nasl index afa304f0..4e200767 100644 --- a/tests/standalone_plugins/nasl/21.04/21_script.nasl +++ b/tests/standalone_plugins/nasl/21.04/21_script.nasl @@ -1,5 +1,6 @@ if(description) { + script_category(ACT_GATHER_INFO); script_dependencies( "foo.nasl" ); exit(0); } diff --git a/tests/standalone_plugins/nasl/22.04/22_script.nasl b/tests/standalone_plugins/nasl/22.04/22_script.nasl index afa304f0..4e200767 100644 --- a/tests/standalone_plugins/nasl/22.04/22_script.nasl +++ b/tests/standalone_plugins/nasl/22.04/22_script.nasl @@ -1,5 +1,6 @@ if(description) { + script_category(ACT_GATHER_INFO); script_dependencies( "foo.nasl" ); exit(0); } diff --git a/tests/standalone_plugins/nasl/common/bar.nasl b/tests/standalone_plugins/nasl/common/bar.nasl index b0cbf4c5..994299d0 100644 --- a/tests/standalone_plugins/nasl/common/bar.nasl +++ b/tests/standalone_plugins/nasl/common/bar.nasl @@ -1,5 +1,6 @@ if(description) { + script_category(ACT_GATHER_INFO); script_dependencies( "foo.nasl", "foo.nasl" ); if(FEED_NAME == "GSF" || FEED_NAME == "GEF" || FEED_NAME == "SCM") diff --git a/tests/standalone_plugins/nasl/common/foo.nasl b/tests/standalone_plugins/nasl/common/foo.nasl index e3e7f715..83a66f3c 100644 --- a/tests/standalone_plugins/nasl/common/foo.nasl +++ b/tests/standalone_plugins/nasl/common/foo.nasl @@ -1,5 +1,6 @@ if(description) { + script_category(ACT_ATTACK); script_dependencies( "foobar.nasl" ); exit(0); } diff --git a/tests/standalone_plugins/nasl/common/foobar.nasl b/tests/standalone_plugins/nasl/common/foobar.nasl index 46e57bcb..0d6941ce 100644 --- a/tests/standalone_plugins/nasl/common/foobar.nasl +++ b/tests/standalone_plugins/nasl/common/foobar.nasl @@ -1,5 +1,7 @@ if(description) { + script_category(ACT_GATHER_INFO); script_dependencies( "bar.nasl" ); exit(0); + script_tag(name:"deprecated", value:TRUE); } diff --git a/tests/standalone_plugins/nasl/common/gsf/enterprise_script.nasl b/tests/standalone_plugins/nasl/common/gsf/enterprise_script.nasl index e69de29b..fa5fff33 100644 --- a/tests/standalone_plugins/nasl/common/gsf/enterprise_script.nasl +++ b/tests/standalone_plugins/nasl/common/gsf/enterprise_script.nasl @@ -0,0 +1,5 @@ +if(description) +{ + script_category(ACT_GATHER_INFO); + exit(0); +} diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py index d935bec3..78da38e4 100644 --- a/tests/standalone_plugins/test_dependency_graph.py +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -51,8 +51,8 @@ def test_get_feed(self): def test_create_graph(self): scripts = [ - Script("foo.nasl", None, "community", ["bar.nasl"], []), - Script("bar.nasl", None, "enterprise", [], []), + Script("foo.nasl", "community", [("bar.nasl", False)], 0, False), + Script("bar.nasl", "enterprise", [], 0, False), ] graph = create_graph(scripts) self.assertEqual(len(list(graph.nodes)), 2) @@ -68,4 +68,4 @@ def test_full_run(self): patch.object(sys, "argv", test_args), ): return_code = main() - self.assertEqual(return_code, 2) + self.assertEqual(return_code, 4) diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index 5d55ebbc..f7dec59c 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -12,12 +12,23 @@ from troubadix.helper import CURRENT_ENCODING from troubadix.helper.helper import is_enterprise_folder -from troubadix.helper.patterns import _get_special_script_tag_pattern +from troubadix.helper.patterns import ( + ScriptTag, + SpecialScriptTag, + _get_special_script_tag_pattern, + get_script_tag_pattern, + get_special_script_tag_pattern, +) +from troubadix.plugins.dependency_category_order import ( + VTCategory, +) EXTENSIONS = (".nasl",) # not sure if inc files can also have dependencies DEPENDENCY_PATTERN = _get_special_script_tag_pattern( "dependencies", flags=re.DOTALL | re.MULTILINE ) +CATEGORY_PATTERN = get_special_script_tag_pattern(SpecialScriptTag.CATEGORY) +DEPRECATED_PATTERN = get_script_tag_pattern(ScriptTag.DEPRECATED) IF_BLOCK_PATTERN = re.compile( r'if\s*\(FEED_NAME\s*==\s*"GSF"\s*\|\|\s*FEED_NAME\s*==\s*"GEF"\s*\|\|\s*FEED_NAME\s*==\s*"SCM"\)\s*' r"(?:\{[^}]*\}\s*|[^\{;]*;)" @@ -27,14 +38,10 @@ @dataclass class Script: name: str - path: Path feed: str - ungated_dependencies: list[str] # not in a enterprise gate - gated_dependencies: list[str] # inside a enterprise gate - - @property - def dependencies(self) -> list[str]: - return self.ungated_dependencies + self.gated_dependencies + dependencies: list[tuple[str, bool]] # (dependency_name, is_gated) + category: int + deprecated: bool def directory_type(string: str) -> Path: @@ -46,7 +53,7 @@ def directory_type(string: str) -> Path: def parse_args() -> Namespace: parser = ArgumentParser( - description="Check for files with unwanted file extensions", + description="Tool for analysing the dependencies in the NASL repository.", ) parser.add_argument( "root", @@ -88,28 +95,34 @@ def get_feed(root, feed) -> list[Script]: def get_scripts(directory) -> list[Script]: scripts = [] - for root, _, files in os.walk(directory): - root_path = Path(root) - for file in files: - if file.endswith(EXTENSIONS): - path = root_path / file # absolute path for file access - relative_path = path.relative_to( - directory - ) # relative path to \nasl will be used as identifier - name = str(relative_path) - feed = determine_feed(relative_path) - ungated_dependencies, gated_dependencies = extract_dependencies( - path - ) - scripts.append( - Script( - name, - path, - feed, - ungated_dependencies, - gated_dependencies, - ) - ) + # use path glob? + file_generator = ( + (Path(root) / file_str) + for root, _, files in os.walk(directory) + for file_str in files + if file_str.endswith(EXTENSIONS) + ) + + for path in file_generator: + try: + content = path.read_text(encoding=CURRENT_ENCODING) + except Exception as e: + logging.error(f"Error reading file {path}: {e}") + continue + + try: + relative_path = path.relative_to(directory) # used as identifier + name = str(relative_path) + feed = determine_feed(relative_path) + dependencies = extract_dependencies(content) + category = extract_category(content) + deprecated = extract_deprecated_status(content) + scripts.append( + Script(name, feed, dependencies, category, deprecated) + ) + except Exception as e: + logging.error(f"Error processing {path}: {e}") + return scripts @@ -121,6 +134,10 @@ def determine_feed(script_relative_path: Path) -> str: return "community" +def extract_deprecated_status(content) -> bool: + return bool(DEPRECATED_PATTERN.search(content)) + + def split_dependencies(value: str) -> list[str]: """ removes blank lines, strips comments, cleans dependencies, @@ -136,31 +153,30 @@ def split_dependencies(value: str) -> list[str]: ] -def extract_dependencies(file_path: Path) -> tuple[list[str], list[str]]: - ungated_deps = [] - gated_deps = [] +def extract_dependencies(content: str) -> list[tuple[str, bool]]: + dependencies = [] - try: - with file_path.open("r", encoding=CURRENT_ENCODING) as file: - content = file.read() + if_blocks = [ + (match.start(), match.end()) + for match in IF_BLOCK_PATTERN.finditer(content) + ] - if_blocks = [ - (m.start(), m.end()) for m in IF_BLOCK_PATTERN.finditer(content) - ] + for match in DEPENDENCY_PATTERN.finditer(content): + start, end = match.span() + is_gated = any( + start >= block_start and end <= block_end + for block_start, block_end in if_blocks + ) + dep_list = split_dependencies(match.group("value")) + dependencies.extend((dep, is_gated) for dep in dep_list) - for match in DEPENDENCY_PATTERN.finditer(content): - start, end = match.span() - is_gated = any( - start >= block_start and end <= block_end - for block_start, block_end in if_blocks - ) - dependencies = split_dependencies(match.group("value")) - (gated_deps if is_gated else ungated_deps).extend(dependencies) + return dependencies - except Exception as e: - logging.error(f"Error processing {file_path}: {e}") - return (ungated_deps, gated_deps) +def extract_category(content) -> int: + match = CATEGORY_PATTERN.search(content) + category_value = match.group("value") + return VTCategory[category_value] def create_graph(scripts: list[Script]): @@ -169,11 +185,14 @@ def create_graph(scripts: list[Script]): # Add nodes and edges based on dependencies for script in scripts: # explicit add incase the script has no dependencies - graph.add_node(script.name, feed=script.feed) - for dep in script.ungated_dependencies: - graph.add_edge(script.name, dep, is_gated=False) - for dep in script.gated_dependencies: - graph.add_edge(script.name, dep, is_gated=True) + graph.add_node( + script.name, + feed=script.feed, + category=script.category, + deprecated=script.deprecated, + ) + for dep, is_gated in script.dependencies: + graph.add_edge(script.name, dep, is_gated=is_gated) return graph @@ -182,10 +201,9 @@ def check_duplicates(scripts: list[Script]): checks for a script depending on a script multiple times """ for script in scripts: + dependencies = [dep for dep, _ in script.dependencies] duplicates = { - dep - for dep in script.dependencies - if script.dependencies.count(dep) > 1 + dep for dep in dependencies if dependencies.count(dep) > 1 } if duplicates: logging.warning( @@ -199,7 +217,7 @@ def check_missing_dependencies(scripts: list[Script], graph: nx.DiGraph) -> int: the list of scripts created from the local file system, logs the scripts dependending on the missing script """ - dependencies = {dep for script in scripts for dep in script.dependencies} + dependencies = {dep for script in scripts for dep, _ in script.dependencies} script_names = {script.name for script in scripts} missing_dependencies = dependencies - script_names if not missing_dependencies: @@ -249,21 +267,53 @@ def check_cross_feed_dependecies(graph): and if they are contained within a gate. """ gated_cfd = cross_feed_dependencies(graph, gated_status=True) - logging.info(f" {len(gated_cfd)} gated cross-feed-dependencies were found:") - for u, v in gated_cfd: - logging.info(f"gated cross-feed-dependency: {u} depends on {v}") + for dependent, dependency in gated_cfd: + logging.info( + f"gated cross-feed-dependency: {dependent} depends on {dependency}" + ) ungated_cfd = cross_feed_dependencies(graph, gated_status=False) - logging.info( - f" {len(ungated_cfd)} ungated cross-feed-dependencies were found:" - ) - for u, v in ungated_cfd: - logging.error(f"ungated cross-feed-dependency: {u} depends on {v}") + if not ungated_cfd: + return 0 + for dependent, dependency in ungated_cfd: + logging.error( + f"ungated cross-feed-dependency: {dependent} depends on {dependency}" + ) - if ungated_cfd: - return 1 - else: + return 1 + + +def check_category_order(graph): + problematic_edges = [ + (dependent, dependency) + for dependent, dependency in graph.edges() + if graph.nodes[dependent]["category"] + < graph.nodes[dependency].get("category", -1) + ] + + if not problematic_edges: return 0 + for dependent, dependency in problematic_edges: + logging.error( + "Not allowed category order." + f" {dependent} is higher in the execution order than {dependency}" + ) + return 1 + + +def check_deprecated_dependencies(graph) -> int: + deprecated_edges = [ + (dependent, dependency) + for dependent, dependency in graph.edges() + if graph.nodes[dependency].get("deprecated", False) + ] + if not deprecated_edges: + return 0 + for dependent, dependency in deprecated_edges: + logging.error( + f"Deprecated dependency: {dependent} depends on {dependency}" + ) + return 1 def main(): @@ -285,6 +335,8 @@ def main(): failed += check_missing_dependencies(scripts, graph) failed += check_cycles(graph) failed += check_cross_feed_dependecies(graph) + failed += check_category_order(graph) + failed += check_deprecated_dependencies(graph) return failed From c980a56159c7b144a7eb6e3d5e7bb642518868e5 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Mon, 20 Jan 2025 12:04:12 +0100 Subject: [PATCH 08/21] Change: dependency_graph checks result structure and output --- .../test_dependency_graph.py | 2 +- .../standalone_plugins/dependency_graph.py | 164 ++++++++++++------ 2 files changed, 108 insertions(+), 58 deletions(-) diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py index 78da38e4..ade82e35 100644 --- a/tests/standalone_plugins/test_dependency_graph.py +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -68,4 +68,4 @@ def test_full_run(self): patch.object(sys, "argv", test_args), ): return_code = main() - self.assertEqual(return_code, 4) + self.assertEqual(return_code, 1) diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index f7dec59c..ecb2c02a 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -5,7 +5,7 @@ import re import sys from argparse import ArgumentParser, Namespace -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path import networkx as nx @@ -44,6 +44,51 @@ class Script: deprecated: bool +@dataclass +class Result: + name: str + warnings: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + def has_errors(self) -> bool: + return bool(self.errors) + + def has_warnings(self) -> bool: + return bool(self.warnings) + + +class Reporter: + def __init__(self, verbosity) -> None: + self.verbosity = verbosity + + def report(self, results: list[Result]): + for result in results: + if self.verbosity >= 2: + self.print_statistic(result) + self.print_divider("-") + if self.verbosity >= 1: + self.print_warnings(result) + self.print_errors(result) + if self.verbosity >= 2: + self.print_divider("=") + + def print_divider(self, char="-", length=40): + print(char * length) + + def print_statistic(self, result: Result): + print( + f"{result.name} - warnings: {len(result.warnings)}, errors: {len(result.errors)}" + ) + + def print_warnings(self, result: Result): + for warning in result.warnings: + print(f"warning: {warning}") + + def print_errors(self, result: Result): + for error in result.errors: + print(f"error: {error}") + + def directory_type(string: str) -> Path: directory_path = Path(string) if not directory_path.is_dir(): @@ -71,6 +116,7 @@ def parse_args() -> Namespace: default="WARNING", help="Set the logging level (INFO, WARNING, ERROR)", ) + parser.add_argument("-v", "--verbose", action="count", default=0) return parser.parse_args() @@ -196,57 +242,60 @@ def create_graph(scripts: list[Script]): return graph -def check_duplicates(scripts: list[Script]): +def check_duplicates(scripts: list[Script]) -> Result: """ checks for a script depending on a script multiple times """ + warnings = [] for script in scripts: dependencies = [dep for dep, _ in script.dependencies] duplicates = { dep for dep in dependencies if dependencies.count(dep) > 1 } if duplicates: - logging.warning( - f"Duplicate dependencies in {script.name}: {', '.join(duplicates)}" - ) + msg = f"Duplicate dependencies in {script.name}: {', '.join(duplicates)}" + warnings.append(msg) + return Result(name="check_duplicates", warnings=warnings) -def check_missing_dependencies(scripts: list[Script], graph: nx.DiGraph) -> int: + +def check_missing_dependencies( + scripts: list[Script], graph: nx.DiGraph +) -> Result: """ Checks if any scripts that are depended on are missing from the list of scripts created from the local file system, logs the scripts dependending on the missing script """ + errors = [] dependencies = {dep for script in scripts for dep, _ in script.dependencies} script_names = {script.name for script in scripts} missing_dependencies = dependencies - script_names - if not missing_dependencies: - return 0 for missing in missing_dependencies: depending_scripts = graph.predecessors(missing) - logging.error(f"missing dependency file: {missing}:") + msg = f"missing dependency file: {missing}:" for script in depending_scripts: - logging.info(f" - used by: {script}") + msg += f"\n - used by: {script}" + errors.append(msg) - return 1 + return Result(name="missing_dependencies", errors=errors) -def check_cycles(graph) -> int: +def check_cycles(graph) -> Result: """ checks for cyclic dependencies """ if nx.is_directed_acyclic_graph(graph): - return 0 + return Result(name="check_cycles") - cyles = nx.simple_cycles(graph) - for cycle in cyles: - logging.error(f"cyclic dependency: {cycle}") + cycles = nx.simple_cycles(graph) - return 1 + errors = [f"cyclic dependency: {cycle}" for cycle in cycles] + return Result(name="check_cycles", errors=errors) -def cross_feed_dependencies(graph, gated_status: bool): +def cross_feed_dependencies(graph, gated_status: bool) -> list[tuple[str, str]]: """ creates a list of script and dependency for scripts in community feed that depend on scripts in enterprise folders @@ -261,29 +310,29 @@ def cross_feed_dependencies(graph, gated_status: bool): return cross_feed_dependencies -def check_cross_feed_dependecies(graph): +def check_cross_feed_dependecies(graph) -> Result: """ Checks if scripts in the community feed have dependencies to enterprise scripts, and if they are contained within a gate. """ gated_cfd = cross_feed_dependencies(graph, gated_status=True) - for dependent, dependency in gated_cfd: - logging.info( - f"gated cross-feed-dependency: {dependent} depends on {dependency}" - ) + warnings = [ + f"gated cross-feed-dependency: {dependent} depends on {dependency}" + for dependent, dependency in gated_cfd + ] ungated_cfd = cross_feed_dependencies(graph, gated_status=False) - if not ungated_cfd: - return 0 - for dependent, dependency in ungated_cfd: - logging.error( - f"ungated cross-feed-dependency: {dependent} depends on {dependency}" - ) + errors = [ + f"ungated cross-feed-dependency: {dependent} depends on {dependency}" + for dependent, dependency in ungated_cfd + ] - return 1 + return Result( + name="check_cross_feed_dependencies", warnings=warnings, errors=errors + ) -def check_category_order(graph): +def check_category_order(graph) -> Result: problematic_edges = [ (dependent, dependency) for dependent, dependency in graph.edges() @@ -291,29 +340,20 @@ def check_category_order(graph): < graph.nodes[dependency].get("category", -1) ] - if not problematic_edges: - return 0 - for dependent, dependency in problematic_edges: - logging.error( - "Not allowed category order." - f" {dependent} is higher in the execution order than {dependency}" - ) - return 1 + errors = [ + f"{dependent} depends on {dependency} which has a lower category order" + for dependent, dependency in problematic_edges + ] + return Result(name="check_category_order", errors=errors) -def check_deprecated_dependencies(graph) -> int: - deprecated_edges = [ - (dependent, dependency) +def check_deprecated_dependencies(graph) -> Result: + errors = [ + f"{dependent} depends on deprectated script {dependency}" for dependent, dependency in graph.edges() if graph.nodes[dependency].get("deprecated", False) ] - if not deprecated_edges: - return 0 - for dependent, dependency in deprecated_edges: - logging.error( - f"Deprecated dependency: {dependent} depends on {dependency}" - ) - return 1 + return Result(name="check_deprecated_dependencies", errors=errors) def main(): @@ -329,16 +369,26 @@ def main(): logging.info(f"nodes (scripts) in graph: {graph.number_of_nodes()}") logging.info(f"edges (dependencies) in graph: {graph.number_of_edges()}") - failed = 0 + results = [ + check_duplicates(scripts), + check_missing_dependencies(scripts, graph), + check_cycles(graph), + check_cross_feed_dependecies(graph), + check_category_order(graph), + check_deprecated_dependencies(graph), + ] + reporter = Reporter(args.verbose) + reporter.report(results) - check_duplicates(scripts) - failed += check_missing_dependencies(scripts, graph) - failed += check_cycles(graph) - failed += check_cross_feed_dependecies(graph) - failed += check_category_order(graph) - failed += check_deprecated_dependencies(graph) + has_errors = any(result.has_errors() for result in results) + has_warnings = any(result.has_warnings() for result in results) - return failed + if has_errors: + return 1 + elif has_warnings: + return 2 + else: + return 0 if __name__ == "__main__": From f2bcd326c5363ef7acfd2b0da1d3d5a56b3c1660 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Wed, 22 Jan 2025 13:56:25 +0100 Subject: [PATCH 09/21] Change: various refactors in dependency graph standalone --- tests/standalone_plugins/nasl/common/foo.nasl | 2 +- .../standalone_plugins/dependency_graph.py | 92 ++++++++++++------- 2 files changed, 59 insertions(+), 35 deletions(-) diff --git a/tests/standalone_plugins/nasl/common/foo.nasl b/tests/standalone_plugins/nasl/common/foo.nasl index 83a66f3c..568da7e7 100644 --- a/tests/standalone_plugins/nasl/common/foo.nasl +++ b/tests/standalone_plugins/nasl/common/foo.nasl @@ -1,7 +1,7 @@ if(description) { script_category(ACT_ATTACK); - script_dependencies( "foobar.nasl" ); + script_dependencies( "foobar.nasl", "gsf/enterprise_script.nasl" ); exit(0); } diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index ecb2c02a..254f1bf1 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -4,8 +4,10 @@ import os import re import sys -from argparse import ArgumentParser, Namespace +from argparse import ArgumentParser, ArgumentTypeError, Namespace from dataclasses import dataclass, field +from enum import Flag, auto +from functools import reduce from pathlib import Path import networkx as nx @@ -35,6 +37,24 @@ ) # Matches specific if blocks used to gate code to run only for enterprise feeds +class Feed(Flag): + COMMON = auto() + FEED_21_04 = auto() + FEED_22_04 = auto() + FULL = COMMON | FEED_21_04 | FEED_22_04 + + def __str__(self): + # Make enum values user-friendly for argparse help + return self.name.lower() + + +def feed_type(value: str) -> Feed: + try: + return Feed[value.upper()] + except KeyError: + raise ArgumentTypeError(f"Invalid Feed value: '{value}'") + + @dataclass class Script: name: str @@ -65,7 +85,7 @@ def report(self, results: list[Result]): for result in results: if self.verbosity >= 2: self.print_statistic(result) - self.print_divider("-") + self.print_divider() if self.verbosity >= 1: self.print_warnings(result) self.print_errors(result) @@ -103,12 +123,16 @@ def parse_args() -> Namespace: parser.add_argument( "root", type=directory_type, + nargs="?", help="directory that should be linted", ) parser.add_argument( + "-f", "--feed", - choices=["21.04", "22.04", "common", "full"], - default="full", + type=feed_type, + choices=Feed, + nargs="+", + default=[Feed.FULL], help="feed", ) parser.add_argument( @@ -117,39 +141,32 @@ def parse_args() -> Namespace: help="Set the logging level (INFO, WARNING, ERROR)", ) parser.add_argument("-v", "--verbose", action="count", default=0) + return parser.parse_args() # Usefull? Or is full only ever used and can therfore be removed? -def get_feed(root, feed) -> list[Script]: - match feed: - case "21.04": - return get_scripts(root / "common") + get_scripts(root / "21.04") - case "22.04": - return get_scripts(root / "common") + get_scripts(root / "22.04") - case "common": - return get_scripts(root / "common") - case "full": - return ( - get_scripts(root / "common") - + get_scripts(root / "21.04") - + get_scripts(root / "22.04") - ) - case _: - return [] +def get_feed(root, feeds: list[Feed]) -> list[Script]: + def combine(x, y): + return x | y -def get_scripts(directory) -> list[Script]: + feed = reduce(combine, feeds) scripts = [] - # use path glob? - file_generator = ( - (Path(root) / file_str) - for root, _, files in os.walk(directory) - for file_str in files - if file_str.endswith(EXTENSIONS) - ) + if feed & Feed.COMMON: + scripts.extend(get_scripts(root / "common")) + if feed & Feed.FEED_21_04: + scripts.extend(get_scripts(root / "21.04")) + if feed & Feed.FEED_22_04: + scripts.extend(get_scripts(root / "22.04")) - for path in file_generator: + return scripts + + +def get_scripts(directory: Path) -> list[Script]: + scripts = [] + + for path in directory.rglob("*.nasl"): try: content = path.read_text(encoding=CURRENT_ENCODING) except Exception as e: @@ -358,9 +375,19 @@ def check_deprecated_dependencies(graph) -> Result: def main(): args = parse_args() + logging.basicConfig( level=args.log.upper(), format="%(levelname)s: %(message)s" ) + if args.root is None: + vtdir = os.environ.get("VTDIR") + if not vtdir: + raise RuntimeError( + "The environment variable 'VTDIR' is not set, and no path was provided." + ) + args.root = Path(vtdir) + logging.info(f"using root {vtdir} from 'VTDIR'") + logging.info("starting troubadix dependency analysis") scripts = get_feed(args.root, args.feed) @@ -380,12 +407,9 @@ def main(): reporter = Reporter(args.verbose) reporter.report(results) - has_errors = any(result.has_errors() for result in results) - has_warnings = any(result.has_warnings() for result in results) - - if has_errors: + if any(result.has_errors() for result in results): return 1 - elif has_warnings: + elif any(result.has_warnings() for result in results): return 2 else: return 0 From 332a2377eabb1abdda404bec028f70fc1733e5fc Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Wed, 22 Jan 2025 14:05:24 +0100 Subject: [PATCH 10/21] Change: correct dependency graph unittests not fixed in last commit --- tests/standalone_plugins/test_dependency_graph.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py index ade82e35..773752cf 100644 --- a/tests/standalone_plugins/test_dependency_graph.py +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -8,6 +8,7 @@ from unittest.mock import patch from troubadix.standalone_plugins.dependency_graph import ( + Feed, Script, create_graph, get_feed, @@ -25,7 +26,7 @@ def test_parse_args_ok(self): "prog", NASL_DIR, "--feed", - "22.04", + "feed_22_04", "--log", "info", ] @@ -33,7 +34,7 @@ def test_parse_args_ok(self): args = parse_args() self.assertTrue(args) self.assertEqual(args.root, Path(NASL_DIR)) - self.assertEqual(args.feed, "22.04") + self.assertEqual(args.feed, [Feed.FEED_22_04]) self.assertEqual(args.log, "info") @patch("sys.stderr", new_callable=StringIO) @@ -45,7 +46,7 @@ def test_parse_args_no_dir(self, mock_stderr): self.assertRegex(mock_stderr.getvalue(), "invalid directory_type") def test_get_feed(self): - feed = "full" + feed = [Feed.FULL] scripts = get_feed(Path(NASL_DIR), feed) self.assertEqual(len(scripts), 6) From 83265176974cf027e46bda584119b403de0acda2 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Fri, 24 Jan 2025 12:14:13 +0100 Subject: [PATCH 11/21] Change: dependency graph minor refactors --- troubadix/standalone_plugins/dependency_graph.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index 254f1bf1..ddb8d69d 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -25,7 +25,6 @@ VTCategory, ) -EXTENSIONS = (".nasl",) # not sure if inc files can also have dependencies DEPENDENCY_PATTERN = _get_special_script_tag_pattern( "dependencies", flags=re.DOTALL | re.MULTILINE ) @@ -145,13 +144,8 @@ def parse_args() -> Namespace: return parser.parse_args() -# Usefull? Or is full only ever used and can therfore be removed? def get_feed(root, feeds: list[Feed]) -> list[Script]: - - def combine(x, y): - return x | y - - feed = reduce(combine, feeds) + feed = reduce((lambda x, y: x | y), feeds) scripts = [] if feed & Feed.COMMON: scripts.extend(get_scripts(root / "common")) @@ -179,7 +173,7 @@ def get_scripts(directory: Path) -> list[Script]: feed = determine_feed(relative_path) dependencies = extract_dependencies(content) category = extract_category(content) - deprecated = extract_deprecated_status(content) + deprecated = bool(DEPRECATED_PATTERN.search(content)) scripts.append( Script(name, feed, dependencies, category, deprecated) ) @@ -197,10 +191,6 @@ def determine_feed(script_relative_path: Path) -> str: return "community" -def extract_deprecated_status(content) -> bool: - return bool(DEPRECATED_PATTERN.search(content)) - - def split_dependencies(value: str) -> list[str]: """ removes blank lines, strips comments, cleans dependencies, From 288c3dc1e9e0e50438a5794fc95dd6a9771e2a3b Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Mon, 27 Jan 2025 15:46:55 +0100 Subject: [PATCH 12/21] Change: dependency graph rename variables --- .../test_dependency_graph.py | 4 +- .../standalone_plugins/dependency_graph.py | 54 +++++++++++++------ 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py index 773752cf..57e11e6b 100644 --- a/tests/standalone_plugins/test_dependency_graph.py +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -8,6 +8,7 @@ from unittest.mock import patch from troubadix.standalone_plugins.dependency_graph import ( + Dependency, Feed, Script, create_graph, @@ -51,8 +52,9 @@ def test_get_feed(self): self.assertEqual(len(scripts), 6) def test_create_graph(self): + dependency1 = Dependency("bar.nasl", False) scripts = [ - Script("foo.nasl", "community", [("bar.nasl", False)], 0, False), + Script("foo.nasl", "community", [dependency1], 0, False), Script("bar.nasl", "enterprise", [], 0, False), ] graph = create_graph(scripts) diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index ddb8d69d..6e23530c 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -9,6 +9,7 @@ from enum import Flag, auto from functools import reduce from pathlib import Path +from typing import NamedTuple import networkx as nx @@ -30,7 +31,7 @@ ) CATEGORY_PATTERN = get_special_script_tag_pattern(SpecialScriptTag.CATEGORY) DEPRECATED_PATTERN = get_script_tag_pattern(ScriptTag.DEPRECATED) -IF_BLOCK_PATTERN = re.compile( +ENTERPRISE_FEED_CHECK_PATTERN = re.compile( r'if\s*\(FEED_NAME\s*==\s*"GSF"\s*\|\|\s*FEED_NAME\s*==\s*"GEF"\s*\|\|\s*FEED_NAME\s*==\s*"SCM"\)\s*' r"(?:\{[^}]*\}\s*|[^\{;]*;)" ) # Matches specific if blocks used to gate code to run only for enterprise feeds @@ -54,11 +55,18 @@ def feed_type(value: str) -> Feed: raise ArgumentTypeError(f"Invalid Feed value: '{value}'") +class Dependency(NamedTuple): + name: str + # Indicates whether the dependency will only run if an enterprise feed is used. + # Controlled by a specific if check. Does not indicate the script's feed. + is_enterprise_feed: bool + + @dataclass class Script: name: str feed: str - dependencies: list[tuple[str, bool]] # (dependency_name, is_gated) + dependencies: list[Dependency] category: int deprecated: bool @@ -206,22 +214,24 @@ def split_dependencies(value: str) -> list[str]: ] -def extract_dependencies(content: str) -> list[tuple[str, bool]]: +def extract_dependencies(content: str) -> list[Dependency]: dependencies = [] if_blocks = [ (match.start(), match.end()) - for match in IF_BLOCK_PATTERN.finditer(content) + for match in ENTERPRISE_FEED_CHECK_PATTERN.finditer(content) ] for match in DEPENDENCY_PATTERN.finditer(content): start, end = match.span() - is_gated = any( + is_enterprise_feed = any( start >= block_start and end <= block_end for block_start, block_end in if_blocks ) dep_list = split_dependencies(match.group("value")) - dependencies.extend((dep, is_gated) for dep in dep_list) + dependencies.extend( + Dependency(dep, is_enterprise_feed) for dep in dep_list + ) return dependencies @@ -244,8 +254,12 @@ def create_graph(scripts: list[Script]): category=script.category, deprecated=script.deprecated, ) - for dep, is_gated in script.dependencies: - graph.add_edge(script.name, dep, is_gated=is_gated) + for dependency in script.dependencies: + graph.add_edge( + script.name, + dependency.name, + is_enterprise_feed=dependency.is_enterprise_feed, + ) return graph @@ -275,7 +289,9 @@ def check_missing_dependencies( logs the scripts dependending on the missing script """ errors = [] - dependencies = {dep for script in scripts for dep, _ in script.dependencies} + dependencies = { + dep.name for script in scripts for dep in script.dependencies + } script_names = {script.name for script in scripts} missing_dependencies = dependencies - script_names @@ -302,17 +318,19 @@ def check_cycles(graph) -> Result: return Result(name="check_cycles", errors=errors) -def cross_feed_dependencies(graph, gated_status: bool) -> list[tuple[str, str]]: +def cross_feed_dependencies( + graph, is_enterprise_checked: bool +) -> list[tuple[str, str]]: """ creates a list of script and dependency for scripts in community feed that depend on scripts in enterprise folders """ cross_feed_dependencies = [ (u, v) - for u, v, is_gated in graph.edges.data("is_gated") + for u, v, is_enterprise_feed in graph.edges.data("is_enterprise_feed") if graph.nodes[u]["feed"] == "community" and graph.nodes[v].get("feed", "unknown") == "enterprise" - and is_gated == gated_status + and is_enterprise_feed == is_enterprise_checked ] # unknown as standard value due to non existend nodes not having a feed value return cross_feed_dependencies @@ -320,17 +338,19 @@ def cross_feed_dependencies(graph, gated_status: bool) -> list[tuple[str, str]]: def check_cross_feed_dependecies(graph) -> Result: """ Checks if scripts in the community feed have dependencies to enterprise scripts, - and if they are contained within a gate. + and if they are correctly contained within a is_enterprise_feed check. """ - gated_cfd = cross_feed_dependencies(graph, gated_status=True) + gated_cfd = cross_feed_dependencies(graph, is_enterprise_checked=True) warnings = [ - f"gated cross-feed-dependency: {dependent} depends on {dependency}" + f"cross-feed-dependency: {dependent}(community feed) " + f"depends on {dependency}(enterprise feed)" for dependent, dependency in gated_cfd ] - ungated_cfd = cross_feed_dependencies(graph, gated_status=False) + ungated_cfd = cross_feed_dependencies(graph, is_enterprise_checked=False) errors = [ - f"ungated cross-feed-dependency: {dependent} depends on {dependency}" + f"unchecked cross-feed-dependency: {dependent}(community feed) " + f"depends on {dependency}(enterprise feed), but the current feed is not properly checked" for dependent, dependency in ungated_cfd ] From 9c4f3e6655e44913079b9d734165fd3a29a2ed06 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Tue, 28 Jan 2025 12:04:23 +0100 Subject: [PATCH 13/21] Add: dir and file type for argparser that only allow existing --- troubadix/argparser.py | 18 ++++++++++++++++++ troubadix/standalone_plugins/changed_oid.py | 10 ++-------- .../standalone_plugins/dependency_graph.py | 10 ++-------- troubadix/standalone_plugins/deprecate_vts.py | 15 +++------------ .../standalone_plugins/file_extensions.py | 18 +++--------------- .../standalone_plugins/last_modification.py | 16 ++++------------ troubadix/standalone_plugins/no_solution.py | 10 ++-------- .../standalone_plugins/version_updated.py | 10 ++-------- 8 files changed, 36 insertions(+), 71 deletions(-) diff --git a/troubadix/argparser.py b/troubadix/argparser.py index fb06efa6..e7bc7f52 100644 --- a/troubadix/argparser.py +++ b/troubadix/argparser.py @@ -26,6 +26,7 @@ from pontos.terminal import Terminal +# allows non existent paths and directory paths def directory_type(string: str) -> Path: directory_path = Path(string) if directory_path.exists() and not directory_path.is_dir(): @@ -33,6 +34,15 @@ def directory_type(string: str) -> Path: return directory_path +# allows only existing directory paths +def directory_type_existing(string: str) -> Path: + directory_path = Path(string) + if not directory_path.is_dir(): + raise ValueError(f"{string} is not a directory.") + return directory_path + + +# allows non existent paths and file paths def file_type(string: str) -> Path: file_path = Path(string) if file_path.exists() and not file_path.is_file(): @@ -40,6 +50,14 @@ def file_type(string: str) -> Path: return file_path +# allows only existing file paths +def file_type_existing(string: str) -> Path: + file_path = Path(string) + if not file_path.is_file(): + raise ValueError(f"{string} is not a file.") + return file_path + + def check_cpu_count(number: str) -> int: """Make sure this value is valid Default: use half of the available cores to not block the machine""" diff --git a/troubadix/standalone_plugins/changed_oid.py b/troubadix/standalone_plugins/changed_oid.py index ea43d8bb..a8e3375a 100644 --- a/troubadix/standalone_plugins/changed_oid.py +++ b/troubadix/standalone_plugins/changed_oid.py @@ -23,16 +23,10 @@ from pathlib import Path from typing import Iterable +from troubadix.argparser import file_type_existing from troubadix.standalone_plugins.common import git -def file_type(string: str) -> Path: - file_path = Path(string) - if not file_path.is_file(): - raise ValueError(f"{string} is not a file.") - return file_path - - def parse_args(args: Iterable[str]) -> Namespace: parser = ArgumentParser( description="Check for changed oid", @@ -52,7 +46,7 @@ def parse_args(args: Iterable[str]) -> Namespace: "-f", "--files", nargs="+", - type=file_type, + type=file_type_existing, default=[], help=( "List of files to diff. " diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index 6e23530c..0059c69a 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -13,6 +13,7 @@ import networkx as nx +from troubadix.argparser import directory_type_existing from troubadix.helper import CURRENT_ENCODING from troubadix.helper.helper import is_enterprise_folder from troubadix.helper.patterns import ( @@ -116,20 +117,13 @@ def print_errors(self, result: Result): print(f"error: {error}") -def directory_type(string: str) -> Path: - directory_path = Path(string) - if not directory_path.is_dir(): - raise ValueError(f"{string} is not a directory.") - return directory_path - - def parse_args() -> Namespace: parser = ArgumentParser( description="Tool for analysing the dependencies in the NASL repository.", ) parser.add_argument( "root", - type=directory_type, + type=directory_type_existing, nargs="?", help="directory that should be linted", ) diff --git a/troubadix/standalone_plugins/deprecate_vts.py b/troubadix/standalone_plugins/deprecate_vts.py index 2dba9b28..bb60b869 100644 --- a/troubadix/standalone_plugins/deprecate_vts.py +++ b/troubadix/standalone_plugins/deprecate_vts.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: 2024 Greenbone AG import re -from argparse import ArgumentParser, ArgumentTypeError, Namespace +from argparse import ArgumentParser, Namespace from dataclasses import dataclass from enum import Enum from pathlib import Path @@ -10,7 +10,7 @@ from pontos.terminal.terminal import ConsoleTerminal -from troubadix.argparser import directory_type, file_type +from troubadix.argparser import directory_type, file_type, file_type_existing from troubadix.helper.patterns import ( ScriptTag, SpecialScriptTag, @@ -37,15 +37,6 @@ class DeprecatedFile: KB_ITEMS_PATTERN = re.compile(r"set_kb_item\(.+\);") -def existing_file_type(string: str) -> Path: - file_path = Path(string) - if not file_path.exists(): - raise ArgumentTypeError(f'File "{string}" does not exist.') - if not file_path.is_file(): - raise ArgumentTypeError(f'"{string}" is not a file.') - return file_path - - def update_summary(file: DeprecatedFile, deprecation_reason: str) -> str: """Update the summary of the nasl script by adding the information that the script has been deprecated, and if possible, the oid of @@ -232,7 +223,7 @@ def parse_args(args: Iterable[str] = None) -> Namespace: "--from-file", metavar="", default=None, - type=existing_file_type, + type=file_type_existing, help=( "Path to a single file that contains a list of files " "to be deprecated, separated by new lines." diff --git a/troubadix/standalone_plugins/file_extensions.py b/troubadix/standalone_plugins/file_extensions.py index 732df390..42d7c50e 100644 --- a/troubadix/standalone_plugins/file_extensions.py +++ b/troubadix/standalone_plugins/file_extensions.py @@ -7,19 +7,7 @@ from pathlib import Path from typing import List - -def directory_type(string: str) -> Path: - directory_path = Path(string) - if not directory_path.is_dir(): - raise ValueError(f"{string} is not a directory.") - return directory_path - - -def file_type(string: str) -> Path: - file_path = Path(string) - if not file_path.is_file(): - raise ValueError(f"{string} is not a file.") - return file_path +from troubadix.argparser import directory_type_existing, file_type_existing def parse_args() -> Namespace: @@ -28,11 +16,11 @@ def parse_args() -> Namespace: ) parser.add_argument( "dir", - type=directory_type, + type=directory_type_existing, help="directory that should be linted", ) parser.add_argument( - "--ignore-file", type=file_type, help="path to ignore file" + "--ignore-file", type=file_type_existing, help="path to ignore file" ) parser.add_argument( "--gen-ignore-entries", diff --git a/troubadix/standalone_plugins/last_modification.py b/troubadix/standalone_plugins/last_modification.py index f004a69d..5bfe133f 100644 --- a/troubadix/standalone_plugins/last_modification.py +++ b/troubadix/standalone_plugins/last_modification.py @@ -20,13 +20,14 @@ import datetime import re import sys -from argparse import ArgumentParser, ArgumentTypeError, Namespace +from argparse import ArgumentParser, Namespace from pathlib import Path from typing import Iterable, Sequence from pontos.terminal import Terminal from pontos.terminal.terminal import ConsoleTerminal +from troubadix.argparser import file_type_existing from troubadix.helper import CURRENT_ENCODING from troubadix.helper.patterns import ( LAST_MODIFICATION_ANY_VALUE_PATTERN, @@ -35,15 +36,6 @@ from troubadix.troubadix import from_file -def existing_file_type(string: str) -> Path: - file_path = Path(string) - if not file_path.exists(): - raise ArgumentTypeError(f'File "{string}" does not exist.') - if not file_path.is_file(): - raise ArgumentTypeError(f'"{string}" is not a file.') - return file_path - - def update(nasl_file: Path, terminal: Terminal): file_content = nasl_file.read_text(encoding=CURRENT_ENCODING) @@ -104,12 +96,12 @@ def parse_args(args: Sequence[str] = None) -> Namespace: what_group.add_argument( "--files", nargs="+", - type=existing_file_type, + type=file_type_existing, help="List of files that should be updated", ) what_group.add_argument( "--from-file", - type=existing_file_type, + type=file_type_existing, help=( "Pass a file that contains a List of files " "containing paths to files, that should be " diff --git a/troubadix/standalone_plugins/no_solution.py b/troubadix/standalone_plugins/no_solution.py index e1da5c21..298a8660 100644 --- a/troubadix/standalone_plugins/no_solution.py +++ b/troubadix/standalone_plugins/no_solution.py @@ -25,6 +25,7 @@ from pontos.terminal.terminal import ConsoleTerminal +from troubadix.argparser import directory_type_existing from troubadix.helper import CURRENT_ENCODING from troubadix.helper.patterns import ( ScriptTag, @@ -49,13 +50,6 @@ MONTH_AS_DAYS = 365 / 12 -def directory_type(string: str) -> Path: - file_path = Path(string) - if not file_path.is_dir(): - raise ValueError(f"{string} is not a directory.") - return file_path - - def parse_solution_date(date_string: str) -> datetime: """Convert date string to date trying different formats""" @@ -83,7 +77,7 @@ def parse_args() -> Namespace: "-d", "--directory", dest="directory", - type=directory_type, + type=directory_type_existing, help="Specify the directory to scan for nasl scripts", ) diff --git a/troubadix/standalone_plugins/version_updated.py b/troubadix/standalone_plugins/version_updated.py index 43548068..fd04a7b7 100644 --- a/troubadix/standalone_plugins/version_updated.py +++ b/troubadix/standalone_plugins/version_updated.py @@ -23,6 +23,7 @@ from pathlib import Path from typing import Iterable, List +from troubadix.argparser import file_type_existing from troubadix.helper import is_ignore_file from troubadix.helper.patterns import ( LAST_MODIFICATION_ANY_VALUE_PATTERN, @@ -46,13 +47,6 @@ ] -def file_type(string: str) -> Path: - file_path = Path(string) - if not file_path.is_file(): - raise ValueError(f"{string} is not a file.") - return file_path - - def parse_args(args: Iterable[str]) -> Namespace: parser = ArgumentParser( description="Check for changed files that did not alter " @@ -73,7 +67,7 @@ def parse_args(args: Iterable[str]) -> Namespace: "-f", "--files", nargs="+", - type=file_type, + type=file_type_existing, default=[], help=( "List of files to diff. " From 226f2b50a99ca3ff17451d719862ed260d0a9cad Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Tue, 28 Jan 2025 12:29:26 +0100 Subject: [PATCH 14/21] Change: reuse method for splitting dependencies --- troubadix/plugins/dependencies.py | 26 +++++++++++-------- .../standalone_plugins/dependency_graph.py | 16 +----------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/troubadix/plugins/dependencies.py b/troubadix/plugins/dependencies.py index dca3c71d..0a34ce5f 100644 --- a/troubadix/plugins/dependencies.py +++ b/troubadix/plugins/dependencies.py @@ -31,6 +31,20 @@ ) +def split_dependencies(value: str) -> list[str]: + """ + Remove single and/or double quotes, spaces + and create a list by using the comma as a separator + additionally, check and filter for inline comments + """ + dependencies = [] + for line in value.splitlines(): + subject = line[: line.index("#")] if "#" in line else line + _dependencies = re.sub(r'[\'"\s]', "", subject).split(",") + dependencies += [dep for dep in _dependencies if dep != ""] + return dependencies + + class CheckDependencies(FilePlugin): name = "check_dependencies" @@ -60,17 +74,7 @@ def run( for match in matches: if match: - # Remove single and/or double quotes, spaces - # and create a list by using the comma as a separator - # additionally, check and filter for inline comments - dependencies = [] - - for line in match.group("value").splitlines(): - subject = line[: line.index("#")] if "#" in line else line - _dependencies = re.sub(r'[\'"\s]', "", subject).split(",") - dependencies += [dep for dep in _dependencies if dep != ""] - - for dep in dependencies: + for dep in split_dependencies(match.group("value")): if not any( (root / vers / dep).exists() for vers in FEED_VERSIONS ): diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py index 0059c69a..3a18040d 100644 --- a/troubadix/standalone_plugins/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph.py @@ -23,6 +23,7 @@ get_script_tag_pattern, get_special_script_tag_pattern, ) +from troubadix.plugins.dependencies import split_dependencies from troubadix.plugins.dependency_category_order import ( VTCategory, ) @@ -193,21 +194,6 @@ def determine_feed(script_relative_path: Path) -> str: return "community" -def split_dependencies(value: str) -> list[str]: - """ - removes blank lines, strips comments, cleans dependencies, - splits them by commas, and excludes empty strings. - """ - return [ - dep - for line in value.splitlines() - if line.strip() # Ignore blank or whitespace-only lines - # ignore comment, clean line of unwanted chars, split by ',' - for dep in re.sub(r'[\'"\s]', "", line.split("#", 1)[0]).split(",") - if dep # Include only non-empty - ] - - def extract_dependencies(content: str) -> list[Dependency]: dependencies = [] From a2c65e007e638486631f85f3f10a11bec0d3dd60 Mon Sep 17 00:00:00 2001 From: mbrinkhoff Date: Thu, 30 Jan 2025 00:06:36 +0100 Subject: [PATCH 15/21] Change: Cleanup dependency_graph --- pyproject.toml | 2 +- .../standalone_plugins/dependency_graph.py | 409 ------------------ .../dependency_graph/__init__.py | 2 + .../dependency_graph/checks.py | 127 ++++++ .../dependency_graph/cli.py | 59 +++ .../dependency_graph/dependency_graph.py | 212 +++++++++ .../dependency_graph/models.py | 46 ++ 7 files changed, 447 insertions(+), 410 deletions(-) delete mode 100644 troubadix/standalone_plugins/dependency_graph.py create mode 100644 troubadix/standalone_plugins/dependency_graph/__init__.py create mode 100644 troubadix/standalone_plugins/dependency_graph/checks.py create mode 100644 troubadix/standalone_plugins/dependency_graph/cli.py create mode 100644 troubadix/standalone_plugins/dependency_graph/dependency_graph.py create mode 100644 troubadix/standalone_plugins/dependency_graph/models.py diff --git a/pyproject.toml b/pyproject.toml index a81528c3..a6508c02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ troubadix-changed-cves = 'troubadix.standalone_plugins.changed_cves:main' troubadix-allowed-rev-diff = 'troubadix.standalone_plugins.allowed_rev_diff:main' troubadix-file-extensions = 'troubadix.standalone_plugins.file_extensions:main' troubadix-deprecate-vts = 'troubadix.standalone_plugins.deprecate_vts:main' -troubadix-dependency-graph = 'troubadix.standalone_plugins.dependency_graph:main' +troubadix-dependency-graph = 'troubadix.standalone_plugins.dependency_graph.dependency_graph:main' [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/troubadix/standalone_plugins/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph.py deleted file mode 100644 index 3a18040d..00000000 --- a/troubadix/standalone_plugins/dependency_graph.py +++ /dev/null @@ -1,409 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2024 Greenbone AG -import logging -import os -import re -import sys -from argparse import ArgumentParser, ArgumentTypeError, Namespace -from dataclasses import dataclass, field -from enum import Flag, auto -from functools import reduce -from pathlib import Path -from typing import NamedTuple - -import networkx as nx - -from troubadix.argparser import directory_type_existing -from troubadix.helper import CURRENT_ENCODING -from troubadix.helper.helper import is_enterprise_folder -from troubadix.helper.patterns import ( - ScriptTag, - SpecialScriptTag, - _get_special_script_tag_pattern, - get_script_tag_pattern, - get_special_script_tag_pattern, -) -from troubadix.plugins.dependencies import split_dependencies -from troubadix.plugins.dependency_category_order import ( - VTCategory, -) - -DEPENDENCY_PATTERN = _get_special_script_tag_pattern( - "dependencies", flags=re.DOTALL | re.MULTILINE -) -CATEGORY_PATTERN = get_special_script_tag_pattern(SpecialScriptTag.CATEGORY) -DEPRECATED_PATTERN = get_script_tag_pattern(ScriptTag.DEPRECATED) -ENTERPRISE_FEED_CHECK_PATTERN = re.compile( - r'if\s*\(FEED_NAME\s*==\s*"GSF"\s*\|\|\s*FEED_NAME\s*==\s*"GEF"\s*\|\|\s*FEED_NAME\s*==\s*"SCM"\)\s*' - r"(?:\{[^}]*\}\s*|[^\{;]*;)" -) # Matches specific if blocks used to gate code to run only for enterprise feeds - - -class Feed(Flag): - COMMON = auto() - FEED_21_04 = auto() - FEED_22_04 = auto() - FULL = COMMON | FEED_21_04 | FEED_22_04 - - def __str__(self): - # Make enum values user-friendly for argparse help - return self.name.lower() - - -def feed_type(value: str) -> Feed: - try: - return Feed[value.upper()] - except KeyError: - raise ArgumentTypeError(f"Invalid Feed value: '{value}'") - - -class Dependency(NamedTuple): - name: str - # Indicates whether the dependency will only run if an enterprise feed is used. - # Controlled by a specific if check. Does not indicate the script's feed. - is_enterprise_feed: bool - - -@dataclass -class Script: - name: str - feed: str - dependencies: list[Dependency] - category: int - deprecated: bool - - -@dataclass -class Result: - name: str - warnings: list[str] = field(default_factory=list) - errors: list[str] = field(default_factory=list) - - def has_errors(self) -> bool: - return bool(self.errors) - - def has_warnings(self) -> bool: - return bool(self.warnings) - - -class Reporter: - def __init__(self, verbosity) -> None: - self.verbosity = verbosity - - def report(self, results: list[Result]): - for result in results: - if self.verbosity >= 2: - self.print_statistic(result) - self.print_divider() - if self.verbosity >= 1: - self.print_warnings(result) - self.print_errors(result) - if self.verbosity >= 2: - self.print_divider("=") - - def print_divider(self, char="-", length=40): - print(char * length) - - def print_statistic(self, result: Result): - print( - f"{result.name} - warnings: {len(result.warnings)}, errors: {len(result.errors)}" - ) - - def print_warnings(self, result: Result): - for warning in result.warnings: - print(f"warning: {warning}") - - def print_errors(self, result: Result): - for error in result.errors: - print(f"error: {error}") - - -def parse_args() -> Namespace: - parser = ArgumentParser( - description="Tool for analysing the dependencies in the NASL repository.", - ) - parser.add_argument( - "root", - type=directory_type_existing, - nargs="?", - help="directory that should be linted", - ) - parser.add_argument( - "-f", - "--feed", - type=feed_type, - choices=Feed, - nargs="+", - default=[Feed.FULL], - help="feed", - ) - parser.add_argument( - "--log", - default="WARNING", - help="Set the logging level (INFO, WARNING, ERROR)", - ) - parser.add_argument("-v", "--verbose", action="count", default=0) - - return parser.parse_args() - - -def get_feed(root, feeds: list[Feed]) -> list[Script]: - feed = reduce((lambda x, y: x | y), feeds) - scripts = [] - if feed & Feed.COMMON: - scripts.extend(get_scripts(root / "common")) - if feed & Feed.FEED_21_04: - scripts.extend(get_scripts(root / "21.04")) - if feed & Feed.FEED_22_04: - scripts.extend(get_scripts(root / "22.04")) - - return scripts - - -def get_scripts(directory: Path) -> list[Script]: - scripts = [] - - for path in directory.rglob("*.nasl"): - try: - content = path.read_text(encoding=CURRENT_ENCODING) - except Exception as e: - logging.error(f"Error reading file {path}: {e}") - continue - - try: - relative_path = path.relative_to(directory) # used as identifier - name = str(relative_path) - feed = determine_feed(relative_path) - dependencies = extract_dependencies(content) - category = extract_category(content) - deprecated = bool(DEPRECATED_PATTERN.search(content)) - scripts.append( - Script(name, feed, dependencies, category, deprecated) - ) - except Exception as e: - logging.error(f"Error processing {path}: {e}") - - return scripts - - -def determine_feed(script_relative_path: Path) -> str: - parts = script_relative_path.parts - if is_enterprise_folder(parts[0]): - return "enterprise" - else: - return "community" - - -def extract_dependencies(content: str) -> list[Dependency]: - dependencies = [] - - if_blocks = [ - (match.start(), match.end()) - for match in ENTERPRISE_FEED_CHECK_PATTERN.finditer(content) - ] - - for match in DEPENDENCY_PATTERN.finditer(content): - start, end = match.span() - is_enterprise_feed = any( - start >= block_start and end <= block_end - for block_start, block_end in if_blocks - ) - dep_list = split_dependencies(match.group("value")) - dependencies.extend( - Dependency(dep, is_enterprise_feed) for dep in dep_list - ) - - return dependencies - - -def extract_category(content) -> int: - match = CATEGORY_PATTERN.search(content) - category_value = match.group("value") - return VTCategory[category_value] - - -def create_graph(scripts: list[Script]): - graph = nx.DiGraph() - - # Add nodes and edges based on dependencies - for script in scripts: - # explicit add incase the script has no dependencies - graph.add_node( - script.name, - feed=script.feed, - category=script.category, - deprecated=script.deprecated, - ) - for dependency in script.dependencies: - graph.add_edge( - script.name, - dependency.name, - is_enterprise_feed=dependency.is_enterprise_feed, - ) - return graph - - -def check_duplicates(scripts: list[Script]) -> Result: - """ - checks for a script depending on a script multiple times - """ - warnings = [] - for script in scripts: - dependencies = [dep for dep, _ in script.dependencies] - duplicates = { - dep for dep in dependencies if dependencies.count(dep) > 1 - } - if duplicates: - msg = f"Duplicate dependencies in {script.name}: {', '.join(duplicates)}" - warnings.append(msg) - - return Result(name="check_duplicates", warnings=warnings) - - -def check_missing_dependencies( - scripts: list[Script], graph: nx.DiGraph -) -> Result: - """ - Checks if any scripts that are depended on are missing from - the list of scripts created from the local file system, - logs the scripts dependending on the missing script - """ - errors = [] - dependencies = { - dep.name for script in scripts for dep in script.dependencies - } - script_names = {script.name for script in scripts} - missing_dependencies = dependencies - script_names - - for missing in missing_dependencies: - depending_scripts = graph.predecessors(missing) - msg = f"missing dependency file: {missing}:" - for script in depending_scripts: - msg += f"\n - used by: {script}" - errors.append(msg) - - return Result(name="missing_dependencies", errors=errors) - - -def check_cycles(graph) -> Result: - """ - checks for cyclic dependencies - """ - if nx.is_directed_acyclic_graph(graph): - return Result(name="check_cycles") - - cycles = nx.simple_cycles(graph) - - errors = [f"cyclic dependency: {cycle}" for cycle in cycles] - return Result(name="check_cycles", errors=errors) - - -def cross_feed_dependencies( - graph, is_enterprise_checked: bool -) -> list[tuple[str, str]]: - """ - creates a list of script and dependency for scripts - in community feed that depend on scripts in enterprise folders - """ - cross_feed_dependencies = [ - (u, v) - for u, v, is_enterprise_feed in graph.edges.data("is_enterprise_feed") - if graph.nodes[u]["feed"] == "community" - and graph.nodes[v].get("feed", "unknown") == "enterprise" - and is_enterprise_feed == is_enterprise_checked - ] # unknown as standard value due to non existend nodes not having a feed value - return cross_feed_dependencies - - -def check_cross_feed_dependecies(graph) -> Result: - """ - Checks if scripts in the community feed have dependencies to enterprise scripts, - and if they are correctly contained within a is_enterprise_feed check. - """ - gated_cfd = cross_feed_dependencies(graph, is_enterprise_checked=True) - warnings = [ - f"cross-feed-dependency: {dependent}(community feed) " - f"depends on {dependency}(enterprise feed)" - for dependent, dependency in gated_cfd - ] - - ungated_cfd = cross_feed_dependencies(graph, is_enterprise_checked=False) - errors = [ - f"unchecked cross-feed-dependency: {dependent}(community feed) " - f"depends on {dependency}(enterprise feed), but the current feed is not properly checked" - for dependent, dependency in ungated_cfd - ] - - return Result( - name="check_cross_feed_dependencies", warnings=warnings, errors=errors - ) - - -def check_category_order(graph) -> Result: - problematic_edges = [ - (dependent, dependency) - for dependent, dependency in graph.edges() - if graph.nodes[dependent]["category"] - < graph.nodes[dependency].get("category", -1) - ] - - errors = [ - f"{dependent} depends on {dependency} which has a lower category order" - for dependent, dependency in problematic_edges - ] - return Result(name="check_category_order", errors=errors) - - -def check_deprecated_dependencies(graph) -> Result: - errors = [ - f"{dependent} depends on deprectated script {dependency}" - for dependent, dependency in graph.edges() - if graph.nodes[dependency].get("deprecated", False) - ] - return Result(name="check_deprecated_dependencies", errors=errors) - - -def main(): - args = parse_args() - - logging.basicConfig( - level=args.log.upper(), format="%(levelname)s: %(message)s" - ) - if args.root is None: - vtdir = os.environ.get("VTDIR") - if not vtdir: - raise RuntimeError( - "The environment variable 'VTDIR' is not set, and no path was provided." - ) - args.root = Path(vtdir) - logging.info(f"using root {vtdir} from 'VTDIR'") - - logging.info("starting troubadix dependency analysis") - - scripts = get_feed(args.root, args.feed) - graph = create_graph(scripts) - - logging.info(f"nodes (scripts) in graph: {graph.number_of_nodes()}") - logging.info(f"edges (dependencies) in graph: {graph.number_of_edges()}") - - results = [ - check_duplicates(scripts), - check_missing_dependencies(scripts, graph), - check_cycles(graph), - check_cross_feed_dependecies(graph), - check_category_order(graph), - check_deprecated_dependencies(graph), - ] - reporter = Reporter(args.verbose) - reporter.report(results) - - if any(result.has_errors() for result in results): - return 1 - elif any(result.has_warnings() for result in results): - return 2 - else: - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/troubadix/standalone_plugins/dependency_graph/__init__.py b/troubadix/standalone_plugins/dependency_graph/__init__.py new file mode 100644 index 00000000..8eba994f --- /dev/null +++ b/troubadix/standalone_plugins/dependency_graph/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2025 Greenbone AG diff --git a/troubadix/standalone_plugins/dependency_graph/checks.py b/troubadix/standalone_plugins/dependency_graph/checks.py new file mode 100644 index 00000000..2ee50dc5 --- /dev/null +++ b/troubadix/standalone_plugins/dependency_graph/checks.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2025 Greenbone AG + + +import networkx as nx + +from .models import Result, Script + + +def check_duplicates(scripts: list[Script]) -> Result: + """ + checks for a script depending on a script multiple times + """ + warnings = [] + for script in scripts: + dependencies = [dep.name for dep in script.dependencies] + duplicates = { + dep for dep in dependencies if dependencies.count(dep) > 1 + } + if duplicates: + msg = f"Duplicate dependencies in {script.name}: {', '.join(duplicates)}" + warnings.append(msg) + + return Result(name="check_duplicates", warnings=warnings) + + +def check_missing_dependencies( + scripts: list[Script], graph: nx.DiGraph +) -> Result: + """ + Checks if any scripts that are depended on are missing from + the list of scripts created from the local file system, + logs the scripts dependending on the missing script + """ + errors = [] + dependencies = { + dep.name for script in scripts for dep in script.dependencies + } + script_names = {script.name for script in scripts} + missing_dependencies = dependencies - script_names + + for missing in missing_dependencies: + depending_scripts = graph.predecessors(missing) + msg = f"missing dependency file: {missing}:" + for script in depending_scripts: + msg += f"\n - used by: {script}" + errors.append(msg) + + return Result(name="missing_dependencies", errors=errors) + + +def check_cycles(graph) -> Result: + """ + checks for cyclic dependencies + """ + if nx.is_directed_acyclic_graph(graph): + return Result(name="check_cycles") + + cycles = nx.simple_cycles(graph) + + errors = [f"cyclic dependency: {cycle}" for cycle in cycles] + return Result(name="check_cycles", errors=errors) + + +def cross_feed_dependencies( + graph, is_enterprise_checked: bool +) -> list[tuple[str, str]]: + """ + creates a list of script and dependency for scripts + in community feed that depend on scripts in enterprise folders + """ + cross_feed_dependencies = [ + (u, v) + for u, v, is_enterprise_feed in graph.edges.data("is_enterprise_feed") + if graph.nodes[u]["feed"] == "community" + and graph.nodes[v].get("feed", "unknown") == "enterprise" + and is_enterprise_feed == is_enterprise_checked + ] # unknown as standard value due to non existend nodes not having a feed value + return cross_feed_dependencies + + +def check_cross_feed_dependecies(graph) -> Result: + """ + Checks if scripts in the community feed have dependencies to enterprise scripts, + and if they are correctly contained within a is_enterprise_feed check. + """ + gated_cfd = cross_feed_dependencies(graph, is_enterprise_checked=True) + warnings = [ + f"cross-feed-dependency: {dependent}(community feed) " + f"depends on {dependency}(enterprise feed)" + for dependent, dependency in gated_cfd + ] + + ungated_cfd = cross_feed_dependencies(graph, is_enterprise_checked=False) + errors = [ + f"unchecked cross-feed-dependency: {dependent}(community feed) " + f"depends on {dependency}(enterprise feed), but the current feed is not properly checked" + for dependent, dependency in ungated_cfd + ] + + return Result( + name="check_cross_feed_dependencies", warnings=warnings, errors=errors + ) + + +def check_category_order(graph) -> Result: + problematic_edges = [ + (dependent, dependency) + for dependent, dependency in graph.edges() + if graph.nodes[dependent]["category"] + < graph.nodes[dependency].get("category", -1) + ] + + errors = [ + f"{dependent} depends on {dependency} which has a lower category order" + for dependent, dependency in problematic_edges + ] + return Result(name="check_category_order", errors=errors) + + +def check_deprecated_dependencies(graph) -> Result: + errors = [ + f"{dependent} depends on deprectated script {dependency}" + for dependent, dependency in graph.edges() + if graph.nodes[dependency].get("deprecated", False) + ] + return Result(name="check_deprecated_dependencies", errors=errors) diff --git a/troubadix/standalone_plugins/dependency_graph/cli.py b/troubadix/standalone_plugins/dependency_graph/cli.py new file mode 100644 index 00000000..28848dcc --- /dev/null +++ b/troubadix/standalone_plugins/dependency_graph/cli.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2025 Greenbone AG + + +import logging +import os +from argparse import ArgumentError, ArgumentParser, ArgumentTypeError, Namespace +from pathlib import Path + +from troubadix.argparser import directory_type_existing + +from .models import Feed + + +def feed_type(value: str) -> Feed: + try: + return Feed[value.upper()] + except KeyError: + raise ArgumentTypeError(f"Invalid Feed value: '{value}'") + + +def parse_args() -> Namespace: + parser = ArgumentParser( + description="Tool for analysing the dependencies in the NASL repository.", + ) + parser.add_argument( + "root", + type=directory_type_existing, + default=os.environ.get("VTDIR"), + help="directory that should be linted", + ) + parser.add_argument( + "-f", + "--feed", + type=feed_type, + choices=Feed, + nargs="+", + default=[Feed.FULL], + help="feed", + ) + parser.add_argument( + "--log", + default="WARNING", + help="Set the logging level (INFO, WARNING, ERROR)", + ) + parser.add_argument("-v", "--verbose", action="count", default=0) + + args = parser.parse_args() + + if not args.root: + vtdir = os.environ.get("VTDIR") + if not vtdir: + raise ArgumentError( + "The environment variable 'VTDIR' is not set, and no path was provided." + ) + args.root = Path(vtdir) + logging.info(f"using root {vtdir} from 'VTDIR'") + + return args diff --git a/troubadix/standalone_plugins/dependency_graph/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph/dependency_graph.py new file mode 100644 index 00000000..03bc65ea --- /dev/null +++ b/troubadix/standalone_plugins/dependency_graph/dependency_graph.py @@ -0,0 +1,212 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2025 Greenbone AG + + +import logging +import re +import sys +from functools import reduce +from pathlib import Path + +import networkx as nx + +from troubadix.helper import CURRENT_ENCODING +from troubadix.helper.helper import is_enterprise_folder +from troubadix.helper.patterns import ( + ScriptTag, + SpecialScriptTag, + _get_special_script_tag_pattern, + get_script_tag_pattern, + get_special_script_tag_pattern, +) +from troubadix.plugins.dependencies import split_dependencies +from troubadix.plugins.dependency_category_order import ( + VTCategory, +) + +from .checks import ( + check_category_order, + check_cross_feed_dependecies, + check_cycles, + check_deprecated_dependencies, + check_duplicates, + check_missing_dependencies, +) +from .cli import Feed, parse_args +from .models import Dependency, Result, Script + +DEPENDENCY_PATTERN = _get_special_script_tag_pattern( + "dependencies", flags=re.DOTALL | re.MULTILINE +) +CATEGORY_PATTERN = get_special_script_tag_pattern(SpecialScriptTag.CATEGORY) +DEPRECATED_PATTERN = get_script_tag_pattern(ScriptTag.DEPRECATED) +ENTERPRISE_FEED_CHECK_PATTERN = re.compile( + r'if\s*\(FEED_NAME\s*==\s*"GSF"\s*\|\|\s*FEED_NAME\s*==\s*"GEF"\s*\|\|\s*FEED_NAME\s*==\s*"SCM"\)\s*' + r"(?:\{[^}]*\}\s*|[^\{;]*;)" +) # Matches specific if blocks used to gate code to run only for enterprise feeds + + +class Reporter: + def __init__(self, verbosity) -> None: + self.verbosity = verbosity + + def report(self, results: list[Result]): + for result in results: + if self.verbosity >= 2: + self.print_statistic(result) + self.print_divider() + if self.verbosity >= 1: + self.print_warnings(result) + self.print_errors(result) + if self.verbosity >= 2: + self.print_divider("=") + + def print_divider(self, char="-", length=40): + print(char * length) + + def print_statistic(self, result: Result): + print( + f"{result.name} - warnings: {len(result.warnings)}, errors: {len(result.errors)}" + ) + + def print_warnings(self, result: Result): + for warning in result.warnings: + print(f"warning: {warning}") + + def print_errors(self, result: Result): + for error in result.errors: + print(f"error: {error}") + + +def get_feed(root, feeds: list[Feed]) -> list[Script]: + feed = reduce((lambda x, y: x | y), feeds) + scripts = [] + if feed & Feed.COMMON: + scripts.extend(get_scripts(root / "common")) + if feed & Feed.FEED_21_04: + scripts.extend(get_scripts(root / "21.04")) + if feed & Feed.FEED_22_04: + scripts.extend(get_scripts(root / "22.04")) + + return scripts + + +def get_scripts(directory: Path) -> list[Script]: + scripts = [] + + for path in directory.rglob("*.nasl"): + try: + content = path.read_text(encoding=CURRENT_ENCODING) + except Exception as e: + logging.error(f"Error reading file {path}: {e}") + continue + + try: + relative_path = path.relative_to(directory) # used as identifier + name = str(relative_path) + feed = determine_feed(relative_path) + dependencies = extract_dependencies(content) + category = extract_category(content) + deprecated = bool(DEPRECATED_PATTERN.search(content)) + scripts.append( + Script(name, feed, dependencies, category, deprecated) + ) + except Exception as e: + logging.error(f"Error processing {path}: {e}") + + return scripts + + +def determine_feed(script_relative_path: Path) -> str: + parts = script_relative_path.parts + if is_enterprise_folder(parts[0]): + return "enterprise" + else: + return "community" + + +def extract_dependencies(content: str) -> list[Dependency]: + dependencies = [] + + if_blocks = [ + (match.start(), match.end()) + for match in ENTERPRISE_FEED_CHECK_PATTERN.finditer(content) + ] + + for match in DEPENDENCY_PATTERN.finditer(content): + start, end = match.span() + is_enterprise_feed = any( + start >= block_start and end <= block_end + for block_start, block_end in if_blocks + ) + dep_list = split_dependencies(match.group("value")) + dependencies.extend( + Dependency(dep, is_enterprise_feed) for dep in dep_list + ) + + return dependencies + + +def extract_category(content) -> int: + match = CATEGORY_PATTERN.search(content) + category_value = match.group("value") + return VTCategory[category_value] + + +def create_graph(scripts: list[Script]): + graph = nx.DiGraph() + + # Add nodes and edges based on dependencies + for script in scripts: + # explicit add incase the script has no dependencies + graph.add_node( + script.name, + feed=script.feed, + category=script.category, + deprecated=script.deprecated, + ) + for dependency in script.dependencies: + graph.add_edge( + script.name, + dependency.name, + is_enterprise_feed=dependency.is_enterprise_feed, + ) + return graph + + +def main(): + args = parse_args() + + logging.basicConfig( + level=args.log.upper(), format="%(levelname)s: %(message)s" + ) + + logging.info("starting troubadix dependency analysis") + + scripts = get_feed(args.root, args.feed) + graph = create_graph(scripts) + + logging.info(f"nodes (scripts) in graph: {graph.number_of_nodes()}") + logging.info(f"edges (dependencies) in graph: {graph.number_of_edges()}") + + results = [ + check_duplicates(scripts), + check_missing_dependencies(scripts, graph), + check_cycles(graph), + check_cross_feed_dependecies(graph), + check_category_order(graph), + check_deprecated_dependencies(graph), + ] + reporter = Reporter(args.verbose) + reporter.report(results) + + if any(result.has_errors() for result in results): + return 1 + elif any(result.has_warnings() for result in results): + return 2 + else: + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/troubadix/standalone_plugins/dependency_graph/models.py b/troubadix/standalone_plugins/dependency_graph/models.py new file mode 100644 index 00000000..b9aced03 --- /dev/null +++ b/troubadix/standalone_plugins/dependency_graph/models.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2025 Greenbone AG + +from dataclasses import dataclass, field +from enum import Flag, auto + + +class Feed(Flag): + COMMON = auto() + FEED_21_04 = auto() + FEED_22_04 = auto() + FULL = COMMON | FEED_21_04 | FEED_22_04 + + def __str__(self): + # Make enum values user-friendly for argparse help + return self.name.lower() + + +@dataclass +class Dependency: + name: str + # Indicates whether the dependency will only run if an enterprise feed is used. + # Controlled by a specific if check. Does not indicate the script's feed. + is_enterprise_feed: bool + + +@dataclass +class Script: + name: str + feed: str + dependencies: list[Dependency] + category: int + deprecated: bool + + +@dataclass +class Result: + name: str + warnings: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + def has_errors(self) -> bool: + return bool(self.errors) + + def has_warnings(self) -> bool: + return bool(self.warnings) From 2a7db5984ca0af0ee7d7d203f2592c6f662efaae Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Thu, 30 Jan 2025 16:23:12 +0100 Subject: [PATCH 16/21] Change: use restructured imports in dependency graph test --- tests/standalone_plugins/test_dependency_graph.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py index 57e11e6b..8287ab35 100644 --- a/tests/standalone_plugins/test_dependency_graph.py +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -7,14 +7,16 @@ from pathlib import Path from unittest.mock import patch -from troubadix.standalone_plugins.dependency_graph import ( - Dependency, - Feed, - Script, +from troubadix.standalone_plugins.dependency_graph.cli import parse_args +from troubadix.standalone_plugins.dependency_graph.dependency_graph import ( create_graph, get_feed, main, - parse_args, +) +from troubadix.standalone_plugins.dependency_graph.models import ( + Dependency, + Feed, + Script, ) NASL_DIR = "tests/standalone_plugins/nasl" From ea65f2eb3bdaa753a4f998b983938eec35332dd6 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Mon, 10 Feb 2025 11:46:54 +0100 Subject: [PATCH 17/21] Change: dependency graph optimize duplicate check --- troubadix/standalone_plugins/dependency_graph/checks.py | 9 +++++---- .../dependency_graph/dependency_graph.py | 4 ++-- troubadix/standalone_plugins/dependency_graph/models.py | 6 ------ 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/troubadix/standalone_plugins/dependency_graph/checks.py b/troubadix/standalone_plugins/dependency_graph/checks.py index 2ee50dc5..55f16123 100644 --- a/troubadix/standalone_plugins/dependency_graph/checks.py +++ b/troubadix/standalone_plugins/dependency_graph/checks.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: 2025 Greenbone AG +from collections import Counter + import networkx as nx from .models import Result, Script @@ -13,10 +15,9 @@ def check_duplicates(scripts: list[Script]) -> Result: """ warnings = [] for script in scripts: - dependencies = [dep.name for dep in script.dependencies] - duplicates = { - dep for dep in dependencies if dependencies.count(dep) > 1 - } + counter = Counter(dep.name for dep in script.dependencies) + duplicates = [dep for dep, count in counter.items() if count > 1] + if duplicates: msg = f"Duplicate dependencies in {script.name}: {', '.join(duplicates)}" warnings.append(msg) diff --git a/troubadix/standalone_plugins/dependency_graph/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph/dependency_graph.py index 03bc65ea..0c7b24be 100644 --- a/troubadix/standalone_plugins/dependency_graph/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph/dependency_graph.py @@ -200,9 +200,9 @@ def main(): reporter = Reporter(args.verbose) reporter.report(results) - if any(result.has_errors() for result in results): + if any(result.errors for result in results): return 1 - elif any(result.has_warnings() for result in results): + elif any(result.warnings for result in results): return 2 else: return 0 diff --git a/troubadix/standalone_plugins/dependency_graph/models.py b/troubadix/standalone_plugins/dependency_graph/models.py index b9aced03..41f4bda0 100644 --- a/troubadix/standalone_plugins/dependency_graph/models.py +++ b/troubadix/standalone_plugins/dependency_graph/models.py @@ -38,9 +38,3 @@ class Result: name: str warnings: list[str] = field(default_factory=list) errors: list[str] = field(default_factory=list) - - def has_errors(self) -> bool: - return bool(self.errors) - - def has_warnings(self) -> bool: - return bool(self.warnings) From 3df4d6d108cbfbf920c50fa23bfef2077487f498 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Mon, 10 Feb 2025 12:22:03 +0100 Subject: [PATCH 18/21] Change: dependency graph cli incorrect error and remove logging --- .../standalone_plugins/dependency_graph/cli.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/troubadix/standalone_plugins/dependency_graph/cli.py b/troubadix/standalone_plugins/dependency_graph/cli.py index 28848dcc..27577f12 100644 --- a/troubadix/standalone_plugins/dependency_graph/cli.py +++ b/troubadix/standalone_plugins/dependency_graph/cli.py @@ -2,9 +2,8 @@ # SPDX-FileCopyrightText: 2025 Greenbone AG -import logging import os -from argparse import ArgumentError, ArgumentParser, ArgumentTypeError, Namespace +from argparse import ArgumentParser, ArgumentTypeError, Namespace from pathlib import Path from troubadix.argparser import directory_type_existing @@ -24,10 +23,10 @@ def parse_args() -> Namespace: description="Tool for analysing the dependencies in the NASL repository.", ) parser.add_argument( - "root", + "-r", + "--root", type=directory_type_existing, - default=os.environ.get("VTDIR"), - help="directory that should be linted", + help="root for nasl directory that should be linted, uses $VTDIR if no path is given", ) parser.add_argument( "-f", @@ -50,10 +49,10 @@ def parse_args() -> Namespace: if not args.root: vtdir = os.environ.get("VTDIR") if not vtdir: - raise ArgumentError( - "The environment variable 'VTDIR' is not set, and no path was provided." + raise ValueError( + "The environment variable 'VTDIR' is not set," + " and no root path with '--root' was provided." ) args.root = Path(vtdir) - logging.info(f"using root {vtdir} from 'VTDIR'") return args From 8e9d84da8ce1a3e232ac6ba77ebb08e7bad8484f Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Tue, 11 Feb 2025 11:43:31 +0100 Subject: [PATCH 19/21] Change: dependency graph tests --- .../test_dependency_graph.py | 189 +++++++++++++++--- 1 file changed, 157 insertions(+), 32 deletions(-) diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py index 8287ab35..9149a58e 100644 --- a/tests/standalone_plugins/test_dependency_graph.py +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -1,58 +1,159 @@ # SPDX-License-Identifier: GPL-3.0-or-later # SPDX-FileCopyrightText: 2024 Greenbone AG -import sys +import os import unittest -from contextlib import redirect_stderr, redirect_stdout from io import StringIO from pathlib import Path from unittest.mock import patch +from troubadix.plugins.dependency_category_order import VTCategory from troubadix.standalone_plugins.dependency_graph.cli import parse_args from troubadix.standalone_plugins.dependency_graph.dependency_graph import ( + Reporter, create_graph, + determine_feed, + extract_category, + extract_dependencies, get_feed, + get_scripts, main, ) from troubadix.standalone_plugins.dependency_graph.models import ( Dependency, Feed, + Result, Script, ) -NASL_DIR = "tests/standalone_plugins/nasl" +class TestReporter(unittest.TestCase): + def setUp(self): + self.result = Result( + name="TestScript", + warnings=["duplicate dependencies"], + errors=["missing dependencies"], + ) -class TestDependencyGraph(unittest.TestCase): + @patch("sys.stdout", new_callable=StringIO) + def test_report_verbosity_2(self, mock_stdout): + reporter = Reporter(verbosity=2) + reporter.report([self.result]) - def test_parse_args_ok(self): - test_args = [ + output = mock_stdout.getvalue() + + self.assertIn("TestScript - warnings: 1, errors: 1", output) + self.assertIn("warning: duplicate dependencies", output) + self.assertIn("error: missing dependencies", output) + + +class TestCLIArgs(unittest.TestCase): + @patch( + "sys.argv", + [ "prog", - NASL_DIR, + "--root", + "tests/standalone_plugins/nasl", "--feed", "feed_22_04", "--log", "info", - ] - with patch.object(sys, "argv", test_args): - args = parse_args() - self.assertTrue(args) - self.assertEqual(args.root, Path(NASL_DIR)) - self.assertEqual(args.feed, [Feed.FEED_22_04]) - self.assertEqual(args.log, "info") + ], + ) + def test_parse_args_ok(self): + args = parse_args() + self.assertEqual(args.root, Path("tests/standalone_plugins/nasl")) + self.assertEqual(args.feed, [Feed.FEED_22_04]) + self.assertEqual(args.log, "info") @patch("sys.stderr", new_callable=StringIO) + @patch("sys.argv", ["prog", "--root", "not_real_dir"]) def test_parse_args_no_dir(self, mock_stderr): - test_args = ["prog", "not_real_dir"] - with patch.object(sys, "argv", test_args): - with self.assertRaises(SystemExit): - parse_args() - self.assertRegex(mock_stderr.getvalue(), "invalid directory_type") + with self.assertRaises(SystemExit): + parse_args() + self.assertRegex(mock_stderr.getvalue(), "invalid directory_type") + + @patch("sys.stderr", new_callable=StringIO) + @patch( + "sys.argv", + [ + "prog", + "--root", + "tests/standalone_plugins/nasl", + "--feed", + "invalid_feed", + ], + ) + def test_parse_args_invalid_feed(self, mock_stderr): + with self.assertRaises(SystemExit): + parse_args() + self.assertRegex(mock_stderr.getvalue(), "Invalid Feed value") + + @patch.dict(os.environ, {"VTDIR": "/mock/env/path"}) + @patch("sys.argv", ["prog"]) + def test_parse_args_with_env(self): + args = parse_args() + self.assertEqual(args.root, Path("/mock/env/path")) + + @patch("sys.argv", ["prog", "--root", "tests/standalone_plugins/nasl"]) + def test_parse_args_defaults(self): + args = parse_args() + self.assertEqual(args.log, "WARNING") + self.assertEqual(args.feed, [Feed.FULL]) + + +class TestDependencyGraph(unittest.TestCase): + + def setUp(self) -> None: + self.local_root = "tests/standalone_plugins/nasl" + self.script_content = """ +if(description) +{ + script_category(ACT_GATHER_INFO); + script_dependencies( "foo.nasl", "foo.nasl" ); + + if(FEED_NAME == "GSF" || FEED_NAME == "GEF" || FEED_NAME == "SCM") + script_dependencies("gsf/enterprise_script.nasl"); + + exit(0); +} + """ def test_get_feed(self): + # includes: + # - get_scripts + # - determine_feed + # - extract_dependencies + # - extract_categories feed = [Feed.FULL] - scripts = get_feed(Path(NASL_DIR), feed) + scripts = get_feed(Path(self.local_root), feed) self.assertEqual(len(scripts), 6) + @patch("pathlib.Path.read_text") + def test_get_scripts(self, mock_read_text): + mock_read_text.return_value = self.script_content + scripts = get_scripts(Path(self.local_root) / "common") + self.assertEqual(len(scripts), 4) + self.assertEqual(scripts[0].name, "foo.nasl") + self.assertEqual(scripts[0].feed, "community") + self.assertEqual(len(scripts[0].dependencies), 3) + self.assertEqual(scripts[0].category, VTCategory.ACT_GATHER_INFO) + self.assertEqual(scripts[0].deprecated, False) + + def test_determine_feed(self): + community_script = Path("foo/script.nasl") + enterprise_script = Path("gsf/script.nasl") + + self.assertEqual(determine_feed(community_script), "community") + self.assertEqual(determine_feed(enterprise_script), "enterprise") + + def test_extract_dependencies(self): + dependencies = extract_dependencies(self.script_content) + self.assertEqual(len(dependencies), 3) + + def test_extract_category(self): + category = extract_category(self.script_content) + self.assertEqual(category, VTCategory.ACT_GATHER_INFO) + def test_create_graph(self): dependency1 = Dependency("bar.nasl", False) scripts = [ @@ -62,15 +163,39 @@ def test_create_graph(self): graph = create_graph(scripts) self.assertEqual(len(list(graph.nodes)), 2) - def test_full_run(self): - test_args = [ - "prog", - NASL_DIR, - ] - with ( - redirect_stdout(StringIO()), - redirect_stderr(StringIO()), - patch.object(sys, "argv", test_args), - ): - return_code = main() - self.assertEqual(return_code, 1) + @patch("sys.stdout", new_callable=StringIO) # mock_stdout (second argument) + @patch("sys.stderr", new_callable=StringIO) # mock_stderr (first argument) + @patch( + "sys.argv", ["prog", "--root", "tests/standalone_plugins/nasl", "-v"] + ) # no argument + def test_full_run(self, mock_stderr, mock_stdout): + return_code = main() + output = mock_stdout.getvalue() + + self.assertIn("error: missing dependency file: missing.nasl:", output) + self.assertIn( + "error: cyclic dependency: ", # order is random so can't match the output + output, + ) + self.assertIn( + "error: unchecked cross-feed-dependency: foo.nasl(community feed) depends on" + " gsf/enterprise_script.nasl(enterprise feed), but the" + " current feed is not properly checked", + output, + ) + self.assertIn( + "error: bar.nasl depends on foo.nasl which has a lower category order", + output, + ) + self.assertIn( + "error: foo.nasl depends on deprectated script foobar.nasl", output + ) + self.assertIn( + "warning: Duplicate dependencies in bar.nasl: foo.nasl", output + ) + self.assertIn( + "warning: cross-feed-dependency: bar.nasl(community feed)" + " depends on gsf/enterprise_script.nasl(enterprise feed)", + output, + ) + self.assertEqual(return_code, 1) From abcb837208130ab24ad5b864073deb6d123d1140 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Tue, 11 Feb 2025 12:14:43 +0100 Subject: [PATCH 20/21] Change: dependency graph tests remove test with result in random order --- tests/standalone_plugins/test_dependency_graph.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py index 9149a58e..b7dc17c1 100644 --- a/tests/standalone_plugins/test_dependency_graph.py +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -119,11 +119,6 @@ def setUp(self) -> None: """ def test_get_feed(self): - # includes: - # - get_scripts - # - determine_feed - # - extract_dependencies - # - extract_categories feed = [Feed.FULL] scripts = get_feed(Path(self.local_root), feed) self.assertEqual(len(scripts), 6) @@ -133,7 +128,6 @@ def test_get_scripts(self, mock_read_text): mock_read_text.return_value = self.script_content scripts = get_scripts(Path(self.local_root) / "common") self.assertEqual(len(scripts), 4) - self.assertEqual(scripts[0].name, "foo.nasl") self.assertEqual(scripts[0].feed, "community") self.assertEqual(len(scripts[0].dependencies), 3) self.assertEqual(scripts[0].category, VTCategory.ACT_GATHER_INFO) @@ -149,6 +143,12 @@ def test_determine_feed(self): def test_extract_dependencies(self): dependencies = extract_dependencies(self.script_content) self.assertEqual(len(dependencies), 3) + self.assertEqual(dependencies[0].name, "foo.nasl") + self.assertEqual(dependencies[1].name, "foo.nasl") + self.assertEqual(dependencies[2].name, "gsf/enterprise_script.nasl") + self.assertEqual(dependencies[0].is_enterprise_feed, False) + self.assertEqual(dependencies[1].is_enterprise_feed, False) + self.assertEqual(dependencies[2].is_enterprise_feed, True) def test_extract_category(self): category = extract_category(self.script_content) From cc8d77eded84c62f991ef59bae8d66d1bb1f8105 Mon Sep 17 00:00:00 2001 From: Niklas Hargarter Date: Mon, 3 Mar 2025 11:31:14 +0100 Subject: [PATCH 21/21] Change: typos in dependency graph files --- tests/standalone_plugins/test_dependency_graph.py | 2 +- troubadix/standalone_plugins/dependency_graph/checks.py | 6 +++--- .../standalone_plugins/dependency_graph/dependency_graph.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/standalone_plugins/test_dependency_graph.py b/tests/standalone_plugins/test_dependency_graph.py index b7dc17c1..258c1fc2 100644 --- a/tests/standalone_plugins/test_dependency_graph.py +++ b/tests/standalone_plugins/test_dependency_graph.py @@ -188,7 +188,7 @@ def test_full_run(self, mock_stderr, mock_stdout): output, ) self.assertIn( - "error: foo.nasl depends on deprectated script foobar.nasl", output + "error: foo.nasl depends on deprecated script foobar.nasl", output ) self.assertIn( "warning: Duplicate dependencies in bar.nasl: foo.nasl", output diff --git a/troubadix/standalone_plugins/dependency_graph/checks.py b/troubadix/standalone_plugins/dependency_graph/checks.py index 55f16123..a88911a6 100644 --- a/troubadix/standalone_plugins/dependency_graph/checks.py +++ b/troubadix/standalone_plugins/dependency_graph/checks.py @@ -76,11 +76,11 @@ def cross_feed_dependencies( if graph.nodes[u]["feed"] == "community" and graph.nodes[v].get("feed", "unknown") == "enterprise" and is_enterprise_feed == is_enterprise_checked - ] # unknown as standard value due to non existend nodes not having a feed value + ] # unknown as standard value due to non existent nodes not having a feed value return cross_feed_dependencies -def check_cross_feed_dependecies(graph) -> Result: +def check_cross_feed_dependencies(graph) -> Result: """ Checks if scripts in the community feed have dependencies to enterprise scripts, and if they are correctly contained within a is_enterprise_feed check. @@ -121,7 +121,7 @@ def check_category_order(graph) -> Result: def check_deprecated_dependencies(graph) -> Result: errors = [ - f"{dependent} depends on deprectated script {dependency}" + f"{dependent} depends on deprecated script {dependency}" for dependent, dependency in graph.edges() if graph.nodes[dependency].get("deprecated", False) ] diff --git a/troubadix/standalone_plugins/dependency_graph/dependency_graph.py b/troubadix/standalone_plugins/dependency_graph/dependency_graph.py index 0c7b24be..f6ec00ff 100644 --- a/troubadix/standalone_plugins/dependency_graph/dependency_graph.py +++ b/troubadix/standalone_plugins/dependency_graph/dependency_graph.py @@ -26,7 +26,7 @@ from .checks import ( check_category_order, - check_cross_feed_dependecies, + check_cross_feed_dependencies, check_cycles, check_deprecated_dependencies, check_duplicates, @@ -193,7 +193,7 @@ def main(): check_duplicates(scripts), check_missing_dependencies(scripts, graph), check_cycles(graph), - check_cross_feed_dependecies(graph), + check_cross_feed_dependencies(graph), check_category_order(graph), check_deprecated_dependencies(graph), ]