From a4c34c1487d8a32e5c484f1e07459ca5d40153be Mon Sep 17 00:00:00 2001 From: Caelean Date: Thu, 8 Feb 2024 15:38:30 -0800 Subject: [PATCH 1/5] write files at the end --- modguard/init.py | 34 ++++++++++++++++++++++++++++++---- modguard/parsing/boundary.py | 20 +++++--------------- modguard/parsing/public.py | 20 ++++++++------------ 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/modguard/init.py b/modguard/init.py index 0f3efe0a..71aa7528 100644 --- a/modguard/init.py +++ b/modguard/init.py @@ -1,12 +1,24 @@ +from dataclasses import dataclass +from enum import StrEnum import os from . import errors from .check import check_import from .core import PublicMember from .parsing import utils -from .parsing.boundary import ensure_boundary, build_boundary_trie +from .parsing.boundary import add_boundary, has_boundary, build_boundary_trie from .parsing.imports import get_imports from .parsing.public import mark_as_public +class WriteOperation(StrEnum): + BOUNDARY = 'boundary' + PUBLIC = 'public' + +@dataclass +class FileWriteInformation: + location: str + operation: WriteOperation + member_name: str = '' + def init_project(root: str, exclude_paths: list[str] = None): # Core functionality: @@ -20,16 +32,19 @@ def init_project(root: str, exclude_paths: list[str] = None): root = utils.canonical(root) exclude_paths = list(map(utils.canonical, exclude_paths)) if exclude_paths else None + write_operations: list[FileWriteInformation] = [] + boundary_trie = build_boundary_trie(root, exclude_paths=exclude_paths) initial_boundary_paths = [ boundary.full_path for boundary in boundary_trie if boundary.full_path ] for dirpath in utils.walk_pypackages(root, exclude_paths=exclude_paths): - added_boundary = ensure_boundary(dirpath + "/__init__.py") - if added_boundary: + filepath = dirpath + '/__init__.py' + if not has_boundary(filepath): dir_mod_path = utils.file_to_module_path(dirpath) boundary_trie.insert(dir_mod_path) + write_operations.append(FileWriteInformation(location=filepath, operation=WriteOperation.BOUNDARY)) for file_path in utils.walk_pyfiles(root, exclude_paths=exclude_paths): mod_path = utils.file_to_module_path(file_path) @@ -61,9 +76,20 @@ def init_project(root: str, exclude_paths: list[str] = None): file_path, member_name = utils.module_to_file_path(import_mod_path) try: - mark_as_public(file_path, member_name) + write_operations.append(FileWriteInformation(location=file_path, operation=WriteOperation.PUBLIC, member_name=member_name)) violated_boundary.add_public_member(PublicMember(name=import_mod_path)) except errors.ModguardError: print( f"Skipping member {member_name} in {file_path}; could not mark as public" ) + # After we've completed our pass on inserting boundaries and public members, write to files + for write_op in write_operations: + try: + if write_op.operation == WriteOperation.BOUNDARY: + add_boundary(write_op.location) + if write_op.operation == WriteOperation.PUBLIC: + mark_as_public(write_op.location, write_op.member_name) + except errors.ModguardError: + print( + f'Error marking {write_op.operation} in {write_op.operation}' + ) diff --git a/modguard/parsing/boundary.py b/modguard/parsing/boundary.py index c9c54686..8f4a46dd 100644 --- a/modguard/parsing/boundary.py +++ b/modguard/parsing/boundary.py @@ -1,5 +1,6 @@ import ast import os +from typing import Optional from modguard.core.boundary import BoundaryTrie from modguard.errors import ModguardParseError @@ -67,23 +68,12 @@ def has_boundary(file_path: str) -> bool: BOUNDARY_PRELUDE = "import modguard\nmodguard.Boundary()\n" -def _add_boundary(file_path: str, file_content: str): - with open(file_path, "w") as file: - file.write(BOUNDARY_PRELUDE + file_content) - -@public -def ensure_boundary(file_path: str) -> bool: - with open(file_path, "r") as file: +def add_boundary(file_path: str) -> None: + with open(file_path, "r+") as file: file_content = file.read() - - if _has_boundary(file_path, file_content): - # Boundary already exists, don't need to create one - return False - - # Boundary doesn't exist, create one - _add_boundary(file_path, file_content) - return True + file.seek(0) + file.write(BOUNDARY_PRELUDE + file_content) @public diff --git a/modguard/parsing/public.py b/modguard/parsing/public.py index a70d545d..f0c1ef7c 100644 --- a/modguard/parsing/public.py +++ b/modguard/parsing/public.py @@ -246,23 +246,19 @@ def _public_module_prelude(should_import: bool = True) -> str: @public def mark_as_public(file_path: str, member_name: str = ""): - with open(file_path, "r") as file: + with open(file_path, "r+") as file: file_content = file.read() - - try: - parsed_ast = ast.parse(file_content) - except SyntaxError as e: - raise ModguardParseError(f"Syntax error in {file_path}: {e}") - - modguard_public_is_imported = is_modguard_imported(parsed_ast, "public") - - if not member_name: - with open(file_path, "w") as file: + file.seek(0) + try: + parsed_ast = ast.parse(file_content) + except SyntaxError as e: + raise ModguardParseError(f"Syntax error in {file_path}: {e}") + modguard_public_is_imported = is_modguard_imported(parsed_ast, "public") + if not member_name: file.write( _public_module_prelude(should_import=not modguard_public_is_imported) + file_content ) - return member_finder = MemberFinder(member_name) member_finder.visit(parsed_ast) From 20fac4d32317f2efbf0dd6f167eab02423da3a99 Mon Sep 17 00:00:00 2001 From: Caelean Date: Thu, 8 Feb 2024 15:43:21 -0800 Subject: [PATCH 2/5] ruff --- modguard/init.py | 30 ++++++++++++++++++++---------- modguard/parsing/boundary.py | 1 - tests/__init__.py | 1 + 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/modguard/init.py b/modguard/init.py index 2a680f72..c4a199c7 100644 --- a/modguard/init.py +++ b/modguard/init.py @@ -11,15 +11,17 @@ from .parsing.imports import get_imports from .parsing.public import mark_as_public + class WriteOperation(StrEnum): - BOUNDARY = 'boundary' - PUBLIC = 'public' + BOUNDARY = "boundary" + PUBLIC = "public" + @dataclass class FileWriteInformation: location: str operation: WriteOperation - member_name: str = '' + member_name: str = "" def init_project(root: str, exclude_paths: Optional[list[str]] = None): @@ -42,11 +44,15 @@ def init_project(root: str, exclude_paths: Optional[list[str]] = None): ] for dirpath in utils.walk_pypackages(root, exclude_paths=exclude_paths): - filepath = dirpath + '/__init__.py' + filepath = dirpath + "/__init__.py" if not has_boundary(filepath): dir_mod_path = utils.file_to_module_path(dirpath) boundary_trie.insert(dir_mod_path) - write_operations.append(FileWriteInformation(location=filepath, operation=WriteOperation.BOUNDARY)) + write_operations.append( + FileWriteInformation( + location=filepath, operation=WriteOperation.BOUNDARY + ) + ) for file_path in utils.walk_pyfiles(root, exclude_paths=exclude_paths): mod_path = utils.file_to_module_path(file_path) @@ -78,20 +84,24 @@ def init_project(root: str, exclude_paths: Optional[list[str]] = None): file_path, member_name = utils.module_to_file_path(import_mod_path) try: - write_operations.append(FileWriteInformation(location=file_path, operation=WriteOperation.PUBLIC, member_name=member_name)) + write_operations.append( + FileWriteInformation( + location=file_path, + operation=WriteOperation.PUBLIC, + member_name=member_name, + ) + ) violated_boundary.add_public_member(PublicMember(name=import_mod_path)) except errors.ModguardError: print( f"Skipping member {member_name} in {file_path}; could not mark as public" ) # After we've completed our pass on inserting boundaries and public members, write to files - for write_op in write_operations: + for write_op in write_operations: try: if write_op.operation == WriteOperation.BOUNDARY: add_boundary(write_op.location) if write_op.operation == WriteOperation.PUBLIC: mark_as_public(write_op.location, write_op.member_name) except errors.ModguardError: - print( - f'Error marking {write_op.operation} in {write_op.operation}' - ) + print(f"Error marking {write_op.operation} in {write_op.operation}") diff --git a/modguard/parsing/boundary.py b/modguard/parsing/boundary.py index 08c0c2e1..ab1afde2 100644 --- a/modguard/parsing/boundary.py +++ b/modguard/parsing/boundary.py @@ -67,7 +67,6 @@ def has_boundary(file_path: str) -> bool: BOUNDARY_PRELUDE = "import modguard\nmodguard.Boundary()\n" - def add_boundary(file_path: str) -> None: with open(file_path, "r+") as file: file_content = file.read() diff --git a/tests/__init__.py b/tests/__init__.py index e960b710..5c072682 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,3 @@ import modguard + modguard.Boundary() From f0177743db07fea6c6595300208a2c680d488a17 Mon Sep 17 00:00:00 2001 From: Caelean Date: Thu, 8 Feb 2024 15:51:32 -0800 Subject: [PATCH 3/5] continue on member operation --- modguard/parsing/public.py | 52 +++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/modguard/parsing/public.py b/modguard/parsing/public.py index 05ce8aeb..53ab20b0 100644 --- a/modguard/parsing/public.py +++ b/modguard/parsing/public.py @@ -265,32 +265,32 @@ def mark_as_public(file_path: str, member_name: str = ""): _public_module_prelude(should_import=not modguard_public_is_imported) + file_content ) + return - member_finder = MemberFinder(member_name) - member_finder.visit(parsed_ast) - if member_finder.matched_lineno is None: - raise ModguardParseError( - f"Failed to find member {member_name} in file {file_path}" - ) + member_finder = MemberFinder(member_name) + member_finder.visit(parsed_ast) + if member_finder.matched_lineno is None: + raise ModguardParseError( + f"Failed to find member {member_name} in file {file_path}" + ) + + normal_lineno = member_finder.matched_lineno - 1 + file_lines = file_content.splitlines(keepends=True) + if member_finder.matched_assignment: + # Insert a call to public for the member after the assignment + lines_to_write = [ + *file_lines[: normal_lineno + 1], + f"{PUBLIC_CALL}({member_name})\n", + *file_lines[normal_lineno + 1 :], + ] + else: + # Insert a decorator before the function or class definition + lines_to_write = [ + *file_lines[:normal_lineno], + PUBLIC_DECORATOR + "\n", + *file_lines[normal_lineno:], + ] + if not modguard_public_is_imported: + lines_to_write = [IMPORT_MODGUARD + "\n", *lines_to_write] - normal_lineno = member_finder.matched_lineno - 1 - file_lines = file_content.splitlines(keepends=True) - if member_finder.matched_assignment: - # Insert a call to public for the member after the assignment - lines_to_write = [ - *file_lines[: normal_lineno + 1], - f"{PUBLIC_CALL}({member_name})\n", - *file_lines[normal_lineno + 1 :], - ] - else: - # Insert a decorator before the function or class definition - lines_to_write = [ - *file_lines[:normal_lineno], - PUBLIC_DECORATOR + "\n", - *file_lines[normal_lineno:], - ] - if not modguard_public_is_imported: - lines_to_write = [IMPORT_MODGUARD + "\n", *lines_to_write] - - with open(file_path, "w") as file: file.write("".join(lines_to_write)) From 3dbf8ce82473f5d4f62511dd90a2b7d0db6bfbf6 Mon Sep 17 00:00:00 2001 From: Caelean Date: Thu, 8 Feb 2024 16:24:00 -0800 Subject: [PATCH 4/5] add boundary for new public members --- modguard/parsing/boundary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modguard/parsing/boundary.py b/modguard/parsing/boundary.py index ab1afde2..e47b7832 100644 --- a/modguard/parsing/boundary.py +++ b/modguard/parsing/boundary.py @@ -56,7 +56,7 @@ def _has_boundary(file_path: str, file_content: str) -> bool: boundary_finder.visit(parsed_ast) return boundary_finder.found_boundary - +@public def has_boundary(file_path: str) -> bool: with open(file_path, "r") as file: file_content = file.read() @@ -66,7 +66,7 @@ def has_boundary(file_path: str) -> bool: BOUNDARY_PRELUDE = "import modguard\nmodguard.Boundary()\n" - +@public def add_boundary(file_path: str) -> None: with open(file_path, "r+") as file: file_content = file.read() From daec75887d1d65e98b32541be6470e0de7487f3a Mon Sep 17 00:00:00 2001 From: Caelean Date: Thu, 8 Feb 2024 16:25:50 -0800 Subject: [PATCH 5/5] str enum not supported in older python versions --- modguard/init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modguard/init.py b/modguard/init.py index c4a199c7..e9782d19 100644 --- a/modguard/init.py +++ b/modguard/init.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from enum import StrEnum +from enum import Enum import os from typing import Optional @@ -12,7 +12,7 @@ from .parsing.public import mark_as_public -class WriteOperation(StrEnum): +class WriteOperation(Enum): BOUNDARY = "boundary" PUBLIC = "public"