Merge pull request #10 from commit-0/errors

More lints.

Author: wenting-zhao

commit0/__main__.py

@@ -1,27 +1,18 @@
-import argparse
 import commit0.harness.run_pytest_ids
 import commit0.harness.build
 import commit0.harness.setup
+import sys
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser(description="Commit0 version control system")
-    subparsers = parser.add_subparsers(dest="command", help="Available commands")
-
-    commit0.harness.setup.add_init_args(subparsers.add_parser("clone"))
-    commit0.harness.build.add_init_args(subparsers.add_parser("build"))
-    commit0.harness.run_pytest_ids.add_init_args(subparsers.add_parser("test"))
-
-    args = parser.parse_args()
-
-    if args.command == "clone":
-        commit0.harness.setup.run(args)
-    elif args.command == "build":
-        commit0.harness.build.run(args)
-    elif args.command == "test":
-        commit0.harness.run_pytest_ids.run(args)
-    else:
-        parser.print_help()
+    command = sys.argv[1]
+
+    if command == "clone":
+        commit0.harness.setup.main()
+    elif command == "build":
+        commit0.harness.build.main()
+    elif command == "test":
+        commit0.harness.run_pytest_ids.main()
 
 
 if __name__ == "__main__":

commit0/harness/build.py

@@ -1,24 +1,24 @@
-import argparse
 import logging
 
 import docker
 from datasets import load_dataset
 from typing import Iterator
+
+from omegaconf import DictConfig
 from commit0.harness.docker_build import build_repo_images
 from commit0.harness.spec import make_spec
+from commit0.harness.constants import RepoInstance
+import hydra
 
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
 )
 logger = logging.getLogger(__name__)
 
 
-def main(
-    hf_name: str,
-    base_dir: str,
-    config_file: str,
-) -> None:
-    dataset: Iterator[RepoInstance] = load_dataset(hf_name, split="test")
+@hydra.main(version_base=None, config_path="configs", config_name="base")
+def main(config: DictConfig) -> None:
+    dataset: Iterator[RepoInstance] = load_dataset(hf_name, split="test")  # type: ignore
     specs = []
     for example in dataset:
         spec = make_spec(example)
@@ -29,34 +29,4 @@ def main(
     logger.info("Done building docker images")
 
 
-def add_init_args(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
-        "--hf_name",
-        type=str,
-        help="HF dataset name",
-        default="wentingzhao/commit0_docstring",
-    )
-    parser.add_argument(
-        "--base_dir",
-        type=str,
-        default="repos/",
-        help="base directory to write repos to",
-    )
-    parser.add_argument(
-        "--config_file",
-        type=str,
-        default="config.yml",
-        help="where to write config file to",
-    )
-    parser.set_defaults(func=run)
-
-
-def run(args: argparse.Namespace) -> None:
-    main(
-        hf_name=args.hf_name,
-        base_dir=args.base_dir,
-        config_file=args.config_file,
-    )
-
-
 __all__ = []

commit0/harness/docker_build.py

@@ -6,7 +6,7 @@
 from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
-from typing import List
+from typing import Any
 
 from commit0.harness.constants import (
     BASE_IMAGE_BUILD_DIR,
@@ -22,7 +22,7 @@ def __init__(self, image_name: str, message: str, logger: logging.Logger):
         super().__init__(message)
         self.super_str = super().__str__()
         self.image_name = image_name
-        self.log_path = logger.log_file
+        self.log_path = ""  # logger.log_file
         self.logger = logger
 
     def __str__(self):
@@ -185,7 +185,7 @@ def build_base_images(client: docker.DockerClient, dataset: list) -> None:
 def get_repo_configs_to_build(
     client: docker.DockerClient,
     dataset: list,
-) -> dict[str]:
+) -> dict[str, Any]:
     """Returns a dictionary of image names to build scripts and dockerfiles for repo images.
     Returns only the repo images that need to be built.
 
@@ -229,7 +229,7 @@ def build_repo_images(
     client: docker.DockerClient,
     dataset: list,
     max_workers: int = 4,
-) -> (List[str], List[str]):
+) -> tuple[list[str], list[str]]:
     """Builds the repo images required for the dataset if they do not already exist.
 
     Args:

commit0/harness/docker_utils.py

@@ -192,7 +192,7 @@ def write_to_container(container: Container, data: str, dst: Path) -> None:
 
 def cleanup_container(
     client: docker.DockerClient,
-    container: docker.Container,
+    container: Container,
     logger: Union[None, str, logging.Logger],
 ) -> None:
     """Stop and remove a Docker container.
@@ -251,6 +251,7 @@ def log_info(x: str) -> None:
         )
         try:
             # Get the PID of the container
+            assert container_id is not None
             container_info = client.api.inspect_container(container_id)
             pid = container_info["State"].get("Pid", 0)
 
@@ -338,7 +339,6 @@ def log_info(x: str) -> None:
         def log_error(x: str) -> None:
             return None
 
-        raise_error = True
     else:
         assert isinstance(logger, logging.Logger)
 
@@ -349,8 +349,6 @@ def log_error(x: str) -> None:
         def log_info(x: str) -> None:
             logger.info(x)
 
-        raise_error = False
-
     container = None
     try:
         log_info(f"Creating container for {image_name}...")
@@ -368,6 +366,7 @@ def log_info(x: str) -> None:
         # If an error occurs, clean up the container and raise an exception
         log_error(f"Error creating container for {image_name}: {e}")
         log_info(traceback.format_exc())
+        assert container is not None
         cleanup_container(client, container, logger)
         raise
 

commit0/harness/run_pytest_ids.py

@@ -1,10 +1,8 @@
-import argparse
 from datasets import load_dataset
 import docker
 from enum import StrEnum, auto
 import os
 import traceback
-import yaml
 from pathlib import Path
 import logging
 
@@ -87,7 +85,7 @@ def run_docker(
         # Check the exit code of the command
         if exit_code == 0:
             copy_from_container(container, report_file, Path(log_dir / "report.json"))
-            delete_file_from_container(container, report_file)
+            delete_file_from_container(container, str(report_file))
 
     except EvaluationError as e:
         error_msg = traceback.format_exc()
@@ -97,11 +95,12 @@ def run_docker(
         error_msg = (
             f"Error in running pytest for {spec.repo}: {e}\n"
             f"{traceback.format_exc()}\n"
-            f"Check ({logger.log_file}) for more information."
+            # f"Check ({logger.log_file}) for more information."
         )
         logger.error(error_msg)
     finally:
         # Remove repo container + image, close logger
+        assert container is not None
         cleanup_container(client, container, logger)
         close_logger(logger)
 
@@ -164,6 +163,7 @@ def run_modal(
         # execute tests
         process = sandbox.exec("bash", "-c", "/bin/bash /eval.sh")
         output = []
+        line = ""
         for line in process.stdout:
             output.append(line)
         output = "".join(line)
@@ -178,8 +178,6 @@ def run_modal(
         print(output_s)
 
         timed_out = False
-        total_runtime = 1
-
         test_output = extract_test_output(
             output_s, "--json-report --json-report-file=report.json"
         )
@@ -202,10 +200,12 @@ def run_modal(
 def main(config: DictConfig) -> None:
     OmegaConf.to_yaml(config)
     dataset = load_dataset(config.dataset_name, split="test")
+    spec = None
     for example in dataset:
         if example["repo"].endswith(config.repo):
             spec = make_spec(example)
             break
+    assert spec is not None, "No spec available"
 
     hashed_test_ids = get_hash_string(config.test_ids)
     # set up logging
@@ -231,8 +231,4 @@ def main(config: DictConfig) -> None:
         run_modal(spec, logger, eval_file, config.timeout, log_dir)
 
 
-def run(args: argparse.Namespace) -> None:
-    main()
-
-
 __all__ = []

commit0/harness/setup.py

@@ -3,9 +3,8 @@
 
 import docker
 import hydra
-import yaml
 from datasets import load_dataset
-from omegaconf import DictConfig, OmegaConf
+from omegaconf import DictConfig
 
 from typing import Iterator
 from commit0.harness.utils import clone_repo
@@ -45,8 +44,4 @@ def main(config: DictConfig) -> None:
     logger.info("Done building docker images")
 
 
-def run() -> None:
-    main()
-
-
 __all__ = []

commit0/harness/utils.py

@@ -14,7 +14,7 @@ def __init__(self, repo: str, message: str, logger: logging.Logger):
         super().__init__(message)
         self.super_str = super().__str__()
         self.repo = repo
-        self.log_file = logger.log_file
+        self.log_file = ""  # logger.log_file
         self.logger = logger
 
     def __str__(self):