Merge pull request #16 from commit-0/checks

GitHub actions for build pipeline

Author: srush

.github/workflows/system.yml

@@ -0,0 +1,26 @@
+name: system
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  system:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Set up Python
+        run: uv venv --python 3.12
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v3
+      - name: Install the project
+        run: uv sync
+      - name: Clone
+        run: uv run commit0 clone simpy
+      - name: Setup
+        run: uv run commit0 build simpy
+      - name: Test
+        run: uv run commit0 test-reference simpy tests/test_event.py::test_succeed

commit0/__main__.py

@@ -3,35 +3,58 @@
 import commit0.harness.setup
 import copy
 import sys
+import os
 import hydra
 from hydra.core.config_store import ConfigStore
 from commit0.configs.config_class import Commit0Config
+from commit0.harness.constants import COMMANDS, SPLIT
 
 
 def main() -> None:
     command = sys.argv[1]
+    if command not in COMMANDS:
+        raise ValueError(
+            f"command must be from {', '.join(COMMANDS)}, but you provided {command}"
+        )
     # type check config values
     cs = ConfigStore.instance()
-    cs.store(name="base", node=Commit0Config)
+    cs.store(name="user", node=Commit0Config)
     # have hydra to ignore all command-line arguments
     sys_argv = copy.deepcopy(sys.argv)
     sys.argv = [sys.argv[0]]
     hydra.initialize(version_base=None, config_path="configs")
-    config = hydra.compose(config_name="base")
+    config = hydra.compose(config_name="user")
     # after hydra gets all configs, put command-line arguments back
     sys.argv = sys_argv
+    # repo_split: split from command line has a higher priority than split in hydra
+    if command in ["clone", "build"]:
+        if len(sys.argv) == 3:
+            if sys.argv[2] not in SPLIT:
+                raise ValueError(
+                    f"repo split must be from {', '.join(SPLIT.keys())}, but you provided {sys.argv[2]}"
+                )
+            config.repo_split = sys.argv[2]
+    config.base_dir = os.path.abspath(config.base_dir)
 
     if command == "clone":
         commit0.harness.setup.main(
-            config.dataset_name, config.dataset_split, config.base_dir
+            config.dataset_name,
+            config.dataset_split,
+            config.repo_split,
+            config.base_dir,
         )
     elif command == "build":
         commit0.harness.build.main(
-            config.dataset_name, config.dataset_split, config.num_workers
+            config.dataset_name,
+            config.dataset_split,
+            config.repo_split,
+            config.num_workers,
         )
-    elif command == "test":
+    elif command == "test" or command == "test-reference":
         repo = sys.argv[2]
         test_ids = sys.argv[3]
+        if command == "test-reference":
+            config.branch = "reference"
         commit0.harness.run_pytest_ids.main(
             config.dataset_name,
             config.dataset_split,

commit0/configs/base.yaml

@@ -7,9 +7,9 @@ dataset_split: test
 
 # clone related
 base_dir: repos/
+repo_split: all
 
 # build related
-build: all
 num_workers: 8
 
 # test related

commit0/configs/config_class.py

@@ -9,10 +9,10 @@ class Commit0Config:
 
     # clone related
     base_dir: str
+    repo_split: str
 
     # build related
     # which repo to build, all or one repo
-    build: str
     num_workers: int
 
     # test related

commit0/configs/user.yaml

@@ -0,0 +1,3 @@
+defaults:
+  - base
+  - _self_

commit0/harness/build.py

@@ -6,24 +6,28 @@
 
 from commit0.harness.docker_build import build_repo_images
 from commit0.harness.spec import make_spec
-from commit0.harness.constants import RepoInstance
+from commit0.harness.constants import RepoInstance, SPLIT
 
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
 )
 logger = logging.getLogger(__name__)
 
 
-def main(dataset_name: str, dataset_split: str, num_workers: int) -> None:
+def main(
+    dataset_name: str, dataset_split: str, repo_split: str, num_workers: int
+) -> None:
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     specs = []
     for example in dataset:
+        repo_name = example["repo"].split("/")[-1]
+        if repo_split != "all" and repo_name not in SPLIT[repo_split]:
+            continue
         spec = make_spec(example)
         specs.append(spec)
 
     client = docker.from_env()
     build_repo_images(client, specs, num_workers)
-    logger.info("Done building docker images")
 
 
 __all__ = []

commit0/harness/constants.py

@@ -25,6 +25,88 @@ class RepoInstance(TypedDict):
 # Evaluation backends
 EVAL_BACKENDS = ["local", "modal"]
 
+# available commands
+COMMANDS = ["clone", "build", "test", "test-reference"]
+# repo splits
+SPLIT_MINITORCH = ["minitorch"]
+SPLIT_SIMPY = ["simpy"]
+SPLIT_LITE = [
+    "tinydb",
+    "simpy",
+    "deprecated",
+    "wcwidth",
+    "voluptuous",
+    "cachetools",
+    "imapclient",
+    "marshmallow",
+    "jinja",
+    "cookiecutter",
+]
+SPLIT_ALL = [
+    "statsmodels",
+    "python-progressbar",
+    "xarray",
+    "imbalanced-learn",
+    "web3.py",
+    "scrapy",
+    "seaborn",
+    "pypdf",
+    "pexpect",
+    "pytest",
+    "pylint",
+    "joblib",
+    "dulwich",
+    "virtualenv",
+    "minitorch",
+    "networkx",
+    "requests",
+    "sphinx",
+    "jedi",
+    "moviepy",
+    "loguru",
+    "paramiko",
+    "geopandas",
+    "bitstring",
+    "fastapi",
+    "chardet",
+    "tornado",
+    "python-prompt-toolkit",
+    "attrs",
+    "PyBoy",
+    "pydantic",
+    "filesystem_spec",
+    "tlslite-ng",
+    "graphene",
+    "mimesis",
+    "babel",
+    "dnspython",
+    "portalocker," "cookiecutter",
+    "pyjwt",
+    "python-rsa",
+    "more-itertools",
+    "simpy",
+    "click",
+    "fabric",
+    "jinja",
+    "flask",
+    "sqlparse",
+    "marshmallow",
+    "imapclient",
+    "tinydb",
+    "cachetools",
+    "voluptuous",
+    "parsel",
+    "wcwidth",
+    "deprecated",
+]
+
+SPLIT = {
+    "all": SPLIT_ALL,
+    "minitorch": SPLIT_MINITORCH,
+    "simpy": SPLIT_SIMPY,
+    "lite": SPLIT_LITE,
+}
+
 
 class ResolvedStatus(Enum):
     NO = "RESOLVED_NO"

commit0/harness/run_pytest_ids.py

@@ -7,6 +7,7 @@
 import logging
 
 from typing import Iterator
+from git import Repo
 from commit0.harness.constants import RUN_PYTEST_LOG_DIR, RepoInstance
 from commit0.harness.docker_build import (
     close_logger,
@@ -206,11 +207,13 @@ def main(
 ) -> None:
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     spec = None
+    example = None
     for example in dataset:
         if example["repo"].endswith(repo):
             spec = make_spec(example)
             break
     assert spec is not None, "No spec available"
+    assert example is not None, "No example available"
 
     hashed_test_ids = get_hash_string(test_ids)
     # set up logging
@@ -219,10 +222,17 @@ def main(
     log_file = log_dir / "run_pytest.log"
     logger = setup_logger(repo, log_file)
 
+    if branch == "reference":
+        commit_id = example["reference_commit"]
+    else:
+        local_repo = Repo(f"{base_dir}/{repo}")
+        local_branch = local_repo.branches[branch]
+        commit_id = local_branch.commit.hexsha
+
     # make eval file
     eval_script = spec.eval_script.format(
         local_repo=f"{base_dir}/{repo}",
-        branch_name=branch,
+        commit_id=commit_id,
         test_ids=test_ids,
         ip=get_ip(backend),
         user=get_user(),

commit0/harness/setup.py

@@ -1,14 +1,11 @@
 import logging
 import os
 
-import docker
 from datasets import load_dataset
 
 from typing import Iterator
 from commit0.harness.utils import clone_repo
-from commit0.harness.constants import REPO_IMAGE_BUILD_DIR, RepoInstance
-from commit0.harness.docker_build import build_repo_images
-from commit0.harness.spec import make_spec
+from commit0.harness.constants import RepoInstance, SPLIT
 
 
 logging.basicConfig(
@@ -17,28 +14,15 @@
 logger = logging.getLogger(__name__)
 
 
-def main(dataset_name: str, dataset_split: str, base_dir: str) -> None:
+def main(dataset_name: str, dataset_split: str, repo_split: str, base_dir: str) -> None:
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
-    out = dict()
-    specs = []
     for example in dataset:
-        spec = make_spec(example)
-        specs.append(spec)
         repo_name = example["repo"].split("/")[-1]
-        out[repo_name] = example
-        out[repo_name]["local_path"] = os.path.abspath(
-            os.path.join(base_dir, repo_name)
-        )
+        if repo_split != "all" and repo_name not in SPLIT[repo_split]:
+            continue
         clone_url = f"https://github.com/{example['repo']}.git"
-        clone_repo(
-            clone_url, out[repo_name]["local_path"], example["base_commit"], logger
-        )
-
-    logger.info("Start building docker images")
-    logger.info(f"Please check {REPO_IMAGE_BUILD_DIR} for build details")
-    client = docker.from_env()
-    build_repo_images(client, specs)
-    logger.info("Done building docker images")
+        clone_dir = os.path.abspath(os.path.join(base_dir, repo_name))
+        clone_repo(clone_url, clone_dir, example["base_commit"], logger)
 
 
 __all__ = []

commit0/harness/spec.py

@@ -154,11 +154,12 @@ def make_eval_script_list(instance: RepoInstance, repo_directory: str) -> list[s
         "source .venv/bin/activate",
         f"git remote add {origin_name} ssh://{{user}}@{{ip}}:{{local_repo}}",
         f"git fetch {origin_name}",
-        f"git checkout -b {{branch_name}} {origin_name}/{{branch_name}}",
+        "git checkout {commit_id}",
         "git status",
         f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json {{test_ids}}",
         f"git checkout {instance['base_commit']}",
-        f"git remote remove {origin_name}" "git status",
+        f"git remote remove {origin_name}",
+        "git status",
     ]
     return eval_script_list