call job_generator.py only one time to generate all job yaml files in…

…stead n times
alpaka-group · Oct 9, 2024 · 85cbfaa · 85cbfaa
1 parent f4e1747
commit 85cbfaa
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 28 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -28,10 +28,16 @@ generate:
     - pip3 install -r script/job_generator/requirements.txt
     # it is sufficient to verify once, as the same job matrix is generated, verified and then filtered each time
     # disable verify because we know that the generator is broken: https://github.com/thombashi/allpairspy/pull/10
-    #- python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --verify --wave compile_only_job -o compile_only.yml
-    - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave compile_only_job -o compile_only.yml
-    - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave runtime_job_cpu -o runtime_cpu.yml
-    - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave runtime_job_gpu -o runtime_gpu.yml
+    #- >
+    #  python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --verify --split-waves \
+    #  --wave-out-compile_only_job=compile_only.yml \
+    #  --wave-out-runtime_job_gpu=runtime_cpu.yml \
+    #  --wave-out-runtime_job_cpu=runtime_gpu.yml
+    - >
+      python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --split-waves \
+      --wave-out-compile_only_job=compile_only.yml \
+      --wave-out-runtime_job_gpu=runtime_cpu.yml \
+      --wave-out-runtime_job_cpu=runtime_gpu.yml
     - cat compile_only.yml
     - cat runtime_cpu.yml
     - cat runtime_gpu.yml

diff --git a/script/job_generator/generate_job_yaml.py b/script/job_generator/generate_job_yaml.py
@@ -600,7 +600,7 @@ def write_job_yaml(
         # If there is no CI job, create a dummy job.
         # This can happen if the filter filters out all jobs.
         number_of_jobs = 0
-        for wave_name in WAVE_GROUP_NAMES:
+        for wave_name in job_matrix.keys():
             number_of_jobs += len(job_matrix[wave_name])
 
         if number_of_jobs == 0:
@@ -618,7 +618,7 @@ def write_job_yaml(
 
         stages: Dict[str, List[str]] = {"stages": []}
 
-        for wave_name in WAVE_GROUP_NAMES:
+        for wave_name in job_matrix.keys():
             # setup all stages
             for stage_number in range(len(job_matrix[wave_name])):
                 stages["stages"].append(f"{wave_name}-stage{stage_number}")
@@ -640,7 +640,7 @@ def write_job_yaml(
             job_base_yaml = yaml.load(file, yaml.loader.SafeLoader)
         yaml.dump(job_base_yaml, output_file)
 
-        for wave_name in WAVE_GROUP_NAMES:
+        for wave_name in job_matrix.keys():
             # Writes each job separately to the file.
             # If all jobs would be collected first in dict, the order would be not guarantied.
             for stage_number, wave in enumerate(job_matrix[wave_name]):

diff --git a/script/job_generator/job_generator.py b/script/job_generator/job_generator.py
@@ -42,15 +42,21 @@ def get_args() -> argparse.Namespace:
     Returns:
         argparse.Namespace: The commandline arguments.
     """
-    parser = argparse.ArgumentParser(description="Calculate job matrix and create GitLab CI .yml.")
+    parser = argparse.ArgumentParser(
+        description="Calculate job matrix and create GitLab CI .yml."
+    )
 
-    parser.add_argument("version", type=float, help="Version number of the used CI container.")
+    parser.add_argument(
+        "version", type=float, help="Version number of the used CI container."
+    )
     parser.add_argument(
         "--print-combinations",
         action="store_true",
         help="Display combination matrix.",
     )
-    parser.add_argument("--verify", action="store_true", help="Verify generated combination matrix")
+    parser.add_argument(
+        "--verify", action="store_true", help="Verify generated combination matrix"
+    )
     parser.add_argument(
         "-a",
         "--all",
@@ -83,14 +89,20 @@ def get_args() -> argparse.Namespace:
     )
 
     parser.add_argument(
-        "--wave",
-        type=str,
-        # add `all` and remove `JOB_UNKNOWN` from the choices
-        choices=["all"] + list(set(WAVE_GROUP_NAMES) - set([JOB_UNKNOWN])),
-        default="all",
-        help="Generates only jobs for a specific wave group.",
+        "--split-waves",
+        action="store_true",
+        help="Write job waves in separate output files.",
     )
 
+    for wave_name in list(set(WAVE_GROUP_NAMES) - set([JOB_UNKNOWN])):
+        parser.add_argument(
+            f"--wave-out-{wave_name}",
+            type=str,
+            required="--split-waves" in sys.argv,
+            # add `all` and remove `JOB_UNKNOWN` from the choices
+            help=f"Output path of the job yaml for wave group {wave_name}",
+        )
+
     parser.add_argument(
         "--no-image-check",
         action="store_false",
@@ -169,7 +181,9 @@ def get_args() -> argparse.Namespace:
             if striped_line.strip().startswith(COMMIT_MESSAGE_FILTER_PREFIX):
                 filter_regix = striped_line[len(COMMIT_MESSAGE_FILTER_PREFIX) :].strip()
             if striped_line.startswith(COMMIT_MESSAGE_REORDER_PREFIX):
-                reorder_regix = striped_line[len(COMMIT_MESSAGE_REORDER_PREFIX) :].strip()
+                reorder_regix = striped_line[
+                    len(COMMIT_MESSAGE_REORDER_PREFIX) :
+                ].strip()
 
     if filter_regix:
         job_matrix_yaml = filter_job_list(job_matrix_yaml, filter_regix)
@@ -186,14 +200,19 @@ def get_args() -> argparse.Namespace:
             for job in wave:
                 print(job)
 
-    if args.wave != "all":
-        filter_wave_name = args.wave
-        wave_job_matrix = {filter_wave_name: wave_job_matrix[filter_wave_name]}
-        for wave_name in WAVE_GROUP_NAMES:
-            if wave_name not in wave_job_matrix:
-                wave_job_matrix[wave_name] = []
-
-    write_job_yaml(
-        job_matrix=wave_job_matrix,
-        path=args.output_path,
-    )
+    if not args.split_waves:
+        write_job_yaml(
+            job_matrix=wave_job_matrix,
+            path=args.output_path,
+        )
+    else:
+        for wave_name in list(set(WAVE_GROUP_NAMES) - set([JOB_UNKNOWN])):
+            # the script arguments for the wave output paths are automatically
+            # generated
+            # therefore we need also to construct the argument name to get the
+            # specific output path
+            arg_name = f"wave_out_{wave_name}"
+            write_job_yaml(
+                job_matrix={wave_name: wave_job_matrix[wave_name]},
+                path=args.__getattribute__(arg_name),
+            )