From 87573b06d607f76833beef92897a2028ae71a310 Mon Sep 17 00:00:00 2001 From: Simeon Ehrig Date: Wed, 9 Oct 2024 15:41:24 +0200 Subject: [PATCH] call job_generator.py only one time to generate all job yaml files instead n times --- .gitlab-ci.yml | 6 +-- script/job_generator/generate_job_yaml.py | 6 +-- script/job_generator/job_generator.py | 61 +++++++++++++++-------- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a649e23ea1d2..1ddca1fb401f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,10 +28,8 @@ generate: - pip3 install -r script/job_generator/requirements.txt # it is sufficient to verify once, as the same job matrix is generated, verified and then filtered each time # disable verify because we know that the generator is broken: https://github.com/thombashi/allpairspy/pull/10 - #- python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --verify --wave compile_only_job -o compile_only.yml - - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave compile_only_job -o compile_only.yml - - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave runtime_job_cpu -o runtime_cpu.yml - - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave runtime_job_gpu -o runtime_gpu.yml + # - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --verify --split-waves --wave-out-compile_only_job=compile_only.yml --wave-out-runtime_job_gpu=runtime_cpu.yml --wave-out-runtime_job_cpu=runtime_gpu.yml + - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --split-waves --wave-out-compile_only_job=compile_only.yml --wave-out-runtime_job_gpu=runtime_cpu.yml --wave-out-runtime_job_cpu=runtime_gpu.yml - cat compile_only.yml - cat runtime_cpu.yml - cat runtime_gpu.yml diff --git a/script/job_generator/generate_job_yaml.py b/script/job_generator/generate_job_yaml.py index e66191e8a879..53eaf047a7f6 100644 --- a/script/job_generator/generate_job_yaml.py +++ b/script/job_generator/generate_job_yaml.py @@ -600,7 +600,7 @@ def write_job_yaml( # If there is no CI job, create a dummy job. # This can happen if the filter filters out all jobs. number_of_jobs = 0 - for wave_name in WAVE_GROUP_NAMES: + for wave_name in job_matrix.keys(): number_of_jobs += len(job_matrix[wave_name]) if number_of_jobs == 0: @@ -618,7 +618,7 @@ def write_job_yaml( stages: Dict[str, List[str]] = {"stages": []} - for wave_name in WAVE_GROUP_NAMES: + for wave_name in job_matrix.keys(): # setup all stages for stage_number in range(len(job_matrix[wave_name])): stages["stages"].append(f"{wave_name}-stage{stage_number}") @@ -640,7 +640,7 @@ def write_job_yaml( job_base_yaml = yaml.load(file, yaml.loader.SafeLoader) yaml.dump(job_base_yaml, output_file) - for wave_name in WAVE_GROUP_NAMES: + for wave_name in job_matrix.keys(): # Writes each job separately to the file. # If all jobs would be collected first in dict, the order would be not guarantied. for stage_number, wave in enumerate(job_matrix[wave_name]): diff --git a/script/job_generator/job_generator.py b/script/job_generator/job_generator.py index a3b186372a38..d079d59c08f3 100644 --- a/script/job_generator/job_generator.py +++ b/script/job_generator/job_generator.py @@ -42,15 +42,21 @@ def get_args() -> argparse.Namespace: Returns: argparse.Namespace: The commandline arguments. """ - parser = argparse.ArgumentParser(description="Calculate job matrix and create GitLab CI .yml.") + parser = argparse.ArgumentParser( + description="Calculate job matrix and create GitLab CI .yml." + ) - parser.add_argument("version", type=float, help="Version number of the used CI container.") + parser.add_argument( + "version", type=float, help="Version number of the used CI container." + ) parser.add_argument( "--print-combinations", action="store_true", help="Display combination matrix.", ) - parser.add_argument("--verify", action="store_true", help="Verify generated combination matrix") + parser.add_argument( + "--verify", action="store_true", help="Verify generated combination matrix" + ) parser.add_argument( "-a", "--all", @@ -83,14 +89,20 @@ def get_args() -> argparse.Namespace: ) parser.add_argument( - "--wave", - type=str, - # add `all` and remove `JOB_UNKNOWN` from the choices - choices=["all"] + list(set(WAVE_GROUP_NAMES) - set([JOB_UNKNOWN])), - default="all", - help="Generates only jobs for a specific wave group.", + "--split-waves", + action="store_true", + help="Write job waves in separate output files.", ) + for wave_name in list(set(WAVE_GROUP_NAMES) - set([JOB_UNKNOWN])): + parser.add_argument( + f"--wave-out-{wave_name}", + type=str, + required="--split-waves" in sys.argv, + # add `all` and remove `JOB_UNKNOWN` from the choices + help=f"Output path of the job yaml for wave group {wave_name}", + ) + parser.add_argument( "--no-image-check", action="store_false", @@ -169,7 +181,9 @@ def get_args() -> argparse.Namespace: if striped_line.strip().startswith(COMMIT_MESSAGE_FILTER_PREFIX): filter_regix = striped_line[len(COMMIT_MESSAGE_FILTER_PREFIX) :].strip() if striped_line.startswith(COMMIT_MESSAGE_REORDER_PREFIX): - reorder_regix = striped_line[len(COMMIT_MESSAGE_REORDER_PREFIX) :].strip() + reorder_regix = striped_line[ + len(COMMIT_MESSAGE_REORDER_PREFIX) : + ].strip() if filter_regix: job_matrix_yaml = filter_job_list(job_matrix_yaml, filter_regix) @@ -186,14 +200,19 @@ def get_args() -> argparse.Namespace: for job in wave: print(job) - if args.wave != "all": - filter_wave_name = args.wave - wave_job_matrix = {filter_wave_name: wave_job_matrix[filter_wave_name]} - for wave_name in WAVE_GROUP_NAMES: - if wave_name not in wave_job_matrix: - wave_job_matrix[wave_name] = [] - - write_job_yaml( - job_matrix=wave_job_matrix, - path=args.output_path, - ) + if not args.split_waves: + write_job_yaml( + job_matrix=wave_job_matrix, + path=args.output_path, + ) + else: + for wave_name in list(set(WAVE_GROUP_NAMES) - set([JOB_UNKNOWN])): + # the script arguments for the wave output paths are automatically + # generated + # therefore we need also to construct the argument name to get the + # specific output path + arg_name = f"wave_out_{wave_name}" + write_job_yaml( + job_matrix={wave_name: wave_job_matrix[wave_name]}, + path=args.__getattribute__(arg_name), + )