diff --git a/.github/workflows/ci-dispatch-group.yml b/.github/workflows/ci-dispatch-group.yml index c6df29cd053..c2a08e458b9 100644 --- a/.github/workflows/ci-dispatch-group.yml +++ b/.github/workflows/ci-dispatch-group.yml @@ -15,7 +15,7 @@ permissions: jobs: standlone-jobs: - name: '[Standalone]' + name: "sa.${{ matrix.id }}" permissions: id-token: write contents: read @@ -31,7 +31,7 @@ jobs: command: ${{ matrix.command }} two-stage-jobs: - name: '[TwoStage]' + name: "ts.${{ matrix.id }}" permissions: id-token: write contents: read diff --git a/.github/workflows/ci-dispatch-two-stage.yml b/.github/workflows/ci-dispatch-two-stage.yml index 1c32c85dad6..52d1f3d8dcd 100644 --- a/.github/workflows/ci-dispatch-two-stage.yml +++ b/.github/workflows/ci-dispatch-two-stage.yml @@ -15,7 +15,7 @@ permissions: jobs: producers: - name: '[Producer]' + name: "p.${{ matrix.id }}" # It is impossible to accumulate output variables across a matrix, and we cannot rely on the results of the dispatch-job workflow to determine success. # See the note in ci-dispatch-job.yml for more information. # @@ -35,7 +35,7 @@ jobs: command: ${{ matrix.command }} consumers: - name: '[Consumer]' + name: "c.${{ matrix.id }}" needs: producers # dispatch-job's result is always false, check the outputs instead. See ci-dispatch-job.yml for more information. if: ${{ !cancelled() && fromJson(needs.producers.outputs.success) }} diff --git a/ci/compute-matrix.py b/ci/compute-matrix.py index c67ce087c18..9f1606d015f 100755 --- a/ci/compute-matrix.py +++ b/ci/compute-matrix.py @@ -42,6 +42,7 @@ - two_stage_json: A json object that represents bulk-synchronous producer/consumer jobs, used with ci-dispatch-two-stage.yml. Example: { + "id": "", # Used as a compact unique name for the GHA dispatch workflows. "producers": [ {}, ... ], "consumers": [ {}, ... ] } @@ -49,18 +50,21 @@ - job_json: A json object that represents a single job in a workflow. Used with ci-dispatch-job.yml. Example: { - name: "...", - runner: "...", - image: "...", - command: "..." }, + "id": "", # Used as a compact unique name for the GHA dispatch workflows. + "name": "...", + "runner": "...", + "image": "...", + "command": "..." }, } """ import argparse +import base64 import copy import json import os import re +import struct import sys import yaml @@ -460,6 +464,23 @@ def natural_sort_key(key): print(f"::error file=ci/matrix.yaml::{error_message}", file=sys.stderr) raise Exception(error_message) + # Simple compact global unique ID generator: + # Generates a base64 hash of an incrementing 16-bit integer: + hash_generator = (base64.b64encode(struct.pack(">H", i)).decode('ascii') for i in range(65535)) + # Strips off up-to 2 leading 'A' characters and a single trailing '=' characters, if they exist: + guid_generator = (re.sub(r'^A{0,2}', '', hash).removesuffix("=") for hash in hash_generator) + + # Assign unique IDs where needed: + for group_name, group_json in workflow_dispatch_groups.items(): + if 'standalone' in group_json: + for job_json in group_json['standalone']: + job_json['id'] = next(guid_generator) + if 'two_stage' in group_json: + for two_stage_json in group_json['two_stage']: + two_stage_json['id'] = next(guid_generator) + for job_json in two_stage_json['producers'] + two_stage_json['consumers']: + job_json['id'] = next(guid_generator) + return workflow_dispatch_groups @@ -468,6 +489,7 @@ def pretty_print_workflow(final_workflow, outfile): total_jobs = 0 runner_counts = {} + def print_job_array(key, group_json): nonlocal total_jobs nonlocal runner_counts @@ -491,7 +513,7 @@ def print_job_array(key, group_json): print(f"::group::Runner counts", file=outfile) print(f"::notice:: Total jobs: {total_jobs}", file=outfile) # Sort by descending counts: - runner_counts = {k : v for k, v in sorted(runner_counts.items(), key=lambda item: item[1], reverse=True)} + runner_counts = {k: v for k, v in sorted(runner_counts.items(), key=lambda item: item[1], reverse=True)} for runner, count in runner_counts.items(): print(f"::notice:: {count}x {runner}", file=outfile) print(f"::endgroup::", file=outfile) diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 354b2315576..f46b8dfd50d 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -185,7 +185,7 @@ default_os_lookup: # All known GPUs gpus: - - 'v100' # ?? runners + - 'v100' # 40 runners - 't4' # 8 runners - 'rtx2080' # 8 runners - 'rtxa6000' # 12 runners