Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add unique-ids to shorten generated UI cruft.
Browse files Browse the repository at this point in the history
alliepiper committed Apr 23, 2024
1 parent a4f0c68 commit 3dfbf1b
Showing 4 changed files with 32 additions and 10 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci-dispatch-group.yml
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ permissions:

jobs:
standlone-jobs:
name: '[Standalone]'
name: "sa.${{ matrix.id }}"
permissions:
id-token: write
contents: read
@@ -31,7 +31,7 @@ jobs:
command: ${{ matrix.command }}

two-stage-jobs:
name: '[TwoStage]'
name: "ts.${{ matrix.id }}"
permissions:
id-token: write
contents: read
4 changes: 2 additions & 2 deletions .github/workflows/ci-dispatch-two-stage.yml
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ permissions:

jobs:
producers:
name: '[Producer]'
name: "p.${{ matrix.id }}"
# It is impossible to accumulate output variables across a matrix, and we cannot rely on the results of the dispatch-job workflow to determine success.
# See the note in ci-dispatch-job.yml for more information.
#
@@ -35,7 +35,7 @@ jobs:
command: ${{ matrix.command }}

consumers:
name: '[Consumer]'
name: "c.${{ matrix.id }}"
needs: producers
# dispatch-job's result is always false, check the outputs instead. See ci-dispatch-job.yml for more information.
if: ${{ !cancelled() && fromJson(needs.producers.outputs.success) }}
32 changes: 27 additions & 5 deletions ci/compute-matrix.py
Original file line number Diff line number Diff line change
@@ -42,25 +42,29 @@
- two_stage_json: A json object that represents bulk-synchronous producer/consumer jobs, used with ci-dispatch-two-stage.yml.
Example:
{
"id": "<unique id>", # Used as a compact unique name for the GHA dispatch workflows.
"producers": [ {<job_json>}, ... ],
"consumers": [ {<job_json>}, ... ]
}
- job_json: A json object that represents a single job in a workflow. Used with ci-dispatch-job.yml.
Example:
{
name: "...",
runner: "...",
image: "...",
command: "..." },
"id": "<unique id>", # Used as a compact unique name for the GHA dispatch workflows.
"name": "...",
"runner": "...",
"image": "...",
"command": "..." },
}
"""

import argparse
import base64
import copy
import json
import os
import re
import struct
import sys
import yaml

@@ -460,6 +464,23 @@ def natural_sort_key(key):
print(f"::error file=ci/matrix.yaml::{error_message}", file=sys.stderr)
raise Exception(error_message)

# Simple compact global unique ID generator:
# Generates a base64 hash of an incrementing 16-bit integer:
hash_generator = (base64.b64encode(struct.pack(">H", i)).decode('ascii') for i in range(65535))
# Strips off up-to 2 leading 'A' characters and a single trailing '=' characters, if they exist:
guid_generator = (re.sub(r'^A{0,2}', '', hash).removesuffix("=") for hash in hash_generator)

# Assign unique IDs where needed:
for group_name, group_json in workflow_dispatch_groups.items():
if 'standalone' in group_json:
for job_json in group_json['standalone']:
job_json['id'] = next(guid_generator)
if 'two_stage' in group_json:
for two_stage_json in group_json['two_stage']:
two_stage_json['id'] = next(guid_generator)
for job_json in two_stage_json['producers'] + two_stage_json['consumers']:
job_json['id'] = next(guid_generator)

return workflow_dispatch_groups


@@ -468,6 +489,7 @@ def pretty_print_workflow(final_workflow, outfile):

total_jobs = 0
runner_counts = {}

def print_job_array(key, group_json):
nonlocal total_jobs
nonlocal runner_counts
@@ -491,7 +513,7 @@ def print_job_array(key, group_json):
print(f"::group::Runner counts", file=outfile)
print(f"::notice:: Total jobs: {total_jobs}", file=outfile)
# Sort by descending counts:
runner_counts = {k : v for k, v in sorted(runner_counts.items(), key=lambda item: item[1], reverse=True)}
runner_counts = {k: v for k, v in sorted(runner_counts.items(), key=lambda item: item[1], reverse=True)}
for runner, count in runner_counts.items():
print(f"::notice:: {count}x {runner}", file=outfile)
print(f"::endgroup::", file=outfile)
2 changes: 1 addition & 1 deletion ci/matrix.yaml
Original file line number Diff line number Diff line change
@@ -185,7 +185,7 @@ default_os_lookup:

# All known GPUs
gpus:
- 'v100' # ?? runners
- 'v100' # 40 runners
- 't4' # 8 runners
- 'rtx2080' # 8 runners
- 'rtxa6000' # 12 runners

0 comments on commit 3dfbf1b

Please sign in to comment.