Configure workflow-run-job-linux to use sccache-dist build cluster [s…

…kip-vdc] [skip-docs] [skip-rapids]
NVIDIA · Oct 31, 2024 · d2578ca · d2578ca
1 parent 1ec46e5
commit d2578ca
Show file tree

Hide file tree

Showing 10 changed files with 89 additions and 32 deletions.
diff --git a/.github/actions/workflow-build/build-workflow.py b/.github/actions/workflow-build/build-workflow.py
@@ -389,7 +389,7 @@ def generate_dispatch_job_runner(matrix_job, job_type):
 
     job_info = get_job_type_info(job_type)
     if not job_info['gpu']:
-        return f"{runner_os}-{cpu}-cpu16"
+        return f"{runner_os}-{cpu}-cpu8"
 
     gpu = get_gpu(matrix_job['gpu'])
     suffix = "-testing" if gpu['testing'] else ""

diff --git a/.github/actions/workflow-run-job-linux/action.yml b/.github/actions/workflow-run-job-linux/action.yml
@@ -38,6 +38,9 @@ inputs:
   host:
     description: "The host compiler to use when selecting a devcontainer."
     required: true
+  dist-token:
+    description: "The token used to authenticate with the sccache-dist build cluster."
+    required: false
 
 runs:
   using: "composite"
@@ -70,14 +73,16 @@ runs:
         RUNNER: "${{inputs.runner}}"
         # Dereferencing the command from an env var instead of a GHA input avoids issues with escaping
         # semicolons and other special characters (e.g. `-arch "60;70;80"`).
-        COMMAND: "${{inputs.command}}"
+        COMMAND: "${{inputs.command}} -v"
+        DIST_TOKEN: "${{inputs.dist-token}}"
         AWS_ACCESS_KEY_ID: "${{env.AWS_ACCESS_KEY_ID}}"
         AWS_SESSION_TOKEN: "${{env.AWS_SESSION_TOKEN}}"
         AWS_SECRET_ACCESS_KEY: "${{env.AWS_SECRET_ACCESS_KEY}}"
       run: |
         echo "[host]      github.workspace: ${{github.workspace}}"
         echo "[container] GITHUB_WORKSPACE: ${GITHUB_WORKSPACE:-}"
         echo "[container]              PWD: $(pwd)"
+        echo "[container]             user: $(whoami) ($(id -u):$(id -g))"
 
         # Necessary because we're doing docker-outside-of-docker:
         # Make a symlink in the container that matches the host's ${{github.workspace}}, so that way `$(pwd)`
@@ -90,6 +95,7 @@ runs:
         #! /usr/bin/env bash
         set -eo pipefail
         echo -e "\e[1;34mRunning as '$(whoami)' user in $(pwd):\e[0m"
+        echo -e "\e[1;34mBuild cluster: $(sccache --dist-status)\e[0m"
         echo -e "\e[1;34m${COMMAND}\e[0m"
         eval "${COMMAND}" || exit_code=$?
         if [ ! -z "$exit_code" ]; then
@@ -108,16 +114,24 @@ runs:
         fi
         EOF
 
+        host_path() {
+          sed "s@/__w@$(dirname "$(dirname "${{github.workspace}}")")@" <<< "$1"
+        }
+
+        declare -a extra_launch_args=()
+
         chmod +x ci.sh
 
-        mkdir "$RUNNER_TEMP/.aws";
+        mkdir -p "$RUNNER_TEMP/.aws"
 
         cat <<EOF > "$RUNNER_TEMP/.aws/config"
         [default]
         bucket=rapids-sccache-devs
         region=us-east-2
         EOF
 
+        chmod 0664 "$RUNNER_TEMP/.aws/config"
+
         cat <<EOF > "$RUNNER_TEMP/.aws/credentials"
         [default]
         aws_access_key_id=$AWS_ACCESS_KEY_ID
@@ -126,26 +140,56 @@ runs:
         EOF
 
         chmod 0600 "$RUNNER_TEMP/.aws/credentials"
-        chmod 0664 "$RUNNER_TEMP/.aws/config"
 
-        declare -a gpu_request=()
+        if test "${DIST_TOKEN:+x}" = x; then
+          mkdir -p "$RUNNER_TEMP/.config/sccache"
+
+          mkdir -p "$RUNNER_TEMP/bin"
+
+          # Download newer sccache binary
+          curl -fsSL \
+            "https://github.com/trxcllnt/sccache/releases/download/v0.8.3-rapids.12/sccache-v0.8.3-rapids.12-$(uname -m)-unknown-linux-musl.tar.gz" \
+          | tar -C "$RUNNER_TEMP/bin" -zf - --wildcards --strip-components=1 -x '*/sccache'
+
+          chmod 0755 "$RUNNER_TEMP/bin/sccache"
+
+          OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
+          ARCH="$(dpkg --print-architecture | awk -F'-' '{print $NF}')"
+
+          cat <<EOF > "$RUNNER_TEMP/.config/sccache/config"
+        [cache.disk]
+        size = 0
+        [cache.disk.preprocessor_cache_mode]
+        use_preprocessor_cache_mode = false
+        [dist]
+        scheduler_url = "https://${ARCH}.${OS}.sccache-dist.gha-runners.nvidia.com"
+        [dist.auth]
+        type = "token"
+        # Token must have the "read:enterprise" scope
+        token = "${DIST_TOKEN}"
+        EOF
+
+          chmod 0664 "$RUNNER_TEMP/.config/sccache/config"
+
+          extra_launch_args+=(
+            --env "SCCACHE_NO_CACHE=true"
+            --volume "$(host_path "$RUNNER_TEMP")/.config:/root/.config:ro"
+            --volume "$(host_path "$RUNNER_TEMP")/bin/sccache:/usr/bin/sccache:ro"
+          )
+        fi
 
         # Explicitly pass which GPU to use if on a GPU runner
         if [[ "${RUNNER}" = *"-gpu-"* ]]; then
-          gpu_request+=(--gpus "device=${NVIDIA_VISIBLE_DEVICES}")
+          extra_launch_args+=(--gpus "device=${NVIDIA_VISIBLE_DEVICES}")
         fi
 
-        host_path() {
-          sed "s@/__w@$(dirname "$(dirname "${{github.workspace}}")")@" <<< "$1"
-        }
-
         # Launch this container using the host's docker daemon
         set -x
+
         ${{github.event.repository.name}}/.devcontainer/launch.sh \
           --docker \
           --cuda ${{inputs.cuda}} \
           --host ${{inputs.host}} \
-          "${gpu_request[@]}" \
           --env "CI=$CI" \
           --env "AWS_ROLE_ARN=" \
           --env "COMMAND=$COMMAND" \
@@ -159,9 +203,10 @@ runs:
           --env "GITHUB_WORKSPACE=$GITHUB_WORKSPACE" \
           --env "GITHUB_REPOSITORY=$GITHUB_REPOSITORY" \
           --env "GITHUB_STEP_SUMMARY=$GITHUB_STEP_SUMMARY" \
-          --volume "${{github.workspace}}/ci.sh:/ci.sh" \
+          --volume "${{github.workspace}}/ci.sh:/ci.sh:ro" \
           --volume "$(host_path "$RUNNER_TEMP")/.aws:/root/.aws" \
           --volume "$(dirname "$(dirname "${{github.workspace}}")"):/__w" \
+          "${extra_launch_args[@]}" \
           -- /ci.sh
 
     - name: Prepare job artifacts

diff --git a/.github/workflows/ci-workflow-nightly.yml b/.github/workflows/ci-workflow-nightly.yml
@@ -55,6 +55,7 @@ jobs:
     name: ${{ matrix.name }}
     if: ${{ toJSON(fromJSON(needs.build-workflow.outputs.workflow)['linux_two_stage']['keys']) != '[]' }}
     needs: build-workflow
+    secrets: inherit
     permissions:
       id-token: write
       contents: read
@@ -85,6 +86,7 @@ jobs:
     name: ${{ matrix.name }}
     if: ${{ toJSON(fromJSON(needs.build-workflow.outputs.workflow)['linux_standalone']['keys']) != '[]' }}
     needs: build-workflow
+    secrets: inherit
     permissions:
       id-token: write
       contents: read

diff --git a/.github/workflows/ci-workflow-pull-request.yml b/.github/workflows/ci-workflow-pull-request.yml
@@ -74,6 +74,7 @@ jobs:
       ${{ !contains(github.event.head_commit.message, '[skip-matrix]') &&
           toJSON(fromJSON(needs.build-workflow.outputs.workflow)['linux_two_stage']['keys']) != '[]' }}
     needs: build-workflow
+    secrets: inherit
     permissions:
       id-token: write
       contents: read
@@ -108,6 +109,7 @@ jobs:
       ${{ !contains(github.event.head_commit.message, '[skip-matrix]') &&
           toJSON(fromJSON(needs.build-workflow.outputs.workflow)['linux_standalone']['keys']) != '[]' }}
     needs: build-workflow
+    secrets: inherit
     permissions:
       id-token: write
       contents: read

diff --git a/.github/workflows/workflow-dispatch-standalone-group-linux.yml b/.github/workflows/workflow-dispatch-standalone-group-linux.yml
@@ -35,9 +35,10 @@ jobs:
       - name: Run job
         uses: ./.github/actions/workflow-run-job-linux
         with:
-          id:      ${{ matrix.id }}
-          command: ${{ matrix.command }}
-          image:   ${{ matrix.image }}
-          runner:  ${{ matrix.runner }}
-          cuda:    ${{ matrix.cuda }}
-          host:    ${{ matrix.host }}
+          id:         "${{ matrix.id }}"
+          command:    "${{ matrix.command }}"
+          image:      "${{ matrix.image }}"
+          runner:     "${{ matrix.runner }}"
+          cuda:       "${{ matrix.cuda }}"
+          host:       "${{ matrix.host }}"
+          dist-token: "${{ secrets.SCCACHE_DIST_TOKEN }}"
diff --git a/.github/workflows/workflow-dispatch-two-stage-group-linux.yml b/.github/workflows/workflow-dispatch-two-stage-group-linux.yml
@@ -15,6 +15,7 @@ on:
 jobs:
   dispatch-pcs:
     name: ${{ matrix.id }}
+    secrets: inherit
     permissions:
       id-token: write
       contents: read

diff --git a/.github/workflows/workflow-dispatch-two-stage-linux.yml b/.github/workflows/workflow-dispatch-two-stage-linux.yml
@@ -37,12 +37,13 @@ jobs:
       - name: Run job
         uses: ./.github/actions/workflow-run-job-linux
         with:
-          id:      ${{ fromJSON(inputs.producers)[0].id }}
-          command: ${{ fromJSON(inputs.producers)[0].command }}
-          image:   ${{ fromJSON(inputs.producers)[0].image }}
-          runner:  ${{ fromJSON(inputs.producers)[0].runner }}
-          cuda:    ${{ fromJSON(inputs.producers)[0].cuda }}
-          host:    ${{ fromJSON(inputs.producers)[0].host }}
+          id:         "${{ fromJSON(inputs.producers)[0].id }}"
+          command:    "${{ fromJSON(inputs.producers)[0].command }}"
+          image:      "${{ fromJSON(inputs.producers)[0].image }}"
+          runner:     "${{ fromJSON(inputs.producers)[0].runner }}"
+          cuda:       "${{ fromJSON(inputs.producers)[0].cuda }}"
+          host:       "${{ fromJSON(inputs.producers)[0].host }}"
+          dist-token: "${{ secrets.SCCACHE_DIST_TOKEN }}"
 
   consumers:
     name: "${{ matrix.name }}"
@@ -67,9 +68,10 @@ jobs:
       - name: Run job
         uses: ./.github/actions/workflow-run-job-linux
         with:
-          id:      ${{ matrix.id }}
-          command: ${{ matrix.command }}
-          image:   ${{ matrix.image }}
-          runner:  ${{ matrix.runner }}
-          cuda:    ${{ matrix.cuda }}
-          host:    ${{ matrix.host }}
+          id:         "${{ matrix.id }}"
+          command:    "${{ matrix.command }}"
+          image:      "${{ matrix.image }}"
+          runner:     "${{ matrix.runner }}"
+          cuda:       "${{ matrix.cuda }}"
+          host:       "${{ matrix.host }}"
+          dist-token: "${{ secrets.SCCACHE_DIST_TOKEN }}"
diff --git a/CMakePresets.json b/CMakePresets.json
@@ -13,6 +13,7 @@
       "binaryDir": "${sourceDir}/build/$env{CCCL_BUILD_INFIX}/${presetName}",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_CUDA_FLAGS": "-t=3",
         "CMAKE_CUDA_ARCHITECTURES": "60;70;80",
         "CCCL_ENABLE_UNSTABLE": true,
         "CCCL_ENABLE_LIBCUDACXX": false,

diff --git a/ci/build_common.sh b/ci/build_common.sh
@@ -128,7 +128,9 @@ print_environment_details() {
       CTEST_PARALLEL_LEVEL \
       CCCL_BUILD_INFIX \
       GLOBAL_CMAKE_OPTIONS \
-      TBB_ROOT
+      TBB_ROOT \
+      SCCACHE_BUCKET \
+      SCCACHE_NO_CACHE
 
   echo "Current commit is:"
   git log -1 || echo "Not a repository"
@@ -220,7 +222,7 @@ function build_preset() {
     source "./sccache_stats.sh" "start"
 
     pushd .. > /dev/null
-    run_command "$GROUP_NAME" cmake --build --preset=$PRESET -v
+    run_command "$GROUP_NAME" cmake --build --preset=$PRESET ${VERBOSE:+-v}
     status=$?
     popd > /dev/null
 

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
@@ -8,6 +8,7 @@ workflows:
   #   - {jobs: ['test'], project: 'thrust', std: 17, ctk: 'curr', cxx: ['gcc12', 'llvm16']}
   #
   override:
+    - {jobs: ['build'], project: ['cub', 'thrust'], std: 'max', ctk: 'curr', cxx: ['gcc']}
 
   pull_request:
     # Old CTK