Configure workflow-run-job-linux to use sccache-dist build cluster [s…

…kip-vdc] [skip-docs] [skip-rapids]

.github/actions/workflow-run-job-linux/action.yml

@@ -78,6 +78,7 @@ runs:
         echo "[host]      github.workspace: ${{github.workspace}}"
         echo "[container] GITHUB_WORKSPACE: ${GITHUB_WORKSPACE:-}"
         echo "[container]              PWD: $(pwd)"
+        echo "[container]             user: $(whoami) ($(id -u):$(id -g))"
 
         # Necessary because we're doing docker-outside-of-docker:
         # Make a symlink in the container that matches the host's ${{github.workspace}}, so that way `$(pwd)`
@@ -90,6 +91,10 @@ runs:
         #! /usr/bin/env bash
         set -eo pipefail
         echo -e "\e[1;34mRunning as '$(whoami)' user in $(pwd):\e[0m"
+        echo -e "\e[1;34msccache --dist-status:\e[0m"
+        sccache --dist-status | jq
+        echo -e "\e[1;34msccache --show-stats:\e[0m"
+        sccache --show-stats
         echo -e "\e[1;34m${COMMAND}\e[0m"
         eval "${COMMAND}" || exit_code=$?
         if [ ! -z "$exit_code" ]; then
@@ -110,14 +115,25 @@ runs:
 
         chmod +x ci.sh
 
-        mkdir "$RUNNER_TEMP/.aws";
+        mkdir -p "$RUNNER_TEMP/bin"
+
+        # Download newer sccache binary
+        curl -fsSL \
+          "https://github.com/trxcllnt/sccache/releases/download/v0.8.3-rapids.11/sccache-v0.8.3-rapids.11-$(uname -m)-unknown-linux-musl.tar.gz" \
+        | tar -C "$RUNNER_TEMP/bin" -zf - --wildcards --strip-components=1 -x '*/sccache'
+
+        chmod 0755 "$RUNNER_TEMP/bin/sccache"
+
+        mkdir -p "$RUNNER_TEMP/.aws"
 
         cat <<EOF > "$RUNNER_TEMP/.aws/config"
         [default]
         bucket=rapids-sccache-devs
         region=us-east-2
         EOF
 
+        chmod 0664 "$RUNNER_TEMP/.aws/config"
+
         cat <<EOF > "$RUNNER_TEMP/.aws/credentials"
         [default]
         aws_access_key_id=$AWS_ACCESS_KEY_ID
@@ -126,7 +142,34 @@ runs:
         EOF
 
         chmod 0600 "$RUNNER_TEMP/.aws/credentials"
-        chmod 0664 "$RUNNER_TEMP/.aws/config"
+
+        mkdir -p "$RUNNER_TEMP/.config/sccache"
+
+        OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
+        ARCH="$(dpkg --print-architecture | awk -F'-' '{print $NF}')"
+
+        cat <<EOF > "$RUNNER_TEMP/.config/sccache/config"
+        [cache.disk]
+        size = 0
+
+        [cache.disk.preprocessor_cache_mode]
+        use_preprocessor_cache_mode = false
+
+        [dist]
+        scheduler_url = "https://${ARCH}.${OS}.sccache-dist.gha-runners.nvidia.com"
+
+        [dist.auth]
+        type = "token"
+        # This token needs the "read:enterprise" scope
+        token = "${{github.token}}"
+        EOF
+
+        chmod 0664 "$RUNNER_TEMP/.config/sccache/config"
+
+        # Verify we can talk to the sccache-dist scheduler
+        # SCCACHE_CONF="$RUNNER_TEMP/.config/sccache/config" "$RUNNER_TEMP/bin/sccache" --dist-status
+        # SCCACHE_CONF="$RUNNER_TEMP/.config/sccache/config" "$RUNNER_TEMP/bin/sccache" --dist-status | grep 'SchedulerStatus'
+        # SCCACHE_CONF="$RUNNER_TEMP/.config/sccache/config" "$RUNNER_TEMP/bin/sccache" --stop-server
 
         declare -a gpu_request=()
 
@@ -149,6 +192,7 @@ runs:
           --env "CI=$CI" \
           --env "AWS_ROLE_ARN=" \
           --env "COMMAND=$COMMAND" \
+          --env "SCCACHE_NO_CACHE=true" \
           --env "SCCACHE_IDLE_TIMEOUT=0" \
           --env "GITHUB_ENV=$GITHUB_ENV" \
           --env "GITHUB_SHA=$GITHUB_SHA" \
@@ -159,11 +203,15 @@ runs:
           --env "GITHUB_WORKSPACE=$GITHUB_WORKSPACE" \
           --env "GITHUB_REPOSITORY=$GITHUB_REPOSITORY" \
           --env "GITHUB_STEP_SUMMARY=$GITHUB_STEP_SUMMARY" \
-          --volume "${{github.workspace}}/ci.sh:/ci.sh" \
+          --volume "${{github.workspace}}/ci.sh:/ci.sh:ro" \
           --volume "$(host_path "$RUNNER_TEMP")/.aws:/root/.aws" \
+          --volume "$(host_path "$RUNNER_TEMP")/.config:/root/.config:ro" \
+          --volume "$(host_path "$RUNNER_TEMP")/bin/sccache:/usr/bin/sccache:ro" \
           --volume "$(dirname "$(dirname "${{github.workspace}}")"):/__w" \
           -- /ci.sh
 
+        SCCACHE_CONF="$RUNNER_TEMP/.config/sccache/config" "$RUNNER_TEMP/bin/sccache" --dist-status | grep 'SchedulerStatus'
+
     - name: Prepare job artifacts
       shell: bash --noprofile --norc -euo pipefail {0}
       run: |

CMakePresets.json

@@ -13,6 +13,7 @@
       "binaryDir": "${sourceDir}/build/$env{CCCL_BUILD_INFIX}/${presetName}",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_CUDA_FLAGS": "-t=3",
         "CMAKE_CUDA_ARCHITECTURES": "60;70;80",
         "CCCL_ENABLE_UNSTABLE": true,
         "CCCL_ENABLE_LIBCUDACXX": false,

ci/build_common.sh

@@ -128,7 +128,9 @@ print_environment_details() {
       CTEST_PARALLEL_LEVEL \
       CCCL_BUILD_INFIX \
       GLOBAL_CMAKE_OPTIONS \
-      TBB_ROOT
+      TBB_ROOT \
+      SCCACHE_BUCKET \
+      SCCACHE_NO_CACHE
 
   echo "Current commit is:"
   git log -1 || echo "Not a repository"
@@ -220,7 +222,7 @@ function build_preset() {
     source "./sccache_stats.sh" "start"
 
     pushd .. > /dev/null
-    run_command "$GROUP_NAME" cmake --build --preset=$PRESET -v
+    run_command "$GROUP_NAME" cmake --build --preset=$PRESET ${VERBOSE:+-v}
     status=$?
     popd > /dev/null
 

ci/matrix.yaml

@@ -8,6 +8,7 @@ workflows:
   #   - {jobs: ['test'], project: 'thrust', std: 17, ctk: 'curr', cxx: ['gcc12', 'llvm16']}
   #
   override:
+    - {jobs: ['test_nolid'], project: ['cub'], std: 'max', cxx: ['gcc']}
 
   pull_request:
     # Old CTK