diff --git a/.github/actions/workflow-run-job-linux/action.yml b/.github/actions/workflow-run-job-linux/action.yml index 53bcb6c38e5..3273b80997e 100644 --- a/.github/actions/workflow-run-job-linux/action.yml +++ b/.github/actions/workflow-run-job-linux/action.yml @@ -38,6 +38,9 @@ inputs: host: description: "The host compiler to use when selecting a devcontainer." required: true + dist-token: + description: "The token used to authenticate with the sccache-dist build cluster." + required: false runs: using: "composite" @@ -71,6 +74,7 @@ runs: # Dereferencing the command from an env var instead of a GHA input avoids issues with escaping # semicolons and other special characters (e.g. `-arch "60;70;80"`). COMMAND: "${{inputs.command}}" + DIST_TOKEN: "${{inputs.dist-token}}" AWS_ACCESS_KEY_ID: "${{env.AWS_ACCESS_KEY_ID}}" AWS_SESSION_TOKEN: "${{env.AWS_SESSION_TOKEN}}" AWS_SECRET_ACCESS_KEY: "${{env.AWS_SECRET_ACCESS_KEY}}" @@ -78,6 +82,7 @@ runs: echo "[host] github.workspace: ${{github.workspace}}" echo "[container] GITHUB_WORKSPACE: ${GITHUB_WORKSPACE:-}" echo "[container] PWD: $(pwd)" + echo "[container] user: $(whoami) ($(id -u):$(id -g))" # Necessary because we're doing docker-outside-of-docker: # Make a symlink in the container that matches the host's ${{github.workspace}}, so that way `$(pwd)` @@ -90,6 +95,10 @@ runs: #! /usr/bin/env bash set -eo pipefail echo -e "\e[1;34mRunning as '$(whoami)' user in $(pwd):\e[0m" + echo -e "\e[1;34msccache --dist-status:\e[0m" + sccache --dist-status | jq + echo -e "\e[1;34msccache --show-stats:\e[0m" + sccache --show-stats echo -e "\e[1;34m${COMMAND}\e[0m" eval "${COMMAND}" || exit_code=$? if [ ! -z "$exit_code" ]; then @@ -108,9 +117,15 @@ runs: fi EOF + host_path() { + sed "s@/__w@$(dirname "$(dirname "${{github.workspace}}")")@" <<< "$1" + } + + declare -a extra_launch_args=() + chmod +x ci.sh - mkdir "$RUNNER_TEMP/.aws"; + mkdir -p "$RUNNER_TEMP/.aws" cat < "$RUNNER_TEMP/.aws/config" [default] @@ -118,6 +133,8 @@ runs: region=us-east-2 EOF + chmod 0664 "$RUNNER_TEMP/.aws/config" + cat < "$RUNNER_TEMP/.aws/credentials" [default] aws_access_key_id=$AWS_ACCESS_KEY_ID @@ -126,26 +143,65 @@ runs: EOF chmod 0600 "$RUNNER_TEMP/.aws/credentials" - chmod 0664 "$RUNNER_TEMP/.aws/config" - declare -a gpu_request=() + if test "${DIST_TOKEN:+x}" = x; then + mkdir -p "$RUNNER_TEMP/.config/sccache" + + mkdir -p "$RUNNER_TEMP/bin" + + # Download newer sccache binary + curl -fsSL \ + "https://github.com/trxcllnt/sccache/releases/download/v0.8.3-rapids.12/sccache-v0.8.3-rapids.12-$(uname -m)-unknown-linux-musl.tar.gz" \ + | tar -C "$RUNNER_TEMP/bin" -zf - --wildcards --strip-components=1 -x '*/sccache' + + chmod 0755 "$RUNNER_TEMP/bin/sccache" + + OS="$(uname -s | tr '[:upper:]' '[:lower:]')" + ARCH="$(dpkg --print-architecture | awk -F'-' '{print $NF}')" + + cat < "$RUNNER_TEMP/.config/sccache/config" + [cache.disk] + size = 0 + [cache.disk.preprocessor_cache_mode] + use_preprocessor_cache_mode = false + [dist] + scheduler_url = "https://${ARCH}.${OS}.sccache-dist.gha-runners.nvidia.com" + [dist.auth] + type = "token" + # This token needs the "read:enterprise" scope + token = "${DIST_TOKEN}" + EOF + + chmod 0664 "$RUNNER_TEMP/.config/sccache/config" + + set -x + + # Verify we can talk to the sccache-dist scheduler + SCCACHE_CONF="$RUNNER_TEMP/.config/sccache/config" "$RUNNER_TEMP/bin/sccache" --dist-status + SCCACHE_CONF="$RUNNER_TEMP/.config/sccache/config" "$RUNNER_TEMP/bin/sccache" --dist-status | grep 'SchedulerStatus' + SCCACHE_CONF="$RUNNER_TEMP/.config/sccache/config" "$RUNNER_TEMP/bin/sccache" --stop-server + + extra_launch_args+=( + --env "SCCACHE_NO_CACHE=true" + --volume "$(host_path "$RUNNER_TEMP")/.config:/root/.config:ro" + --volume "$(host_path "$RUNNER_TEMP")/bin/sccache:/usr/bin/sccache:ro" + ) + + set +x + fi # Explicitly pass which GPU to use if on a GPU runner if [[ "${RUNNER}" = *"-gpu-"* ]]; then - gpu_request+=(--gpus "device=${NVIDIA_VISIBLE_DEVICES}") + extra_launch_args+=(--gpus "device=${NVIDIA_VISIBLE_DEVICES}") fi - host_path() { - sed "s@/__w@$(dirname "$(dirname "${{github.workspace}}")")@" <<< "$1" - } - # Launch this container using the host's docker daemon set -x + ${{github.event.repository.name}}/.devcontainer/launch.sh \ --docker \ --cuda ${{inputs.cuda}} \ --host ${{inputs.host}} \ - "${gpu_request[@]}" \ --env "CI=$CI" \ --env "AWS_ROLE_ARN=" \ --env "COMMAND=$COMMAND" \ @@ -159,9 +215,10 @@ runs: --env "GITHUB_WORKSPACE=$GITHUB_WORKSPACE" \ --env "GITHUB_REPOSITORY=$GITHUB_REPOSITORY" \ --env "GITHUB_STEP_SUMMARY=$GITHUB_STEP_SUMMARY" \ - --volume "${{github.workspace}}/ci.sh:/ci.sh" \ + --volume "${{github.workspace}}/ci.sh:/ci.sh:ro" \ --volume "$(host_path "$RUNNER_TEMP")/.aws:/root/.aws" \ --volume "$(dirname "$(dirname "${{github.workspace}}")"):/__w" \ + "${extra_launch_args[@]}" \ -- /ci.sh - name: Prepare job artifacts diff --git a/.github/workflows/ci-workflow-nightly.yml b/.github/workflows/ci-workflow-nightly.yml index 54cd0a7a527..49bfffd9cce 100644 --- a/.github/workflows/ci-workflow-nightly.yml +++ b/.github/workflows/ci-workflow-nightly.yml @@ -55,6 +55,7 @@ jobs: name: ${{ matrix.name }} if: ${{ toJSON(fromJSON(needs.build-workflow.outputs.workflow)['linux_two_stage']['keys']) != '[]' }} needs: build-workflow + secrets: inherit permissions: id-token: write contents: read @@ -85,6 +86,7 @@ jobs: name: ${{ matrix.name }} if: ${{ toJSON(fromJSON(needs.build-workflow.outputs.workflow)['linux_standalone']['keys']) != '[]' }} needs: build-workflow + secrets: inherit permissions: id-token: write contents: read diff --git a/.github/workflows/ci-workflow-pull-request.yml b/.github/workflows/ci-workflow-pull-request.yml index 06cd639164b..dd979ab0799 100644 --- a/.github/workflows/ci-workflow-pull-request.yml +++ b/.github/workflows/ci-workflow-pull-request.yml @@ -74,6 +74,7 @@ jobs: ${{ !contains(github.event.head_commit.message, '[skip-matrix]') && toJSON(fromJSON(needs.build-workflow.outputs.workflow)['linux_two_stage']['keys']) != '[]' }} needs: build-workflow + secrets: inherit permissions: id-token: write contents: read @@ -108,6 +109,7 @@ jobs: ${{ !contains(github.event.head_commit.message, '[skip-matrix]') && toJSON(fromJSON(needs.build-workflow.outputs.workflow)['linux_standalone']['keys']) != '[]' }} needs: build-workflow + secrets: inherit permissions: id-token: write contents: read diff --git a/.github/workflows/workflow-dispatch-standalone-group-linux.yml b/.github/workflows/workflow-dispatch-standalone-group-linux.yml index e0216b7c6d9..90d8e001d87 100644 --- a/.github/workflows/workflow-dispatch-standalone-group-linux.yml +++ b/.github/workflows/workflow-dispatch-standalone-group-linux.yml @@ -35,9 +35,10 @@ jobs: - name: Run job uses: ./.github/actions/workflow-run-job-linux with: - id: ${{ matrix.id }} - command: ${{ matrix.command }} - image: ${{ matrix.image }} - runner: ${{ matrix.runner }} - cuda: ${{ matrix.cuda }} - host: ${{ matrix.host }} + id: "${{ matrix.id }}" + command: "${{ matrix.command }}" + image: "${{ matrix.image }}" + runner: "${{ matrix.runner }}" + cuda: "${{ matrix.cuda }}" + host: "${{ matrix.host }}" + dist-token: "${{ secrets.SCCACHE_DIST_TOKEN }}" diff --git a/.github/workflows/workflow-dispatch-two-stage-group-linux.yml b/.github/workflows/workflow-dispatch-two-stage-group-linux.yml index c84f5c4af6f..dee60607baa 100644 --- a/.github/workflows/workflow-dispatch-two-stage-group-linux.yml +++ b/.github/workflows/workflow-dispatch-two-stage-group-linux.yml @@ -15,6 +15,7 @@ on: jobs: dispatch-pcs: name: ${{ matrix.id }} + secrets: inherit permissions: id-token: write contents: read diff --git a/.github/workflows/workflow-dispatch-two-stage-linux.yml b/.github/workflows/workflow-dispatch-two-stage-linux.yml index f1d9d518d3e..aa5137ceccc 100644 --- a/.github/workflows/workflow-dispatch-two-stage-linux.yml +++ b/.github/workflows/workflow-dispatch-two-stage-linux.yml @@ -37,12 +37,13 @@ jobs: - name: Run job uses: ./.github/actions/workflow-run-job-linux with: - id: ${{ fromJSON(inputs.producers)[0].id }} - command: ${{ fromJSON(inputs.producers)[0].command }} - image: ${{ fromJSON(inputs.producers)[0].image }} - runner: ${{ fromJSON(inputs.producers)[0].runner }} - cuda: ${{ fromJSON(inputs.producers)[0].cuda }} - host: ${{ fromJSON(inputs.producers)[0].host }} + id: "${{ fromJSON(inputs.producers)[0].id }}" + command: "${{ fromJSON(inputs.producers)[0].command }}" + image: "${{ fromJSON(inputs.producers)[0].image }}" + runner: "${{ fromJSON(inputs.producers)[0].runner }}" + cuda: "${{ fromJSON(inputs.producers)[0].cuda }}" + host: "${{ fromJSON(inputs.producers)[0].host }}" + dist-token: "${{ secrets.SCCACHE_DIST_TOKEN }}" consumers: name: "${{ matrix.name }}" @@ -67,9 +68,10 @@ jobs: - name: Run job uses: ./.github/actions/workflow-run-job-linux with: - id: ${{ matrix.id }} - command: ${{ matrix.command }} - image: ${{ matrix.image }} - runner: ${{ matrix.runner }} - cuda: ${{ matrix.cuda }} - host: ${{ matrix.host }} + id: "${{ matrix.id }}" + command: "${{ matrix.command }}" + image: "${{ matrix.image }}" + runner: "${{ matrix.runner }}" + cuda: "${{ matrix.cuda }}" + host: "${{ matrix.host }}" + dist-token: "${{ secrets.SCCACHE_DIST_TOKEN }}" diff --git a/CMakePresets.json b/CMakePresets.json index 447a684c58f..d8d8051cee5 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -13,6 +13,7 @@ "binaryDir": "${sourceDir}/build/$env{CCCL_BUILD_INFIX}/${presetName}", "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", + "CMAKE_CUDA_FLAGS": "-t=3", "CMAKE_CUDA_ARCHITECTURES": "60;70;80", "CCCL_ENABLE_UNSTABLE": true, "CCCL_ENABLE_LIBCUDACXX": false, diff --git a/ci/build_common.sh b/ci/build_common.sh index 0ce0de34ed6..99ade3c96e1 100755 --- a/ci/build_common.sh +++ b/ci/build_common.sh @@ -128,7 +128,9 @@ print_environment_details() { CTEST_PARALLEL_LEVEL \ CCCL_BUILD_INFIX \ GLOBAL_CMAKE_OPTIONS \ - TBB_ROOT + TBB_ROOT \ + SCCACHE_BUCKET \ + SCCACHE_NO_CACHE echo "Current commit is:" git log -1 || echo "Not a repository" @@ -220,7 +222,7 @@ function build_preset() { source "./sccache_stats.sh" "start" pushd .. > /dev/null - run_command "$GROUP_NAME" cmake --build --preset=$PRESET -v + run_command "$GROUP_NAME" cmake --build --preset=$PRESET ${VERBOSE:+-v} status=$? popd > /dev/null diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 2586fab09e6..2584bd14d05 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -8,6 +8,7 @@ workflows: # - {jobs: ['test'], project: 'thrust', std: 17, ctk: 'curr', cxx: ['gcc12', 'llvm16']} # override: + - {jobs: ['build'], project: ['cub', 'thrust'], std: 'max', ctk: 'curr', cxx: ['gcc']} pull_request: # Old CTK