diff --git a/.github/workflows/dependencies-linux.sh b/.github/workflows/dependencies-linux.sh index 38955304..9a39d38b 100755 --- a/.github/workflows/dependencies-linux.sh +++ b/.github/workflows/dependencies-linux.sh @@ -3,7 +3,7 @@ set -eu -o pipefail sudo apt-get update -sudo apt-get install libopenmpi-dev libhdf5-dev +sudo apt-get install libopenmpi-dev libhdf5-dev uuid-dev # libfabric wget https://github.com/ofiwg/libfabric/archive/refs/tags/v1.12.1.tar.gz diff --git a/.github/workflows/nersc.yml b/.github/workflows/nersc.yml index e941f177..db95b3bc 100644 --- a/.github/workflows/nersc.yml +++ b/.github/workflows/nersc.yml @@ -1,7 +1,7 @@ name: NERSC on: - pull_request_target + pull_request env: PR_NUMBER: ${{ github.event.number }} @@ -33,6 +33,6 @@ jobs: run: | PR_SHA=$(git rev-parse --short "$GITHUB_SHA") git fetch origin pull/${PR_NUMBER}/head:PR-${PR_SHA} - git remote add gitlab https://${{ secrets.NERSC_GITLAB_TOKEN_NAME }}:${{ secrets.NERSC_GITLAB_TOKEN }}@${{ secrets.NERSC_GITLAB_URL }} + git remote add gitlab https://${{ secrets.GITLAB_TOKEN_NAME }}:${{ secrets.GITLAB_TOKEN }}@${{ secrets.GITLAB_URL }} git checkout PR-${PR_SHA} git push -f gitlab -u PR-${PR_SHA} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a7c07bcf..cdc68744 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,12 +1,46 @@ variables: PDC_BUILD_PATH: "${CI_PROJECT_DIR}/build" PDC_INSTALL_PATH: "${CI_PROJECT_DIR}/install" - + GIT_CLONE_PATH: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}" + stages: - build - test - metrics +.report-status: + variables: + STATUS_PROJECT: "hpc-io/pdc" + STATUS_NAME: "NERSC / Perlmutter" + script: + # For complete details on the GitHub API please see: + # https://developer.github.com/v3/repos/statuses + - | + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA}" \ + -d "{\"state\":\"${CI_JOB_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"context\":\"${STATUS_NAME}\"}" + +pending: + stage: .pre + extends: + - .report-status + +success: + stage: .post + extends: + - .report-status + +failure: + stage: .post + extends: + - .report-status + rules: + - when: on_failure + perlmutter-no-cache-build: stage: build rules: @@ -14,7 +48,7 @@ perlmutter-no-cache-build: when: manual id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com allow_failure: false tags: - perlmutter @@ -42,7 +76,7 @@ perlmutter-cache-build: when: manual id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ allow_failure: false tags: - perlmutter @@ -72,7 +106,7 @@ perlmutter-no-cache-parallel-pdc: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -83,8 +117,8 @@ perlmutter-no-cache-parallel-pdc: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-pdc" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-pdc" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-pdc" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-pdc" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/no-cache @@ -95,7 +129,7 @@ perlmutter-no-cache-parallel-obj: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -107,8 +141,8 @@ perlmutter-no-cache-parallel-obj: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-obj" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-obj" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-obj" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-obj" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/no-cache @@ -119,7 +153,7 @@ perlmutter-no-cache-parallel-cont: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -131,8 +165,8 @@ perlmutter-no-cache-parallel-cont: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-cont" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-cont" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-cont" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-cont" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/no-cache @@ -143,7 +177,7 @@ perlmutter-no-cache-parallel-prop: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -155,8 +189,8 @@ perlmutter-no-cache-parallel-prop: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-prop" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-prop" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-prop" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-prop" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/no-cache @@ -167,7 +201,7 @@ perlmutter-no-cache-parallel-region: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -179,8 +213,8 @@ perlmutter-no-cache-parallel-region: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-region" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-region" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-region" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-region" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/no-cache @@ -191,7 +225,7 @@ perlmutter-no-cache-parallel-region-all: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -203,8 +237,8 @@ perlmutter-no-cache-parallel-region-all: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/no-cache/pdc-tmp-paralell-region-all" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/no-cache/pdc-data-paralell-region-all" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-tmp-paralell-region-all" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/no-cache/pdc-data-paralell-region-all" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/no-cache @@ -220,7 +254,7 @@ perlmutter-cache-parallel-pdc: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -231,8 +265,8 @@ perlmutter-cache-parallel-pdc: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-pdc" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-pdc" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-pdc" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-pdc" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/cache @@ -243,7 +277,7 @@ perlmutter-cache-parallel-obj: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -255,8 +289,8 @@ perlmutter-cache-parallel-obj: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-obj" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-obj" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-obj" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-obj" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/cache @@ -267,7 +301,7 @@ perlmutter-cache-parallel-cont: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -279,8 +313,8 @@ perlmutter-cache-parallel-cont: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-cont" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-cont" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-cont" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-cont" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/cache @@ -291,7 +325,7 @@ perlmutter-cache-parallel-prop: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -303,8 +337,8 @@ perlmutter-cache-parallel-prop: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-prop" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-prop" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-prop" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-prop" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/cache @@ -315,7 +349,7 @@ perlmutter-cache-parallel-region: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -327,8 +361,8 @@ perlmutter-cache-parallel-region: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-region" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-region" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-region" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-region" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/cache @@ -339,7 +373,7 @@ perlmutter-cache-parallel-region-all: stage: test id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == null' needs: @@ -351,8 +385,8 @@ perlmutter-cache-parallel-region-all: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/cache/pdc-tmp-paralell-region-all" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/cache/pdc-data-paralell-region-all" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-tmp-paralell-region-all" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/cache/pdc-data-paralell-region-all" script: - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - cd ${PDC_BUILD_PATH}/perlmutter/cache @@ -363,7 +397,7 @@ perlmutter-metrics-build: stage: build id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == "true"' allow_failure: false @@ -390,7 +424,7 @@ perlmutter-metrics: stage: metrics id_tokens: SITE_ID_TOKEN: - aud: https://software.nersc.gov/ + aud: https://gitlab.com/ rules: - if: '$METRICS == "true"' needs: @@ -401,8 +435,8 @@ perlmutter-metrics: SCHEDULER_PARAMETERS: "-A ${PDC_PROJECT} --qos=${PDC_QUEUE} --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-metrics" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-metrics" + PDC_TMPDIR: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/pdc-tmp-metrics" + PDC_DATA_LOC: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}/pdc-data-metrics" PDC_CLIENT_LOOKUP: "NONE" PDC_SERVER: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/pdc_server.exe" PDC_SERVER_CLOSE: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/close_server"