From 390a88b76089f4bf4b39ed55fadff63eb8232a94 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 13 Sep 2024 17:31:14 -0600 Subject: [PATCH 1/9] make CI faster for drafts --- .github/workflows/deploy-tests.yml | 33 +++++++++++++++--- .github/workflows/dev-testing.yml | 10 ++++-- .github/workflows/doctests.yml | 8 ++++- .github/workflows/live-service-testing.yml | 10 ++++-- .github/workflows/testing.yml | 39 +++++++++++++++------- 5 files changed, 77 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 9cbf6ba94..f271b8c9a 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -2,7 +2,9 @@ name: Deploy tests on: pull_request: - types: [synchronize, opened, reopened, ready_for_review] # defaults + ready_for_review + types: [synchronize, opened, reopened, ready_for_review] + # Synchronize, open and reopened are the defaults for pull request + # We add ready_for_review to trigger the check for changelog and full tests when ready for review is clicked merge_group: workflow_dispatch: @@ -21,9 +23,9 @@ jobs: name: Auto-detecting CHANGELOG.md updates runs-on: ubuntu-latest steps: - - if: ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'true' }} + - if: ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'true' }} run: echo "CHANGELOG.md has been updated." - - if: ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'false' }} + - if: ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'false' }} run: | echo "CHANGELOG.md has not been updated." 0 @@ -37,7 +39,12 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - + with: + python-versions: > # Ternary operator, condition && value_if_true || value_if_false + ${{github.event.pull_request.draft == true + && '["3.9"]' + || '["3.9", "3.10", "3.11", "3.12"]' + }} run-live-service-tests: needs: assess-file-changes @@ -45,6 +52,12 @@ jobs: uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + with: + python-versions: > # Ternary operator, condition && value_if_true || value_if_false + ${{github.event.pull_request.draft == true + && '["3.9"]' + || '["3.9", "3.10", "3.11", "3.12"]' + }} run-dev-tests: needs: assess-file-changes @@ -55,11 +68,23 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} + with: + python-versions: > # Ternary operator, condition && value_if_true || value_if_false + ${{github.event.pull_request.draft == true + && '["3.9"]' + || '["3.9", "3.10", "3.11", "3.12"]' + }} run-doctests-only: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml + with: + python-versions: > # Ternary operator, condition && value_if_true || value_if_false + ${{github.event.pull_request.draft == true + && '["3.9"]' + || '["3.9", "3.10", "3.11", "3.12"]' + }} check-final-status: name: All tests passing diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index 65be5bffd..d38e44960 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -1,8 +1,12 @@ name: Dev Branch Testing on: - schedule: - - cron: "0 16 * * *" # Daily at noon EST workflow_call: + inputs: + python-versions: + description: 'List of Python versions to use in matrix, as JSON string' + required: true + type: string + default: '["3.9", "3.10", "3.11", "3.12"]' secrets: DANDI_API_KEY: required: true @@ -23,7 +27,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ${{ fromJson(inputs.python-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index c3e467fe0..d92545fef 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -1,6 +1,12 @@ name: Run doctests on: workflow_call: + inputs: + python-versions: + description: 'List of Python versions to use in matrix, as JSON string' + required: true + type: string + default: '["3.9", "3.10", "3.11", "3.12"]' jobs: run: @@ -9,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ${{ fromJson(inputs.python-versions) }} os: [ubuntu-latest, macos-13, windows-latest] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index 28be2d94a..54d2e2519 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -1,8 +1,12 @@ name: Live service testing on: - schedule: - - cron: "0 16 * * *" # Daily at noon EST workflow_call: + inputs: + python-versions: + description: 'List of Python versions to use in matrix, as JSON string' + required: true + type: string + default: '["3.9", "3.10", "3.11", "3.12"]' secrets: DANDI_API_KEY: required: true @@ -17,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ${{ fromJson(inputs.python-versions) }} os: [ubuntu-latest, macos-13, windows-latest] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1d0fb9429..7d762f7f2 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,6 +1,13 @@ name: Minimal and Full Tests + on: workflow_call: + inputs: + python-versions: + description: 'List of Python versions to use in matrix, as JSON string' + required: true + type: string + default: '["3.9", "3.10", "3.11", "3.12"]' secrets: AWS_ACCESS_KEY_ID: required: true @@ -10,7 +17,6 @@ on: required: true CODECOV_TOKEN: required: true - workflow_dispatch: jobs: run: @@ -19,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ${{ fromJson(inputs.python-versions) }} os: [ubuntu-latest, macos-13, windows-latest] steps: - uses: actions/checkout@v4 @@ -79,42 +85,47 @@ jobs: #- name: Run icephys tests # There are no icephys specific tests without data # run: pytest tests/test_icephys -rsx -n auto --dist loadscope - - - name: Install full requirements run: pip install .[full] - - - name: Get ephy_testing_data current head hash id: ephys - run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" + run: | + HASH_EPHY_DATASET=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1) + echo "HASH_EPHY_DATASET=$HASH_EPHY_DATASET" >> $GITHUB_OUTPUT + - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }} uses: actions/cache@v4 id: cache-ephys-datasets with: path: ./ephy_testing_data key: ephys-datasets-2024-08-07-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + - name: Get ophys_testing_data current head hash id: ophys - run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + run: | + HASH_OPHYS_DATASET=$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1) + echo "HASH_OPHYS_DATASET=$HASH_OPHYS_DATASET" >> $GITHUB_OUTPUT + - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} uses: actions/cache@v4 id: cache-ophys-datasets with: path: ./ophys_testing_data key: ophys-datasets-2022-08-18-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + - name: Get behavior_testing_data current head hash id: behavior - run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)" + run: | + HASH_BEHAVIOR_DATASET=$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1) + echo "HASH_BEHAVIOR_DATASET=$HASH_BEHAVIOR_DATASET" >> $GITHUB_OUTPUT + - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} uses: actions/cache@v4 id: cache-behavior-datasets with: path: ./behavior_testing_data - key: behavior-datasets-2023-07-26-${{ matrix.os }}-${{ steps.behavior.outputs.HASH_behavior_DATASET }} - - + key: behavior-datasets-2023-07-26-${{ matrix.os }}-${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' || steps.cache-ophys-datasets.outputs.cache-hit != 'true' || steps.cache-behavior-datasets.outputs.cache-hit != 'true' name: Install and configure AWS CLI @@ -122,18 +133,22 @@ jobs: pip install awscli aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' name: Download ephys dataset from S3 run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ephy_testing_data ./ephy_testing_data + - if: steps.cache-ophys-datasets.outputs.cache-hit != 'true' name: Download ophys dataset from S3 run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ophys_testing_data ./ophys_testing_data + - if: steps.cache-behavior-datasets.outputs.cache-hit != 'true' name: Download behavior dataset from S3 run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/behavior_testing_data ./behavior_testing_data - name: Run full pytest with coverage run: pytest -vv -rsx -n auto --dist loadscope --cov=neuroconv --cov-report xml:./codecov.xml + - name: Upload full coverage to Codecov if: ${{ matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v4 From 20e491d09549703751882c1cf2cba8b03562e036 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 13 Sep 2024 18:30:01 -0600 Subject: [PATCH 2/9] correct error on hash modification --- .github/workflows/testing.yml | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 7d762f7f2..768f7ff2c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -90,42 +90,33 @@ jobs: - name: Get ephy_testing_data current head hash id: ephys - run: | - HASH_EPHY_DATASET=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1) - echo "HASH_EPHY_DATASET=$HASH_EPHY_DATASET" >> $GITHUB_OUTPUT - + run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }} uses: actions/cache@v4 id: cache-ephys-datasets with: path: ./ephy_testing_data key: ephys-datasets-2024-08-07-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - - name: Get ophys_testing_data current head hash id: ophys - run: | - HASH_OPHYS_DATASET=$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1) - echo "HASH_OPHYS_DATASET=$HASH_OPHYS_DATASET" >> $GITHUB_OUTPUT - + run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} uses: actions/cache@v4 id: cache-ophys-datasets with: path: ./ophys_testing_data key: ophys-datasets-2022-08-18-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - name: Get behavior_testing_data current head hash id: behavior - run: | - HASH_BEHAVIOR_DATASET=$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1) - echo "HASH_BEHAVIOR_DATASET=$HASH_BEHAVIOR_DATASET" >> $GITHUB_OUTPUT - + run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)" - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} uses: actions/cache@v4 id: cache-behavior-datasets with: path: ./behavior_testing_data - key: behavior-datasets-2023-07-26-${{ matrix.os }}-${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} + key: behavior-datasets-2023-07-26-${{ matrix.os }}-${{ steps.behavior.outputs.HASH_behavior_DATASET }} + + - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' || steps.cache-ophys-datasets.outputs.cache-hit != 'true' || steps.cache-behavior-datasets.outputs.cache-hit != 'true' name: Install and configure AWS CLI @@ -133,22 +124,18 @@ jobs: pip install awscli aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' name: Download ephys dataset from S3 run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ephy_testing_data ./ephy_testing_data - - if: steps.cache-ophys-datasets.outputs.cache-hit != 'true' name: Download ophys dataset from S3 run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ophys_testing_data ./ophys_testing_data - - if: steps.cache-behavior-datasets.outputs.cache-hit != 'true' name: Download behavior dataset from S3 run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/behavior_testing_data ./behavior_testing_data - name: Run full pytest with coverage run: pytest -vv -rsx -n auto --dist loadscope --cov=neuroconv --cov-report xml:./codecov.xml - - name: Upload full coverage to Codecov if: ${{ matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v4 From 1b4300faa1cc43f54ad89bc09d5a43c21bdc8faf Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 16 Sep 2024 11:11:29 -0600 Subject: [PATCH 3/9] revert workflow_dispatch removal --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 768f7ff2c..2420b2d33 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -17,6 +17,7 @@ on: required: true CODECOV_TOKEN: required: true + workflow_dispatch: jobs: run: From 2c9fbdbe9700ce2429772c22c4d614c08f19599c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 16 Sep 2024 12:49:32 -0600 Subject: [PATCH 4/9] paul request --- .github/workflows/deploy-tests.yml | 21 +++++++++++++++++---- .github/workflows/doctests.yml | 8 +++++++- .github/workflows/live-service-testing.yml | 8 +++++++- .github/workflows/testing.yml | 9 ++++++++- 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index f271b8c9a..928917112 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -40,11 +40,16 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - python-versions: > # Ternary operator, condition && value_if_true || value_if_false + python-versions: > # Ternary operator: condition && value_if_true || value_if_false ${{github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} + os-versions: > + ${{github.event.pull_request.draft == true + && '["ubuntu-latest"]' + || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' + }} run-live-service-tests: needs: assess-file-changes @@ -58,7 +63,11 @@ jobs: && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - + os-versions: > + ${{github.event.pull_request.draft == true + && '["ubuntu-latest"]' + || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' + }} run-dev-tests: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} @@ -80,12 +89,16 @@ jobs: if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml with: - python-versions: > # Ternary operator, condition && value_if_true || value_if_false + python-versions: > # Ternary operator: condition && value_if_true || value_if_false ${{github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - + os-versions: > + ${{github.event.pull_request.draft == true + && '["ubuntu-latest"]' + || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' + }} check-final-status: name: All tests passing if: always() diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index d92545fef..927d588b1 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -7,6 +7,12 @@ on: required: true type: string default: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: + description: 'List of OS versions to use in matrix, as JSON string' + required: true + type: string + default: '["ubuntu-latest", "macos-latest", "windows-latest"]' + jobs: run: @@ -16,7 +22,7 @@ jobs: fail-fast: false matrix: python-version: ${{ fromJson(inputs.python-versions) }} - os: [ubuntu-latest, macos-13, windows-latest] + os: ${{ fromJson(inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index 54d2e2519..e02aac02d 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -7,6 +7,12 @@ on: required: true type: string default: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: + description: 'List of OS versions to use in matrix, as JSON string' + required: true + type: string + default: '["ubuntu-latest", "macos-latest", "windows-latest"]' + secrets: DANDI_API_KEY: required: true @@ -22,7 +28,7 @@ jobs: fail-fast: false matrix: python-version: ${{ fromJson(inputs.python-versions) }} - os: [ubuntu-latest, macos-13, windows-latest] + os: ${{ fromJson(inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2420b2d33..2a00cb37a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -8,6 +8,13 @@ on: required: true type: string default: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: + description: 'List of OS versions to use in matrix, as JSON string' + required: true + type: string + default: '["ubuntu-latest", "macos-latest", "windows-latest"]' + + secrets: AWS_ACCESS_KEY_ID: required: true @@ -27,7 +34,7 @@ jobs: fail-fast: false matrix: python-version: ${{ fromJson(inputs.python-versions) }} - os: [ubuntu-latest, macos-13, windows-latest] + os: ${{ fromJson(inputs.python-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags From 8e3e2b2bc54a540fb6de9f6278c6a137b0881610 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 12:50:57 -0600 Subject: [PATCH 5/9] try env variable --- .github/workflows/deploy-tests.yml | 72 ++++++++++-------------------- 1 file changed, 24 insertions(+), 48 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 928917112..9678195fc 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -3,15 +3,20 @@ name: Deploy tests on: pull_request: types: [synchronize, opened, reopened, ready_for_review] - # Synchronize, open and reopened are the defaults for pull request + # Synchronize, open and reopened are the default types for pull request # We add ready_for_review to trigger the check for changelog and full tests when ready for review is clicked merge_group: workflow_dispatch: -concurrency: # Cancel previous workflows on the same pull request +concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + DRAFT_CONDITION: ${{ github.event.pull_request.draft == true }} + PYTHON_VERSIONS: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} + OS_VERSIONS: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + jobs: assess-file-changes: @@ -19,7 +24,7 @@ jobs: detect-changelog-updates: needs: assess-file-changes - if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' && github.event.pull_request.draft == false }} + if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' && !env.DRAFT_CONDITION }} name: Auto-detecting CHANGELOG.md updates runs-on: ubuntu-latest steps: @@ -40,16 +45,8 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - python-versions: > # Ternary operator: condition && value_if_true || value_if_false - ${{github.event.pull_request.draft == true - && '["3.9"]' - || '["3.9", "3.10", "3.11", "3.12"]' - }} - os-versions: > - ${{github.event.pull_request.draft == true - && '["ubuntu-latest"]' - || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' - }} + python-versions: ${{ env.PYTHON_VERSIONS }} + os-versions: ${{ env.OS_VERSIONS }} run-live-service-tests: needs: assess-file-changes @@ -58,16 +55,9 @@ jobs: secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} with: - python-versions: > # Ternary operator, condition && value_if_true || value_if_false - ${{github.event.pull_request.draft == true - && '["3.9"]' - || '["3.9", "3.10", "3.11", "3.12"]' - }} - os-versions: > - ${{github.event.pull_request.draft == true - && '["ubuntu-latest"]' - || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' - }} + python-versions: ${{ env.PYTHON_VERSIONS }} + os-versions: ${{ env.OS_VERSIONS }} + run-dev-tests: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} @@ -78,40 +68,26 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} with: - python-versions: > # Ternary operator, condition && value_if_true || value_if_false - ${{github.event.pull_request.draft == true - && '["3.9"]' - || '["3.9", "3.10", "3.11", "3.12"]' - }} + python-versions: ${{ env.PYTHON_VERSIONS }} run-doctests-only: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml with: - python-versions: > # Ternary operator: condition && value_if_true || value_if_false - ${{github.event.pull_request.draft == true - && '["3.9"]' - || '["3.9", "3.10", "3.11", "3.12"]' - }} - os-versions: > - ${{github.event.pull_request.draft == true - && '["ubuntu-latest"]' - || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' - }} + python-versions: ${{ env.PYTHON_VERSIONS }} + os-versions: ${{ env.OS_VERSIONS }} + check-final-status: name: All tests passing if: always() - needs: - - run-tests - - run-doctests-only - + - run-tests + - run-doctests-only runs-on: ubuntu-latest - steps: - - name: Decide whether the all jobs succeeded or at least one failed - uses: re-actors/alls-green@release/v1 - with: - allowed-skips: run-tests, run-doctests-only # Each has the option to skip depending on whether src changed - jobs: ${{ toJSON(needs) }} + - name: Decide whether all jobs succeeded or at least one failed + uses: re-actors/alls-green@release/v1 + with: + allowed-skips: run-tests, run-doctests-only + jobs: ${{ toJSON(needs) }} From 290fce9582dfae346da03d7bd801b97d4d06db0f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 12:51:36 -0600 Subject: [PATCH 6/9] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5bebee40..d52300016 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ * Add writing to zarr test for to the test on data [PR #1056](https://github.com/catalystneuro/neuroconv/pull/1056) * Modified the CI to avoid running doctests twice [PR #1077](https://github.com/catalystneuro/neuroconv/pull/#1077) * Consolidated daily workflows into one workflow and added email notifications [PR #1081](https://github.com/catalystneuro/neuroconv/pull/1081) +* Run only the most basic testing while a PR is on draft [PR #1082](https://github.com/catalystneuro/neuroconv/pull/1082) ## v0.6.3 From 23b2fc7551b9a62a47c53017c4099feebd06400c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 14:18:13 -0600 Subject: [PATCH 7/9] try draft explicit --- .github/workflows/deploy-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 9678195fc..6723c3a28 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -13,7 +13,6 @@ concurrency: cancel-in-progress: true env: - DRAFT_CONDITION: ${{ github.event.pull_request.draft == true }} PYTHON_VERSIONS: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} OS_VERSIONS: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} @@ -24,7 +23,7 @@ jobs: detect-changelog-updates: needs: assess-file-changes - if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' && !env.DRAFT_CONDITION }} + if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' && github.event.pull_request.draft == false }} name: Auto-detecting CHANGELOG.md updates runs-on: ubuntu-latest steps: From 4ad2c104eb7381c2ef32c4467277a6dbf18c7a60 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 14:55:35 -0600 Subject: [PATCH 8/9] env does not work; --- .github/workflows/deploy-tests.yml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 6723c3a28..3afd5e956 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -12,10 +12,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - PYTHON_VERSIONS: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - OS_VERSIONS: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} - jobs: assess-file-changes: @@ -23,7 +19,7 @@ jobs: detect-changelog-updates: needs: assess-file-changes - if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' && github.event.pull_request.draft == false }} + if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' && github.event.pull_request.draft == false }} name: Auto-detecting CHANGELOG.md updates runs-on: ubuntu-latest steps: @@ -43,9 +39,9 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - python-versions: ${{ env.PYTHON_VERSIONS }} - os-versions: ${{ env.OS_VERSIONS }} + with: # Ternary operator: condition && value_if_true || value_if_false + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} run-live-service-tests: needs: assess-file-changes @@ -53,9 +49,9 @@ jobs: uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} - with: - python-versions: ${{ env.PYTHON_VERSIONS }} - os-versions: ${{ env.OS_VERSIONS }} + with: # Ternary operator: condition && value_if_true || value_if_false + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} run-dev-tests: needs: assess-file-changes @@ -66,16 +62,16 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} - with: - python-versions: ${{ env.PYTHON_VERSIONS }} + with: # Ternary operator: condition && value_if_true || value_if_false + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} run-doctests-only: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml - with: - python-versions: ${{ env.PYTHON_VERSIONS }} - os-versions: ${{ env.OS_VERSIONS }} + with: # Ternary operator: condition && value_if_true || value_if_false + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} check-final-status: name: All tests passing From 4bd957c082c3e2b3db0fc6bbce1955ef27aaca61 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 15:39:26 -0600 Subject: [PATCH 9/9] fix mistake in testing --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 945055061..736da6030 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -34,7 +34,7 @@ jobs: fail-fast: false matrix: python-version: ${{ fromJson(inputs.python-versions) }} - os: ${{ fromJson(inputs.python-versions) }} + os: ${{ fromJson(inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags