Merge pull request #24847 from cbjeukendrup/ci_cleanup

CI cleanup / don't unexpectedly publish nightly builds when manually run with "publish: off"
musescore · Oct 5, 2024 · 5890de8 · 5890de8
2 parents fc8445e + f63f28c
commit 5890de8
Show file tree

Hide file tree

Showing 27 changed files with 458 additions and 623 deletions.
diff --git a/.github/workflows/build_backend.yml b/.github/workflows/build_backend.yml
@@ -47,50 +47,54 @@ jobs:
         BUILD_NUMBER=$(cat ./build.artifacts/env/build_number.env)
 
         DO_PUBLISH='false'
-        if [ "${{ inputs.publish }}" = "on" ]; then DO_PUBLISH='true'; fi
-        if [ -z "${{ secrets.S3_KEY_CONVERTER }}" ]; then DO_PUBLISH='false'; fi
-        if [ -z "${{ secrets.S3_SECRET_CONVERTER }}" ]; then DO_PUBLISH='false'; fi
+        if [ "${{ inputs.publish }}" = "on" ]; then 
+          DO_PUBLISH='true';
+          if [ -z "${{ secrets.S3_KEY_CONVERTER }}" ]; then 
+            echo "::warning::S3_KEY_CONVERTER is empty; publishing to S3 disabled"
+            DO_PUBLISH='false'
+          fi
+          if [ -z "${{ secrets.S3_SECRET_CONVERTER }}" ]; then
+            echo "::warning::S3_SECRET_CONVERTER is empty; publishing to S3 disabled"
+            DO_PUBLISH='false'
+          fi
+        fi
 
         bash ./buildscripts/ci/tools/make_version_env.sh $BUILD_NUMBER
         VERSION=$(cat ./build.artifacts/env/build_version.env)
         GITHUB_ARTIFACT_NAME="MuseScore-${VERSION}"
 
         echo "github.repository: ${{ github.repository }}"
-        echo "BUILD_MODE=$BUILD_MODE" >> $GITHUB_ENV
-        echo "BUILD_MODE: $BUILD_MODE"
-        echo "BUILD_NUMBER=$BUILD_NUMBER" >> $GITHUB_ENV
-        echo "BUILD_NUMBER: $BUILD_NUMBER"
-        echo "DO_PUBLISH=$DO_PUBLISH" >> $GITHUB_ENV
-        echo "DO_PUBLISH: $DO_PUBLISH"
-        echo "GITHUB_ARTIFACT_NAME=$GITHUB_ARTIFACT_NAME" >> $GITHUB_ENV
-        echo "GITHUB_ARTIFACT_NAME: $GITHUB_ARTIFACT_NAME"
+        echo "BUILD_MODE=$BUILD_MODE" | tee -a $GITHUB_ENV
+        echo "BUILD_NUMBER=$BUILD_NUMBER" | tee -a $GITHUB_ENV
+        echo "DO_PUBLISH=$DO_PUBLISH" | tee -a $GITHUB_ENV
+        echo "GITHUB_ARTIFACT_NAME=$GITHUB_ARTIFACT_NAME" | tee -a $GITHUB_ENV
 
     - name: Setup environment
       run: |
-        sudo bash ./buildscripts/ci/backend/setup.sh
+        bash ./buildscripts/ci/backend/setup.sh
 
     - name: Build
       run: |
-        sudo bash ./buildscripts/ci/backend/build.sh -n ${{ env.BUILD_NUMBER }} --build_videoexport
-    
+        bash ./buildscripts/ci/backend/build.sh -n ${{ env.BUILD_NUMBER }} --build_videoexport
+
     - name: Package
       run: |
-        sudo bash ./buildscripts/ci/backend/package.sh
+        bash ./buildscripts/ci/backend/package.sh
 
-    - name: Publish to S3 
+    - name: Publish to S3
       if: env.DO_PUBLISH == 'true'
       run: |
-        sudo bash ./buildscripts/ci/backend/publish_to_s3.sh --s3_key ${{ secrets.S3_KEY_CONVERTER }} --s3_secret ${{ secrets.S3_SECRET_CONVERTER }}
+        bash ./buildscripts/ci/backend/publish_to_s3.sh --s3_key ${{ secrets.S3_KEY_CONVERTER }} --s3_secret ${{ secrets.S3_SECRET_CONVERTER }}
 
     - name: Build Docker
       if: env.DO_PUBLISH == 'true'
       run: |
-        sudo bash ./buildscripts/ci/backend/build_docker.sh
+        bash ./buildscripts/ci/backend/build_docker.sh
 
-    - name: Publish to Registry 
+    - name: Publish to Registry
       if: env.DO_PUBLISH == 'true'
       run: |
-        sudo bash ./buildscripts/ci/backend/publish_to_registry.sh --token ${{ secrets.PACKAGES_PAT }} 
+        bash ./buildscripts/ci/backend/publish_to_registry.sh --token ${{ secrets.PACKAGES_PAT }}
 
     - name: Upload artifacts on GitHub
       if: ${{ always() }}

diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml
@@ -2,9 +2,6 @@ name: 'Build: Linux'
 
 on:
   pull_request:
-    branches:
-    - master
-
   schedule:
     - cron: '0 3 */1 */1 *' # At 03:00 on every day-of-month for master
     - cron: '0 5 */1 */1 *' # At 05:00 on every day-of-month for current release branch
@@ -46,6 +43,7 @@ env:
 jobs:
   linux_x64:
     runs-on: ubuntu-20.04
+    if: github.event_name != 'schedule' || github.repository == 'musescore/MuseScore'
     steps:
     - name: Cancel Previous Runs
       uses: styfle/[email protected]
@@ -54,7 +52,7 @@ jobs:
     - name: Exit if current release branch configuration is incorrect
       if: ${{ github.event_name == 'schedule' && github.event.schedule == '0 5 */1 */1 *' && env.CURRENT_RELEASE_BRANCH == '' }}
       run: |
-        echo "::error::CURRENT_RELEASE_BRANCH is not set"
+        echo "::error::CURRENT_RELEASE_BRANCH is empty"
         exit 1
     - name: Clone repository (default)
       uses: actions/checkout@v4
@@ -64,69 +62,42 @@ jobs:
       if: ${{ github.event_name == 'schedule' && github.event.schedule == '0 5 */1 */1 *' }}
       with:
         ref: ${{ env.CURRENT_RELEASE_BRANCH }}
-    - name: Ccache cache files
-      uses: actions/cache@v4
-      with:
-        path: ~/.ccache
-        key: ${{github.workflow}}-ccache-$(date -u +"%F-%T")
-        restore-keys: ${{github.workflow}}-ccache-  # restore ccache from either previous build on this branch or on master  
-    - name: Setup ccache
-      run: |
-        sudo bash ./buildscripts/ci/tools/setup_ccache.sh     
     - name: "Configure workflow"
       env:
         pull_request_title: ${{ github.event.pull_request.title }}
         SENTRY_SERVER_MU4_KEY: ${{ secrets.SENTRY_SERVER_MU4_KEY }}
         SENTRY_SERVER_SANDBOX_KEY: ${{ secrets.SENTRY_SERVER_SANDBOX_KEY }}
         SENTRY_PROJECT: ${{ inputs.sentry_project }}
       run: |
-        sudo bash ./buildscripts/ci/tools/make_build_mode_env.sh -e ${{ github.event_name }} -m ${{ inputs.build_mode }}
+        bash ./buildscripts/ci/tools/make_build_mode_env.sh -e ${{ github.event_name }} -m ${{ inputs.build_mode }}
         BUILD_MODE=$(cat ./build.artifacts/env/build_mode.env)
 
-        sudo bash ./buildscripts/ci/tools/make_build_number.sh
+        bash ./buildscripts/ci/tools/make_build_number.sh
         BUILD_NUMBER=$(cat ./build.artifacts/env/build_number.env)
 
-        DO_PUBLISH='false'
-        if [[ "${{ inputs.publish }}" == "on" || "$BUILD_MODE" == "nightly" ]]; then
-          DO_PUBLISH='true'
-          if [ -z "${{ secrets.OSUOSL_SSH_ENCRYPT_SECRET }}" ]; then 
-            echo "warning: not set OSUOSL_SSH_ENCRYPT_SECRET, publish disabled" 
-            DO_PUBLISH='false'
-          fi  
-        fi
-
-        DO_BUILD='true'
-        if [ "$BUILD_MODE" == "nightly" ]; then 
-          if [ "${{ github.repository }}" != "musescore/MuseScore" ]; then 
-            DO_BUILD='false'
-          fi
-        fi
-
         DO_UPDATE_TS='false'
-        if [[ "$BUILD_MODE" == "testing" || "$BUILD_MODE" == "stable" ]]; then 
-            DO_UPDATE_TS='true'
-            if [ -z "${{ secrets.TRANSIFEX_API_TOKEN }}" ]; then 
-              echo "warning: not set TRANSIFEX_API_TOKEN, update .ts disabled" 
-              DO_UPDATE_TS='false'
-            fi 
+        if [[ "$BUILD_MODE" == "testing" || "$BUILD_MODE" == "stable" ]]; then
+          DO_UPDATE_TS='true'
+          if [ -z "${{ secrets.TRANSIFEX_API_TOKEN }}" ]; then
+            echo "::warning::TRANSIFEX_API_TOKEN is empty; updating .ts files disabled"
+            DO_UPDATE_TS='false'
+          fi
         fi
 
         DO_PLACEHOLDER_TRANSLATIONS='false'
-        if [[ "$DO_BUILD" == "true" ]]; then
-          if [[ "$BUILD_MODE" == "nightly" || "$BUILD_MODE" == "devel" ]]; then
-            DO_PLACEHOLDER_TRANSLATIONS='true'
-          fi
+        if [[ "$BUILD_MODE" == "nightly" || "$BUILD_MODE" == "devel" ]]; then
+          DO_PLACEHOLDER_TRANSLATIONS='true'
         fi
 
         DO_UPLOAD_SYMBOLS='false'
         SENTRY_URL=""
 
-        if [ "$SENTRY_SERVER_MU4_KEY" != "" ]; then 
-          if [ -z "$SENTRY_PROJECT" ] && [ "$BUILD_MODE" == "stable" ]; then 
+        if [ "$SENTRY_SERVER_MU4_KEY" != "" ]; then
+          if [ -z "$SENTRY_PROJECT" ] && [ "$BUILD_MODE" == "stable" ]; then
             SENTRY_PROJECT="mu4"
           fi
 
-          if [ "$SENTRY_PROJECT" == "mu4" ]; then 
+          if [ "$SENTRY_PROJECT" == "mu4" ]; then
             DO_UPLOAD_SYMBOLS='true'
             SENTRY_URL=https://sentry.musescore.org/api/4/minidump/?sentry_key=$SENTRY_SERVER_MU4_KEY
           fi
@@ -136,14 +107,18 @@ jobs:
           SENTRY_PROJECT="sandbox"
         fi
 
-        if [ "$SENTRY_PROJECT" == "sandbox" ] && [ "$SENTRY_SERVER_SANDBOX_KEY" != "" ]; then 
+        if [ "$SENTRY_PROJECT" == "sandbox" ] && [ "$SENTRY_SERVER_SANDBOX_KEY" != "" ]; then
           DO_UPLOAD_SYMBOLS='true'
           SENTRY_URL=https://sentry.musescore.org/api/3/minidump/?sentry_key=$SENTRY_SERVER_SANDBOX_KEY
         fi
 
-        if [ $DO_BUILD == 'false' ]; then
-          DO_UPLOAD_SYMBOLS='false'
-          DO_PUBLISH='false'
+        DO_PUBLISH='false'
+        if [[ "${{ inputs.publish }}" == "on" || ("${{ github.event_name }}" == "schedule" && $BUILD_MODE == 'nightly') ]]; then
+          DO_PUBLISH='true'
+          if [ -z "${{ secrets.OSUOSL_SSH_ENCRYPT_SECRET }}" ]; then
+            echo "::warning::OSUOSL_SSH_ENCRYPT_SECRET is empty; not publishing to OSUOSL"
+            DO_PUBLISH='false'
+          fi
         fi
 
         ADD_INFO="_${GITHUB_REF#refs/heads/}"
@@ -152,77 +127,73 @@ jobs:
         UPLOAD_ARTIFACT_NAME="$(tr '":<>|*?/\\’' '_' <<<"MU4_${BUILD_NUMBER}_Lin${ADD_INFO}")"
 
         echo "github.repository: ${{ github.repository }}"
-        echo "BUILD_MODE=$BUILD_MODE" >> $GITHUB_ENV
-        echo "BUILD_MODE: $BUILD_MODE"
-        echo "BUILD_NUMBER=$BUILD_NUMBER" >> $GITHUB_ENV
-        echo "BUILD_NUMBER: $BUILD_NUMBER"
-        echo "DO_BUILD=$DO_BUILD" >> $GITHUB_ENV
-        echo "DO_BUILD: $DO_BUILD"
-        echo "DO_UPDATE_TS=$DO_UPDATE_TS" >> $GITHUB_ENV
-        echo "DO_UPDATE_TS: $DO_UPDATE_TS"
-        echo "DO_PLACEHOLDER_TRANSLATIONS=$DO_PLACEHOLDER_TRANSLATIONS" >> $GITHUB_ENV
-        echo "DO_PLACEHOLDER_TRANSLATIONS: $DO_PLACEHOLDER_TRANSLATIONS"
-        echo "DO_PUBLISH=$DO_PUBLISH" >> $GITHUB_ENV
-        echo "DO_PUBLISH: $DO_PUBLISH"
-        echo "DO_UPLOAD_SYMBOLS=$DO_UPLOAD_SYMBOLS" >> $GITHUB_ENV
-        echo "DO_UPLOAD_SYMBOLS: $DO_UPLOAD_SYMBOLS"
-        echo "SENTRY_PROJECT=$SENTRY_PROJECT" >> $GITHUB_ENV
-        echo "SENTRY_PROJECT: $SENTRY_PROJECT"
-        echo "SENTRY_URL=$SENTRY_URL" >> $GITHUB_ENV
-        echo "SENTRY_URL: $SENTRY_URL"
-        echo "UPLOAD_ARTIFACT_NAME=$UPLOAD_ARTIFACT_NAME" >> $GITHUB_ENV
-        echo "UPLOAD_ARTIFACT_NAME: $UPLOAD_ARTIFACT_NAME"
+        echo "BUILD_MODE=$BUILD_MODE" | tee -a $GITHUB_ENV
+        echo "BUILD_NUMBER=$BUILD_NUMBER" | tee -a $GITHUB_ENV
+        echo "DO_UPDATE_TS=$DO_UPDATE_TS" | tee -a $GITHUB_ENV
+        echo "DO_PLACEHOLDER_TRANSLATIONS=$DO_PLACEHOLDER_TRANSLATIONS" | tee -a $GITHUB_ENV
+        echo "DO_UPLOAD_SYMBOLS=$DO_UPLOAD_SYMBOLS" | tee -a $GITHUB_ENV
+        echo "SENTRY_PROJECT=$SENTRY_PROJECT" | tee -a $GITHUB_ENV
+        echo "SENTRY_URL=$SENTRY_URL" | tee -a $GITHUB_ENV
+        echo "DO_PUBLISH=$DO_PUBLISH" | tee -a $GITHUB_ENV
+        echo "UPLOAD_ARTIFACT_NAME=$UPLOAD_ARTIFACT_NAME" | tee -a $GITHUB_ENV
+
+        echo "CCACHE_TIMESTAMP=$(date -u +"%F-%T")" | tee -a $GITHUB_ENV
+
+    - name: Restore ccache files
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/.ccache
+        key: ${{github.workflow}}-ccache-${{ env.CCACHE_TIMESTAMP }}
+        restore-keys: ${{github.workflow}}-ccache-
+    - name: Setup ccache
+      run: |
+        sudo apt-get update && sudo apt-get install -y ccache
+        bash ./buildscripts/ci/tools/setup_ccache_config.sh
 
     - name: Setup environment
-      if: env.DO_BUILD == 'true'
       run: |
-        sudo bash ./buildscripts/ci/linux/setup.sh
+        bash ./buildscripts/ci/linux/setup.sh
     - name: Generate _en.ts files
-      if: env.DO_BUILD == 'true'
       env:
         LUPDATE_ARGS: ""
         POSTPROCESS_ARGS: ${{ env.DO_PLACEHOLDER_TRANSLATIONS == 'true' && '--generate-placeholder-translations' || '' }}
       run: |
-        sudo bash ./buildscripts/ci/translation/run_lupdate.sh
+        bash ./buildscripts/ci/translation/run_lupdate.sh
     - name: Update .ts files
       if: env.DO_UPDATE_TS == 'true'
       run: |
-        sudo bash ./buildscripts/ci/translation/tx_install.sh -t ${{ secrets.TRANSIFEX_API_TOKEN }} -s linux
-        sudo bash ./buildscripts/ci/translation/tx_pull.sh
+        bash ./buildscripts/ci/translation/tx_install.sh -t ${{ secrets.TRANSIFEX_API_TOKEN }} -s linux
+        bash ./buildscripts/ci/translation/tx_pull.sh
     - name: Build
-      if: env.DO_BUILD == 'true'
       run: |
         C_URL=${SENTRY_URL}; if [ -z "$C_URL" ]; then C_URL="''"; fi
-        sudo bash ./buildscripts/ci/linux/build.sh -n ${{ env.BUILD_NUMBER }} --crash_log_url $C_URL
+        bash ./buildscripts/ci/linux/build.sh -n ${{ env.BUILD_NUMBER }} --crash_log_url $C_URL
         echo "============== ccache ==============="
         ccache -s
-    - name: Package 
-      if: env.DO_BUILD == 'true'
+    - name: Package
       run: |
-        sudo bash ./buildscripts/ci/linux/package.sh
-    - name: Checksum 
-      if: env.DO_BUILD == 'true'
+        bash ./buildscripts/ci/linux/package.sh
+    - name: Checksum
       run: |
-        sudo bash ./buildscripts/ci/tools/checksum.sh 
+        bash ./buildscripts/ci/tools/checksum.sh
     - name: Generate and upload dump symbols
       if: env.DO_UPLOAD_SYMBOLS == 'true'
       run: |
         APP_BIN=$(find "$(pwd)/build.release/src/app/" -type f -name "mscore4portable*" -print -quit)
         ARCH=x86-64
-        sudo cmake -DAPP_BIN=${APP_BIN} \
-                   -DARCH=${ARCH} \
-                   -DSENTRY_URL=https://sentry.musescore.org \
-                   -DSENTRY_ORG=musescore \
-                   -DSENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
-                   -DSENTRY_PROJECT=${SENTRY_PROJECT} \
-                   -P buildscripts/ci/crashdumps/ci_generate_and_upload.cmake
-    - name: Publish package
+        cmake -DAPP_BIN=${APP_BIN} \
+              -DARCH=${ARCH} \
+              -DSENTRY_URL=https://sentry.musescore.org \
+              -DSENTRY_ORG=musescore \
+              -DSENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
+              -DSENTRY_PROJECT=${SENTRY_PROJECT} \
+              -P buildscripts/ci/crashdumps/ci_generate_and_upload.cmake
+    - name: Publish to OSUOSL
       if: env.DO_PUBLISH == 'true'
       run: |
-        sudo bash ./buildscripts/ci/tools/osuosl/publish.sh -s ${{ secrets.OSUOSL_SSH_ENCRYPT_SECRET }} --os linux -v 4
+        bash ./buildscripts/ci/tools/osuosl/publish.sh -s ${{ secrets.OSUOSL_SSH_ENCRYPT_SECRET }} --os linux -v 4
     - name: Upload artifacts on GitHub
-      if: env.DO_BUILD == 'true'
       uses: actions/upload-artifact@v4
       with:
         name: ${{ env.UPLOAD_ARTIFACT_NAME }}
-        path: ./build.artifacts/      
+        path: ./build.artifacts/