diff --git a/.github/ISSUE_TEMPLATE/detox_nightly_build_failure.md b/.github/ISSUE_TEMPLATE/detox_nightly_build_failure.md new file mode 100644 index 00000000000..2f654b51763 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/detox_nightly_build_failure.md @@ -0,0 +1,15 @@ +--- +name: Detox Nightly Build Failure Template +about: Template for reporting a detox nightly build failure +title: "Bug - Detox - Fix Overnight Failures" +labels: bug, QA and Release +assignees: rbontrager + +--- + +## What failed? +Some detox tests have failed in the overnight build. Failures can be found [here](https://github.com/department-of-veterans-affairs/va-mobile-app/actions/workflows/e2e_ios.yml) for iOS and [here](https://github.com/department-of-veterans-affairs/va-mobile-app/actions/workflows/e2e_android.yml) for Android. Please look at the following list for what might be failing: + +- {{env.dateOfIssue}} {{env.OS}}: {{issues}} + + diff --git a/.github/workflows/e2e_android.yml b/.github/workflows/e2e_android.yml index 8577daba4c3..d733a1a0c86 100644 --- a/.github/workflows/e2e_android.yml +++ b/.github/workflows/e2e_android.yml @@ -251,7 +251,7 @@ jobs: if: steps.run_e2e_tests.outcome == 'failure' run: exit 1 - output-slack-results: + output-slack-results-and-update-detox-failure-ticket: if: (!cancelled()) && github.event_name == 'schedule' needs: [matrix-e2e-android, start_slack_thread] runs-on: macos-latest-xl @@ -259,11 +259,14 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.ref }} + - name: Download results uses: actions/download-artifact@v4 with: path: VAMobile/e2e/test_reports + - name: Inform Slack + id: inform_slack shell: bash run: | if grep -rq "" .; then @@ -281,6 +284,58 @@ jobs: https://slack.com/api/chat.postMessage| jq -r '.ts') fi + + - name: Find failing tests + id: failing_tests + shell: bash + run: | + failing_tests_list=$(grep -rl "") + failing_tests="" + for i in ${failing_tests_list[@]}; do + remove_before=${i%.ts*} + remove_after=${remove_before#*test_reports/} + echo "$remove_after" + if [ -z "${failing_tests}" ]; then + failing_tests=$(echo $failing_tests"$remove_after") + else + failing_tests=$(echo $failing_tests", ""$remove_after") + fi + done + echo "FAILING_TEST=$failing_tests" >> "$GITHUB_OUTPUT" + + - name: Github CLI Authentication + run: | + echo "${{ secrets.GITHUB_TOKEN }}" >> token.txt + gh auth login --with-token < token.txt + + - name: Update a ticket on failure (if needed) + id: find_if_ticket_already_exists + run: | + old_body=$(gh issue list --json body --jq '.[] | .body' --state open --search "Bug - Detox - Fix Overnight Failures in:title") + if [[ "$old_body" != '' ]]; then + ticket_number=$(gh issue list --json number --jq '.[] | .number' --state open --search "Bug - Detox - Fix Overnight Failures in:title") + new_body=$(echo "$old_body"$'\n'"- "${{ env.DATE_OF_ISSUE}}" on Android: "${{steps.failing_tests.outputs.FAILING_TEST}}) + echo "$new_body" + echo "TICKET_EXISTS=true" >> "$GITHUB_OUTPUT" + gh issue edit $ticket_number -b "$new_body" + else + echo "TICKET_EXISTS=false" >> "$GITHUB_OUTPUT" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Make a ticket on failure (if needed) + if: steps.inform_slack.outputs.TEST_FAILURE != '' && steps.find_if_ticket_already_exists.outputs.TICKET_EXISTS == '' + uses: JasonEtco/create-an-issue@v2 + with: + filename: .github/ISSUE_TEMPLATE/detox_nightly_build_failure.md + update_existing: true + search_existing: open + env: + dateOfIssue: ${{ env.DATE_OF_ISSUE }} + OS: "Android" + issues: ${{steps.failing_tests.outputs.FAILING_TEST}} + id: create-issue matrix-send_test_results_to_testrail: if: (!cancelled()) && github.event.inputs.run_testRail == 'true' diff --git a/.github/workflows/e2e_ios.yml b/.github/workflows/e2e_ios.yml index ae4757ee543..be004173938 100644 --- a/.github/workflows/e2e_ios.yml +++ b/.github/workflows/e2e_ios.yml @@ -228,7 +228,7 @@ jobs: if: steps.run_e2e_tests.outcome == 'failure' && steps.rerun_e2e_tests.outcome == 'failure' run: exit 1 - output-slack-results: + output-slack-results-and-update-detox-failure-ticket: if: (!cancelled()) && github.event_name == 'schedule' needs: [matrix-e2e-ios, start_slack_thread] runs-on: macos-latest-xl @@ -236,11 +236,14 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.ref }} + - name: Download results uses: actions/download-artifact@v4 with: path: VAMobile/e2e/test_reports + - name: Inform Slack + id: inform_slack shell: bash run: | if grep -rq "" .; then @@ -251,6 +254,8 @@ jobs: https://slack.com/api/chat.postMessage| jq -r '.ts') echo SLACK_THREAD_TS=${ts} >> $GITHUB_OUTPUT + echo "DATE_OF_ISSUE=$(date -d '+%A %b %d, %Y')" >> $GITHUB_ENV + echo TEST_FAILURE="true" >> $GITHUB_OUTPUT else ts=$(curl -X POST -H 'Authorization: Bearer '"$SLACK_API_TOKEN"' ' \ -H 'Content-type: application/json' \ @@ -259,6 +264,57 @@ jobs: jq -r '.ts') fi + - name: Find failing tests + id: failing_tests + shell: bash + run: | + failing_tests_list=$(grep -rl "") + failing_tests="" + for i in ${failing_tests_list[@]}; do + remove_before=${i%.ts*} + remove_after=${remove_before#*test_reports/} + echo "$remove_after" + if [ -z "${failing_tests}" ]; then + failing_tests=$(echo $failing_tests"$remove_after") + else + failing_tests=$(echo $failing_tests", ""$remove_after") + fi + done + echo "FAILING_TEST=$failing_tests" >> "$GITHUB_OUTPUT" + + - name: Github CLI Authentication + run: | + echo "${{ secrets.GITHUB_TOKEN }}" >> token.txt + gh auth login --with-token < token.txt + + - name: Update a ticket on failure (if needed) + id: find_if_ticket_already_exists + run: | + old_body=$(gh issue list --json body --jq '.[] | .body' --state open --search "Bug - Detox - Fix Overnight Failures in:title") + if [[ "$old_body" != '' ]]; then + ticket_number=$(gh issue list --json number --jq '.[] | .number' --state open --search "Bug - Detox - Fix Overnight Failures in:title") + new_body=$(echo "$old_body"$'\n'"- "${{ env.DATE_OF_ISSUE}}" on iOS: "${{steps.failing_tests.outputs.FAILING_TEST}}) + echo "$new_body" + echo "TICKET_EXISTS=true" >> "$GITHUB_OUTPUT" + gh issue edit $ticket_number -b "$new_body" + else + echo "TICKET_EXISTS=false" >> "$GITHUB_OUTPUT" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Make a ticket on failure (if needed) + if: steps.inform_slack.outputs.TEST_FAILURE != '' && steps.find_if_ticket_already_exists.outputs.TICKET_EXISTS == '' + uses: JasonEtco/create-an-issue@v2 + with: + filename: .github/ISSUE_TEMPLATE/detox_nightly_build_failure.md + search_existing: open + env: + dateOfIssue: ${{ env.DATE_OF_ISSUE }} + OS: "iOS" + issues: ${{steps.failing_tests.outputs.FAILING_TEST}} + id: create-issue + matrix-send_test_results_to_testrail: if: (!cancelled()) && github.event.inputs.run_testRail == 'true' needs: [matrix-e2e-ios, output_detox_tests_to_run]