Skip to content

Commit

Permalink
Create a github issue when a test fails.
Browse files Browse the repository at this point in the history
  • Loading branch information
shammowla committed Sep 30, 2024
1 parent a14c204 commit e633772
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 42 deletions.
53 changes: 53 additions & 0 deletions .github/scripts/handle-test-failures.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
module.exports = async ({ github, context, browser }) => {
const fs = require('fs');
const path = require('path');

// Read the test results
const testResultsDir = './test-results';
const files = fs.readdirSync(testResultsDir);

for (const file of files) {
const content = fs.readFileSync(path.join(testResultsDir, file), 'utf8');
let result;
try {
result = JSON.parse(content);
} catch (error) {
console.error(`Error parsing JSON from ${file}: ${error}`);
continue;
}

if (result.status === 'failed') {
const testName = result.name;
const sauceLabsLink = `https://app.saucelabs.com/tests/${result.id}`;

// Search for an existing issue
const issues = await github.rest.issues.listForRepo({
owner: context.repo.owner,
repo: context.repo.repo,
state: 'open',
labels: ['test-failure', browser]
});

const existingIssue = issues.data.find(issue => issue.title.includes(testName));

if (existingIssue) {
// Update existing issue
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: existingIssue.number,
body: `Test failed again in the latest run.\nBrowser: ${browser}\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})`
});
} else {
// Create new issue
await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title: `Test Failure: ${testName} (${browser})`,
body: `The test "${testName}" failed in the ${browser} browser.\n\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})`,
labels: ['test-failure', browser]
});
}
}
}
};
60 changes: 18 additions & 42 deletions .github/workflows/prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -143,22 +143,14 @@ jobs:
echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV
fi
- name: Notify Slack on Failure
- name: Handle Test Failures
if: failure()
uses: 8398a7/action-slack@v3
uses: actions/github-script@v7
with:
status: ${{ job.status }}
fields: repo,message,commit,author,action,eventName,ref,workflow
text: |
Alloy Prod Firefox Tests Failed
Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }}
Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }}
Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }}
Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
const handleTestFailures = require('./.github/scripts/handle-test-failures.js');
await handleTestFailures({ github, context, browser: 'safari' });
alloy-prod-e2e-firefox:
name: Firefox
Expand Down Expand Up @@ -244,22 +236,14 @@ jobs:
echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV
fi
- name: Notify Slack on Failure
- name: Handle Test Failures
if: failure()
uses: 8398a7/action-slack@v3
uses: actions/github-script@v7
with:
status: ${{ job.status }}
fields: repo,message,commit,author,action,eventName,ref,workflow
text: |
Alloy Prod Firefox Tests Failed
Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }}
Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }}
Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }}
Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
const handleTestFailures = require('./.github/scripts/handle-test-failures.js');
await handleTestFailures({ github, context, browser: 'firefox' });
alloy-prod-e2e-chrome:
name: Chrome
Expand Down Expand Up @@ -328,19 +312,11 @@ jobs:
ALLOY_ENV=prod
config-file: ./.sauce/prod-chrome.yml

- name: Notify Slack on Failure
- name: Handle Test Failures
if: failure()
uses: 8398a7/action-slack@v3
uses: actions/github-script@v7
with:
status: ${{ job.status }}
fields: repo,message,commit,author,action,eventName,ref,workflow
text: |
Alloy Prod Firefox Tests Failed
Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }}
Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }}
Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }}
Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
const handleTestFailures = require('./.github/scripts/handle-test-failures.js');
await handleTestFailures({ github, context, browser: 'chrome' });

0 comments on commit e633772

Please sign in to comment.