Skip to content
This repository has been archived by the owner on Oct 28, 2024. It is now read-only.
Integration Test Suite

Create granular event logs #73

Sign in to view logs

Create granular event logs

Create granular event logs #73

Workflow file for this run

.github/workflows/integration.yml at 898be31
# Integration Test Suite
#
# This workflow runs the integration tests. If the workflow is triggered in the merge queue, all integration tests
# are run. If the workflow is triggered in a PR commit, then the files changed in the PR are evaluated to determine
# if any integration tests will run.
name: Integration Test Suite
on:
pull_request:
merge_group:
types: [checks_requested]
concurrency:
# `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue
group: ${{ github.workflow }}-${{ github.event.number || github.event.merge_group.head_sha }}
cancel-in-progress: true
env:
AWS_ACCESS_KEY_ID: "dummy"
AWS_SECRET_ACCESS_KEY: "dummy"
CONTAINER_TOOL: "docker"
DD_ENV: "ci"
DD_API_KEY: ${{ secrets.DD_API_KEY }}
TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }}
TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }}
AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }}
RUST_BACKTRACE: full
TEST_LOG: vector=debug
VERBOSE: true
CI: true
PROFILE: debug
# observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps
# can be removed when we switch back to the upstream openssl-sys crate
CARGO_NET_GIT_FETCH_WITH_CLI: true
jobs:
changes:
if: github.event_name == 'pull_request'
uses: ./.github/workflows/changes.yml
with:
base_ref: ${{ github.event.pull_request.base.ref }}
head_ref: ${{ github.event.pull_request.head.ref }}
source: false
int_tests: true
secrets: inherit
integration-tests:
name: Integration Tests
runs-on: ubuntu-20.04-4core
needs: changes
if: always() && (
github.event_name == 'merge_group' || (
needs.changes.outputs.all-int == 'true'
|| needs.changes.outputs.amqp == 'true'
|| needs.changes.outputs.appsignal == 'true'
|| needs.changes.outputs.aws == 'true'
|| needs.changes.outputs.axiom == 'true'
|| needs.changes.outputs.azure == 'true'
|| needs.changes.outputs.clickhouse == 'true'
|| needs.changes.outputs.databend == 'true'
|| needs.changes.outputs.datadog-agent == 'true'
|| needs.changes.outputs.datadog-logs == 'true'
|| needs.changes.outputs.datadog-metrics == 'true'
|| needs.changes.outputs.datadog-traces == 'true'
|| needs.changes.outputs.dnstap == 'true'
|| needs.changes.outputs.docker-logs == 'true'
|| needs.changes.outputs.elasticsearch == 'true'
|| needs.changes.outputs.eventstoredb == 'true'
|| needs.changes.outputs.fluent == 'true'
|| needs.changes.outputs.gcp == 'true'
|| needs.changes.outputs.greptimedb == 'true'
|| needs.changes.outputs.humio == 'true'
|| needs.changes.outputs.http-client == 'true'
|| needs.changes.outputs.influxdb == 'true'
|| needs.changes.outputs.kafka == 'true'
|| needs.changes.outputs.logstash == 'true'
|| needs.changes.outputs.loki == 'true'
|| needs.changes.outputs.mongodb == 'true'
|| needs.changes.outputs.nats == 'true'
|| needs.changes.outputs.nginx == 'true'
|| needs.changes.outputs.opentelemetry == 'true'
|| needs.changes.outputs.postgres == 'true'
|| needs.changes.outputs.prometheus == 'true'
|| needs.changes.outputs.pulsar == 'true'
|| needs.changes.outputs.redis == 'true'
|| needs.changes.outputs.splunk == 'true'
|| needs.changes.outputs.webhdfs == 'true'
)
)
timeout-minutes: 90
steps:
- uses: actions/checkout@v3
with:
submodules: "recursive"
- run: sudo npm -g install @datadog/datadog-ci
- run: docker image prune -af ; docker container prune -f
- name: Determine if secrets are defined (PR author is team member).
if: github.event_name == 'pull_request'
env:
GH_APP_DATADOG_VECTOR_CI_APP_ID: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }}
run: |
if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]] ; then
echo "PR_HAS_ACCESS_TO_SECRETS=true" >> "$GITHUB_ENV"
else
echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV"
fi
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.amqp == 'true' }}
name: amqp
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int amqp
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.appsignal == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: appsignal
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int appsignal
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.aws == 'true' }}
name: aws
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int aws
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.axiom == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: axiom
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int axiom
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.azure == 'true' }}
name: azure
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int azure
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.clickhouse == 'true' }}
name: clickhouse
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int clickhouse
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.databend == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: databend
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int databend
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-agent == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: datadog-agent
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int datadog-agent
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-logs == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: datadog-logs
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int datadog-logs
- run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-metrics == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: datadog-metrics
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int datadog-metrics
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-traces == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: datadog-traces
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int datadog-traces
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }}
name: dnstap
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int dnstap
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.docker-logs == 'true' }}
name: docker-logs
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int docker-logs
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.elasticsearch == 'true' }}
name: elasticsearch
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int elasticsearch
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.eventstoredb == 'true' }}
name: eventstoredb
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int eventstoredb
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.fluent == 'true' }}
name: fluent
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int fluent
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.gcp == 'true' }}
name: gcp
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int gcp
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.greptimedb == 'true' }}
name: greptimedb
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int greptimedb
- run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.humio == 'true' }}
name: humio
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int humio
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.http-client == 'true' }}
name: http-client
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int http-client
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.influxdb == 'true' }}
name: influxdb
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int influxdb
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.kafka == 'true' }}
name: kafka
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int kafka
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.logstash == 'true' }}
name: logstash
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int logstash
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.loki == 'true' }}
name: loki
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int loki
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.mongodb == 'true' }}
name: mongodb
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int mongodb
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.mqtt == 'true' }}
name: mqtt
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int mqtt
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nats == 'true' }}
name: nats
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int nats
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nginx == 'true' }}
name: nginx
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int nginx
- run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.opentelemetry == 'true' }}
name: opentelemetry
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int opentelemetry
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.postgres == 'true' }}
name: postgres
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int postgres
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.prometheus == 'true' }}
name: prometheus
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int prometheus
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.pulsar == 'true' }}
name: pulsar
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int pulsar
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.redis == 'true' }}
name: redis
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int redis
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' }}
name: shutdown
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int shutdown
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.splunk == 'true' }}
name: splunk
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int splunk
- if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.webhdfs == 'true' }}
name: webhdfs
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
command: bash scripts/ci-int-e2e-test.sh int webhdfs
integration-test-suite:
name: Integration Test Suite
runs-on: ubuntu-latest
timeout-minutes: 5
if: always()
needs:
- changes
- integration-tests
env:
FAILED: ${{ contains(needs.*.result, 'failure') }}
steps:
- run: |
echo "failed=${{ env.FAILED }}"
if [[ "$FAILED" == "true" ]] ; then
exit 1
else
exit 0
fi