Skip to content

Main

Main #14070

Workflow file for this run

name: Main
on:
push:
branches:
- develop
tags:
- v*
pull_request:
branches:
- develop
repository_dispatch:
types: [rebuild]
workflow_dispatch:
inputs:
skip-test:
description: "Skip test"
type: choice
required: true
default: "false"
options:
- "true"
- "false"
plugin-version:
description: "Plugin version"
required: false
type: string
default: "LATEST"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
JAVA_VERSION: "21"
DOCKER_APT_PACKAGES: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip
DOCKER_PYTHON_LIBRARIES: kestra
PLUGIN_VERSION: ${{ github.event.inputs.plugin-version != null && github.event.inputs.plugin-version || 'LATEST' }}
jobs:
build-artifacts:
name: Build Artifacts
runs-on: ubuntu-latest
timeout-minutes: 60
outputs:
docker-tag: ${{ steps.vars.outputs.tag }}
docker-artifact-name: ${{ steps.vars.outputs.artifact }}
plugins: ${{ steps.plugins-list.outputs.plugins }}
steps:
# Checkout
- uses: actions/checkout@v4
with:
fetch-depth: 0
# Checkout GitHub Actions
- uses: actions/checkout@v4
with:
repository: kestra-io/actions
path: actions
ref: main
# Npm
- name: Npm install
shell: bash
working-directory: ui
run: npm ci
- name: Npm lint
uses: reviewdog/action-eslint@v1
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
reporter: github-pr-review # Change reporter.
workdir: "ui"
# Setup build
- uses: ./actions/.github/actions/setup-build
id: build
with:
java-enabled: true
node-enabled: true
caches-enabled: true
# Get Plugins List
- name: Get Plugins List
uses: ./.github/actions/plugins-list
if: "!startsWith(github.ref, 'refs/tags/v')"
id: plugins-list
with:
plugin-version: ${{ env.PLUGIN_VERSION }}
# Set Plugins List
- name: Set Plugin List
id: plugins
if: "!startsWith(github.ref, 'refs/tags/v')"
run: |
PLUGINS="${{ steps.plugins-list.outputs.plugins }}"
TAG=${GITHUB_REF#refs/*/}
if [[ $TAG = "master" || $TAG == v* ]]; then
echo "plugins=$PLUGINS" >> $GITHUB_OUTPUT
else
echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots $PLUGINS" >> $GITHUB_OUTPUT
fi
# Build
- name: Build with Gradle
run: |
./gradlew executableJar
- name: Copy exe to image
run: |
cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra
# Docker Tag
- name: Set up Vars
id: vars
run: |
TAG=${GITHUB_REF#refs/*/}
if [[ $TAG = "master" ]]
then
TAG="latest";
elif [[ $TAG = "develop" ]]
then
TAG="develop";
elif [[ $TAG = v* ]]
then
TAG="${TAG}";
else
TAG="build-${{ github.run_id }}";
fi
echo "tag=${TAG}" >> $GITHUB_OUTPUT
echo "artifact=docker-kestra-${TAG}" >> $GITHUB_OUTPUT
# Docker setup
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# Docker Build
- name: Build & Export Docker Image
uses: docker/build-push-action@v6
if: "!startsWith(github.ref, 'refs/tags/v')"
with:
context: .
push: false
file: Dockerfile
tags: |
kestra/kestra:${{ steps.vars.outputs.tag }}
build-args: |
KESTRA_PLUGINS=${{ steps.plugins.outputs.plugins }}
APT_PACKAGES=${{ env.DOCKER_APT_PACKAGES }}
PYTHON_LIBRARIES=${{ env.DOCKER_PYTHON_LIBRARIES }}
outputs: type=docker,dest=/tmp/${{ steps.vars.outputs.artifact }}.tar
# Upload artifacts
- name: Upload JAR
uses: actions/upload-artifact@v4
with:
name: jar
path: build/libs/
- name: Upload Executable
uses: actions/upload-artifact@v4
with:
name: exe
path: build/executable/
- name: Upload Docker
uses: actions/upload-artifact@v4
if: "!startsWith(github.ref, 'refs/tags/v')"
with:
name: ${{ steps.vars.outputs.artifact }}
path: /tmp/${{ steps.vars.outputs.artifact }}.tar
# Run Reusable Workflow from QA repository
check-e2e:
name: Check E2E Tests
needs: build-artifacts
if: ${{ (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') && !startsWith(github.ref, 'refs/tags/v') }}
uses: ./.github/workflows/e2e.yml
strategy:
fail-fast: false
matrix:
backends: ["postgres"]
with:
tags: oss
docker-artifact-name: ${{ needs.build-artifacts.outputs.docker-artifact-name }}
docker-image-tag: kestra/kestra:${{ needs.build-artifacts.outputs.docker-tag }}
backend: ${{ matrix.backends }}
secrets:
GITHUB_AUTH_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
check:
env:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
name: Check & Publish
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
# Checkout
- uses: actions/checkout@v4
with:
fetch-depth: 0
# Checkout GitHub Actions
- uses: actions/checkout@v4
with:
repository: kestra-io/actions
path: actions
ref: main
# Setup build
- uses: ./actions/.github/actions/setup-build
id: build
with:
java-enabled: true
node-enabled: true
python-enabled: true
caches-enabled: true
# Services
- name: Build the docker-compose stack
run: docker compose -f docker-compose-ci.yml up -d
if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
# Gradle check
- name: Build with Gradle
if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
env:
GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
run: |
echo $GOOGLE_SERVICE_ACCOUNT | base64 -d > ~/.gcp-service-account.json
export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp-service-account.json
./gradlew check javadoc --parallel
# Sonar
- name: Analyze with Sonar
if: ${{ env.SONAR_TOKEN != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
run: ./gradlew sonar --info
# Allure check
- name: Auth to Google Cloud
id: auth
if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 }}
uses: "google-github-actions/auth@v2"
with:
credentials_json: "${{ secrets.GOOGLE_SERVICE_ACCOUNT }}"
- uses: rlespinasse/github-slug-action@v5
- name: Publish allure report
uses: andrcuns/[email protected]
if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
env:
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
JAVA_HOME: /usr/lib/jvm/default-jvm/
with:
storageType: gcs
resultsGlob: "**/build/allure-results"
bucket: internal-kestra-host
baseUrl: "https://internal.kestra.io"
prefix: ${{ format('{0}/{1}', github.repository, 'allure/java') }}
copyLatest: true
ignoreMissingResults: true
# Jacoco
- name: "Set up Cloud SDK"
if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
uses: "google-github-actions/setup-gcloud@v2"
- name: "Copy jacoco files"
if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
run: |
mv build/reports/jacoco/testCodeCoverageReport build/reports/jacoco/test/
mv build/reports/jacoco/test/testCodeCoverageReport.xml build/reports/jacoco/test/jacocoTestReport.xml
gsutil -m rsync -d -r build/reports/jacoco/test/ gs://internal-kestra-host/${{ format('{0}/{1}', github.repository, 'jacoco') }}
# report test
- name: Test Report
uses: mikepenz/action-junit-report@v5
if: success() || failure()
with:
report_paths: "**/build/test-results/**/TEST-*.xml"
# Codecov
- uses: codecov/codecov-action@v5
if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
with:
token: ${{ secrets.CODECOV_TOKEN }}
release:
name: Github Release
runs-on: ubuntu-latest
needs: [check, check-e2e]
if: |
always() &&
startsWith(github.ref, 'refs/tags/v') &&
needs.check.result == 'success' &&
(needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
steps:
# Download Exec
- name: Download executable
uses: actions/download-artifact@v4
if: startsWith(github.ref, 'refs/tags/v')
with:
name: exe
path: build/executable
# GitHub Release
- name: Create GitHub release
id: create_github_release
uses: "marvinpinto/action-automatic-releases@latest"
if: startsWith(github.ref, 'refs/tags/v')
continue-on-error: true
with:
repo_token: "${{ secrets.GITHUB_TOKEN }}"
prerelease: false
files: |
build/executable/*
# Trigger gha workflow to bump helm chart version
- name: trigger the Helm chart version bump
uses: peter-evans/repository-dispatch@v3
if: steps.create_github_release.conclusion == 'success'
with:
token: ${{ secrets.GH_PERSONAL_TOKEN }}
repository: kestra-io/helm-charts
event-type: update-helm-chart-version
client-payload: |-
{
"new_version": "${{ github.ref_name }}",
"github_repository": "${{ github.repository }}",
"github_actor": "${{ github.actor }}"
}
docker:
name: Publish Docker
runs-on: ubuntu-latest
needs: [build-artifacts, check, check-e2e]
if: github.ref == 'refs/heads/develop'
strategy:
matrix:
image:
- tag: ${{needs.build-artifacts.outputs.docker-tag}}-no-plugins
packages: ""
python-libraries: ""
- tag: ${{needs.build-artifacts.outputs.docker-tag}}
plugins: ${{needs.build-artifacts.outputs.plugins}}
packages: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip
python-libraries: kestra
steps:
- uses: actions/checkout@v4
# Docker setup
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# Docker Login
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
# Vars
- name: Set image name
id: vars
run: |
TAG=${GITHUB_REF#refs/*/}
if [[ $TAG = "master" || $TAG == v* ]]; then
echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
else
echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots ${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
fi
# Build Docker Image
- name: Download executable
uses: actions/download-artifact@v4
with:
name: exe
path: build/executable
- name: Copy exe to image
run: |
cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra
# Docker Build and push
- name: Build Docker Image
uses: docker/build-push-action@v6
with:
context: .
push: true
tags: kestra/kestra:${{ matrix.image.tag }}
platforms: linux/amd64,linux/arm64
build-args: |
KESTRA_PLUGINS=${{ steps.vars.outputs.plugins }}
APT_PACKAGES=${{matrix.image.packages}}
PYTHON_LIBRARIES=${{matrix.image.python-libraries}}
maven:
name: Publish to Maven
runs-on: ubuntu-latest
needs: [check, check-e2e]
if: |
always() &&
github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v') &&
needs.check.result == 'success' &&
(needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
steps:
- uses: actions/checkout@v4
# Checkout GitHub Actions
- uses: actions/checkout@v4
with:
repository: kestra-io/actions
path: actions
ref: main
# Setup build
- uses: ./actions/.github/actions/setup-build
id: build
with:
java-enabled: true
node-enabled: true
caches-enabled: true
# Publish
- name: Publish package to Sonatype
if: github.ref == 'refs/heads/develop'
env:
ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
run: |
mkdir -p ~/.gradle/
echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
./gradlew publishToSonatype
# Release
- name: Release package to Maven Central
if: startsWith(github.ref, 'refs/tags/v')
env:
ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
run: |
echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository
end:
runs-on: ubuntu-latest
needs:
- check-e2e
- check
- maven
- docker
- release
if: always()
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
steps:
# Update
- name: Update internal
uses: benc-uk/workflow-dispatch@v1
if: github.ref == 'refs/heads/develop' && needs.docker.result == 'success'
with:
workflow: oss-build.yml
repo: kestra-io/infra
ref: master
token: ${{ secrets.GH_PERSONAL_TOKEN }}
# Slack
- name: Slack notification
uses: Gamesight/slack-workflow-status@master
if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
name: GitHub Actions
icon_emoji: ":github-actions:"
channel: "C02DQ1A7JLR" # _int_git channel