benchmarks.yml

name: Run Benchmarks

on:
  workflow_dispatch:
    # Allows manually triggering release
    inputs:
      release_version:
        description: "Version to benchmark (set to 'unreleased' to benchmark the unstable branch)"
        required: false
        default: "unreleased"
      exp_strategy:
        description: "Limit the benchmark runs to just the given strategy -- leave blank to run all default strategies (run with `arrays-encoding` or see the *STRATEGIES file for supported strategies)"
        required: false
        default: ""
  # schedule:
  # * is a special character in YAML so you have to quote this string
  # Automatically prepare a minor version release every Saturday
  # See https://crontab.guru/#0_0_*_*_6
  # - cron: "0 0 * * 6"
  pull_request:

env:
  VERSION: ${{ github.event.inputs.release_version || 'unreleased' }}
  STRATEGY: ${{ github.event.inputs.exp_strategy }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
  run-benchmarks:
    runs-on: ubuntu-latest
    steps:
      - name: checkout benchmarks repo
        uses: actions/checkout@v2
        with:
          # Fetch a non-shallow copy of repo to prevent "shallow update not
          # allowed" errors
          # See https://stackoverflow.com/a/62293647/1187277
          fetch-depth: 0

      - name: set up java
        uses: actions/setup-java@v1
        with:
          java-version: 1.8

      - name: install apalache
        run: |
          ./scripts/get-apalache.sh
          echo "version=$(cat _VERSION)" >> $GITHUB_ENV

      - name: set up python 3.7
        uses: actions/setup-python@v1
        with:
          python-version: 3.7

      - name: install python dependencies
        run: |
          pip install pipenv
          pipenv install --system --deploy --ignore-pipfile

      - name: run benchmarks
        run: |
          if [[ -n "${{ env.STRATEGY }}" ]]
          then
            if [[ "${{ env.STRATEGY }}" == "arrays-encoding" ]]
            then
              make arrays-encoding version="${{ env.version }}"
            else
              make run strat="${{ env.STRATEGY }}" version="${{ env.version }}"
            fi
          else
            while read strat; do
              make run strat=$strat version="${{ env.version }}"
            done < STRATEGIES
          fi
      - name: open PR with report
        run: |
          git config --global user.name "$GITHUB_ACTOR"
          git config --global user.email "github@actions.ci"
          git checkout -b "report/${{ env.STRATEGY }}${{ env.version }}"
          git add results
          commit_msg="Add reports from v${{ env.version }}"
          git commit -m "$commit_msg"
          hub pull-request --push \
            --message="$commit_msg" \
            --message="# Review Instructions" \
            --message="- If this is a SNAPSHOT version, please review the reports and CLOSE WITHOUT MERGING." \
            --message="- If this a regular release, you can merge to publish the results." \
            --base="master" \
            --reviewer=konnov,shonfeder