Upgrade CI (18 Aug 2021 19:15:54)

lsst · Aug 18, 2021 · c12bc61 · c12bc61
1 parent d58a5eb
commit c12bc61
Show file tree

Hide file tree

Showing 8 changed files with 110 additions and 61 deletions.
diff --git a/.github/workflows/operator_ci_index_sha.yaml b/.github/workflows/operator_ci_index_sha.yaml
@@ -20,7 +20,8 @@ env:
   OPP_INDEX_MIRROR: 1
   OPP_MIRROR_LATEST_TAG: "latest"
   OPP_PRODUCTION_TYPE: "k8s"
-  OPP_MIRROR_INDEX_MULTIARCH: "registry.redhat.io/openshift4/ose-operator-registry:v4.5"
+  OPP_MIRROR_INDEX_MULTIARCH_BASE: "registry.redhat.io/openshift4/ose-operator-registry:v4.5"
+  OPP_MULTIARCH_SUPPORTED_VERSIONS: "v4.5 v4.6 v4.7"
   OPP_MIRROR_INDEX_MULTIARCH_POSTFIX: "s"
   OPP_MIRROR_INDEX_REGISTRY: "quay.io"
   OPP_MIRROR_INDEX_ORGANIZATION: "operatorhubio"

diff --git a/.github/workflows/operator_release.yaml b/.github/workflows/operator_release.yaml
@@ -12,6 +12,10 @@ on:
         description: 'Force to run release'
         required: true
         default: '0'
+      delete:
+        description: 'Delete operator (cockroachdb or cockroachdb/5.0.4) for debuging only'
+        required: false
+        default: ''
 
 env:
   OPP_DEBUG: 1
@@ -34,7 +38,8 @@ env:
   OPP_MIRROR_INDEX_ORGANIZATION: "operatorhubio"
   OPP_MIRROR_INDEX_NAME: "catalog_prod"
   # OPP_MIRROR_INDEX_ENABLED: "0"
-  OPP_MIRROR_INDEX_MULTIARCH: "registry.redhat.io/openshift4/ose-operator-registry:v4.5"
+  OPP_MIRROR_INDEX_MULTIARCH_BASE: "registry.redhat.io/openshift4/ose-operator-registry:v4.5"
+  OPP_MULTIARCH_SUPPORTED_VERSIONS: "v4.5 v4.6 v4.7"
   OPP_MIRROR_INDEX_MULTIARCH_POSTFIX: "s"
   IIB_INPUT_REGISTRY_USER: "jbreza"
   OPP_REGISTRY_MIRROR_USER: "framework_automation"
@@ -45,15 +50,12 @@ env:
 
 
   OPP_ALLOW_FORCE_RELEASE: ${{ github.event.inputs.release }}
+  OPP_REMOVE_OPERATOR_AFTER_CLONE_PATH: "${{ github.event.inputs.delete }}"
   REGISTRY_RELEASE_API_TOKEN: ${{ secrets.REGISTRY_RELEASE_API_TOKEN }}
   REGISTRY_MIRROR_PW: ${{ secrets.REGISTRY_MIRROR_PW }}
-
   GODEBUG: x509ignoreCN=0
   CI_OHIO_TRIGGER_TOKEN: ${{ secrets.CI_OHIO_TRIGGER_TOKEN }}
   OHIO_REGISTRY_TOKEN: ${{ secrets.OHIO_REGISTRY_TOKEN }}
-
-  #QUAY_API_TOKEN_OPENSHIFT_COMMUNITY_OP: ${{ secrets.QUAY_API_TOKEN_OPENSHIFT_COMMUNITY_OP }}
-  #QUAY_API_TOKEN_OPERATORHUBIO: ${{ secrets.QUAY_API_TOKEN_OPERATORHUBIO }}
   QUAY_APPREG_TOKEN: ${{ secrets.QUAY_APPREG_TOKEN }}
   QUAY_COURIER_TOKEN: ${{ secrets.QUAY_COURIER_TOKEN }}
   REPO_GHA_PAT: ${{ secrets.REPO_GHA_PAT }}
@@ -62,7 +64,6 @@ jobs:
   pr-check:
     name: "PR-traffic-light"
     runs-on: ubuntu-latest
-
     steps:
       - name: Block until there is no running workflow
         uses: softprops/turnstyle@v1
@@ -83,7 +84,7 @@ jobs:
           python-version: '3.x'
       - name: Install dependencies
         run: python -m pip install --upgrade pip yq
-      - name: Operator test
+      - name: Operator traffic light
         id: op-traffic-light
         env:
           OPP_ADDED_MODIFIED_FILES: "${{ steps.files.outputs.added_modified }}"
@@ -110,89 +111,127 @@ jobs:
       opp_release_delete_appreg: "${{ steps.op-traffic-light.outputs.opp_release_delete_appreg }}"
       opp_pr_title: "${{ steps.op-traffic-light.outputs.opp_pr_title }}"
 
-  release-k8s_latest:
-    name: "Release / k8s latest"
+  remove-k8s:
+    name: "Remove / k8s"
+
     needs: pr-check
     if: needs.pr-check.outputs.opp_release_ready == '1'
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
       - name: Remove operator from index
-        if: needs.pr-check.outputs.opp_op_delete == '1'
+        if: needs.pr-check.outputs.opp_op_delete == '1' && needs.pr-check.outputs.opp_name != ''
         env:
           OPP_LABELS: "${{ needs.pr-check.outputs.opp_labels }}"
         run: |
           echo "OPP_LABELS=$OPP_LABELS"
           echo "op_delete operators/${{ needs.pr-check.outputs.opp_name }}/dummy $OPP_THIS_REPO $OPP_THIS_BRANCH"
           bash <(curl -sL $OPP_SCRIPT_URL) op_delete operators/${{ needs.pr-check.outputs.opp_name }}/dummy $OPP_THIS_REPO $OPP_THIS_BRANCH
-      - name: Index recreate only (Only ci.yaml was changed)
-        if: needs.pr-check.outputs.opp_ci_yaml_only == '1'
+      - name: Operator release (operator version overwrite)
+        if: needs.pr-check.outputs.opp_ci_yaml_only == '0' && needs.pr-check.outputs.opp_ver_overwrite == '1' && needs.pr-check.outputs.opp_name != '' && needs.pr-check.outputs.opp_version != ''
         env:
           OPP_LABELS: "${{ needs.pr-check.outputs.opp_labels }}"
-          OPP_CI_YAML_ONLY: 1
+          OPP_IIB_INSTALL: 0
         run: |
           echo "OPP_LABELS=$OPP_LABELS"
-          echo "orange operators/${{ needs.pr-check.outputs.opp_name }}/sync $OPP_THIS_REPO $OPP_THIS_BRANCH"
-          bash <(curl -sL $OPP_SCRIPT_URL) orange operators/${{ needs.pr-check.outputs.opp_name }}/sync $OPP_THIS_REPO $OPP_THIS_BRANCH
-      - name: Operator release (sync)
-        if: needs.pr-check.outputs.opp_ci_yaml_only == '0' && needs.pr-check.outputs.opp_ver_overwrite == '0'
+          echo "op_delete operators/${{ needs.pr-check.outputs.opp_name }}/${{ needs.pr-check.outputs.opp_version }} $OPP_THIS_REPO $OPP_THIS_BRANCH"
+          bash <(curl -sL $OPP_SCRIPT_URL) op_delete operators/${{ needs.pr-check.outputs.opp_name }}/${{ needs.pr-check.outputs.opp_version }} $OPP_THIS_REPO $OPP_THIS_BRANCH
+
+  operators-k8s:
+    name: "Index check / k8s"
+    needs: [pr-check, remove-k8s ]
+
+    if: needs.pr-check.outputs.opp_release_ready == '1'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Index check
+        id: operators
         env:
           OPP_LABELS: "${{ needs.pr-check.outputs.opp_labels }}"
+          OPP_INDEX_CHECK_ONLY: 1
         run: |
           echo "OPP_LABELS=$OPP_LABELS"
+          echo "'orange' operators/sync $OPP_THIS_REPO $OPP_THIS_BRANCH"
+          bash <(curl -sL $OPP_SCRIPT_URL) orange operators/sync $OPP_THIS_REPO $OPP_THIS_BRANCH
+      - name: Operators for sync
+        run: |
+          echo "Operators : ${{ steps.operators.outputs.opp_uncomplete_operators }}"
+    outputs:
+      opp_uncomplete_operators: "${{ steps.operators.outputs.opp_uncomplete_operators }}"
+
+  bundles-k8s:
+    name: "Bundles / k8s"
+    needs: [ pr-check, remove-k8s, operators-k8s ]
+    if: needs.pr-check.outputs.opp_release_ready == '1' && needs.operators-k8s.outputs.opp_uncomplete_operators != ''
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Build bundles
+        env:
+          OPP_LABELS: "${{ needs.pr-check.outputs.opp_labels }}"
+          OPP_SKIP_INDEX: 1
+          OPP_FORCE_OPERATORS: "${{ needs.operators-k8s.outputs.opp_uncomplete_operators }}"
+
+        run: |
+          echo "OPP_LABELS=$OPP_LABELS"
+          echo "OPP_FORCE_OPERATORS=$OPP_FORCE_OPERATORS"
           echo "orange operators/sync $OPP_THIS_REPO $OPP_THIS_BRANCH"
           bash <(curl -sL $OPP_SCRIPT_URL) orange operators/sync $OPP_THIS_REPO $OPP_THIS_BRANCH
-      - name: Operator release (operator version overwrite)
-        if: needs.pr-check.outputs.opp_ci_yaml_only == '0' && needs.pr-check.outputs.opp_ver_overwrite == '1'
+
+  release-k8s:
+    name: "Index / k8s"
+    needs: [ pr-check, operators-k8s, bundles-k8s ]
+    if: needs.pr-check.outputs.opp_release_ready == '1' && needs.operators-k8s.outputs.opp_uncomplete_operators != ''
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Index sync
         env:
           OPP_LABELS: "${{ needs.pr-check.outputs.opp_labels }}"
+          OPP_FORCE_INDEX_UPDATE: 1
+          OPP_FORCE_OPERATORS: "${{ needs.operators-k8s.outputs.opp_uncomplete_operators }}"
+
         run: |
           echo "OPP_LABELS=$OPP_LABELS"
-          echo "orange operators/${{ needs.pr-check.outputs.opp_name }}/${{ needs.pr-check.outputs.opp_version }} $OPP_THIS_REPO $OPP_THIS_BRANCH"
-          bash <(curl -sL $OPP_SCRIPT_URL) orange operators/${{ needs.pr-check.outputs.opp_name }}/${{ needs.pr-check.outputs.opp_version }} $OPP_THIS_REPO $OPP_THIS_BRANCH
+          echo "orange operators/sync $OPP_THIS_REPO $OPP_THIS_BRANCH"
+          bash <(curl -sL $OPP_SCRIPT_URL) orange operators/sync $OPP_THIS_REPO $OPP_THIS_BRANCH
 
 
   ohio-image-k8s:
-    name: "Release / k8s ohio image"
-    needs: [ pr-check, release-k8s_latest ]
-    # if: needs.pr-check.outputs.opp_release_ready == '1'
+    name: "OHIO image / k8s"
+    needs: [ pr-check, release-k8s ]
+
+    if: needs.pr-check.outputs.opp_release_ready == '1'
     runs-on: ubuntu-latest
     env:
       OHIO_INPUT_CATALOG_IMAGE: "quay.io/operatorhubio/catalog:latest"
       OHIO_REGISTRY_IMAGE: "quay.io/operator-framework/upstream-community-operators:latest"
 
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
       - name: Ohio image
         run: |
           echo "ohio_image operators/sync $OPP_THIS_REPO $OPP_THIS_BRANCH"
           bash <(curl -sL $OPP_SCRIPT_CLEANUP_URL)
           bash <(curl -sL $OPP_SCRIPT_URL) ohio_image operators/sync $OPP_THIS_REPO $OPP_THIS_BRANCH
-
   ohio-page-k8s:
     name: "Release / operatorhub.io"
     needs: ohio-image-k8s
-    # if: needs.pr-check.outputs.opp_release_ready == '1'
+
+    if: needs.pr-check.outputs.opp_release_ready == '1'
     runs-on: ubuntu-latest
     env:
       CI_OHIO_BRANCH: "latest"
       REPO_GHA_PAT: ${{ secrets.REPO_GHA_PAT }}
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
       - name: Ohio image
         run: |
           echo "$OPP_SCRIPT_TRIGGER_OHIO_URL ${CI_OHIO_BRANCH}"
           bash <(curl -sL $OPP_SCRIPT_TRIGGER_OHIO_URL) ${CI_OHIO_BRANCH}
 
   slack-notification:
     name: "Slack notification"
+    needs: [ pr-check, remove-k8s, operators-k8s, release-k8s, ohio-image-k8s, ohio-page-k8s ]
 
 
-    needs: [ pr-check, release-k8s_latest, ohio-image-k8s, ohio-page-k8s ]
-
     if: failure()
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/operator_test.yaml b/.github/workflows/operator_test.yaml
@@ -25,7 +25,7 @@ env:
   OPP_RELEASE_INDEX_REGISTRY: "quay.io"
   OPP_RELEASE_INDEX_ORGANIZATION: "operatorhubio"
   OPP_RELEASE_INDEX_NAME: "catalog"
-  OPP_MIRROR_INDEX_MULTIARCH: "registry.redhat.io/openshift4/ose-operator-registry:v4.5"
+  OPP_MIRROR_INDEX_MULTIARCH_BASE: "registry.redhat.io/openshift4/ose-operator-registry:v4.5"
   OPP_MIRROR_INDEX_MULTIARCH_POSTFIX: "s"
   IIB_INPUT_REGISTRY_USER: "jbreza"
   OPP_PROD: 0

diff --git a/.github/workflows/upgrade.yaml b/.github/workflows/upgrade.yaml
@@ -15,14 +15,20 @@ on:
         description: 'Source branch'     
         required: true
         default: 'ci/latest'
+      cluster:
+        description: 'Cluster type (k8s, ocp)'
+        required: true
+        default: 'k8s'
+
+
 env:
   OPP_INPUT_REPO: "${{ github.event.inputs.repo }}"
   OPP_INPUT_BRANCH: "${{ github.event.inputs.branch }}"
   OPP_SCRIPT_UPGRADE_URL: "https://raw.githubusercontent.com/redhat-openshift-ecosystem/community-operators-pipeline/ci/latest/ci/scripts/opp-update.sh"
 
 
 jobs:
-  build:
+  upgrade:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -31,8 +37,11 @@ jobs:
         #persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
         # fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
     - name: Create local changes
+      env:
+        CLUSTER_TYPE: ${{ github.event.inputs.cluster }}
       run: |
-        bash <(curl -sL $OPP_SCRIPT_UPGRADE_URL)
+        echo "$OPP_INPUT_REPO $OPP_INPUT_BRANCH"
+        bash <(curl -sL $OPP_SCRIPT_UPGRADE_URL) $CLUSTER_TYPE
     - name: Commit files
       run: |
         git config --local user.email "41898282+framework-automation[bot]@users.noreply.github.com"

diff --git a/scripts/ci/Dockerfile.ci-operator → ci/prow/Dockerfile.ci-operator b/scripts/ci/Dockerfile.ci-operator → ci/prow/Dockerfile.ci-operator
diff --git a/scripts/ci/openshift-deploy-core.sh → ci/prow/openshift-deploy-core.sh b/scripts/ci/openshift-deploy-core.sh → ci/prow/openshift-deploy-core.sh
@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
-#./openshift-deploy.sh test-only https://github.com/J0zi/community-operators.git bundle2 https://github.com/J0zi/operator-test-playbooks.git CVP-1793-exit-non-relevant-ocp-test
+#./openshift-deploy.sh test-only https://github.com/J0zi/community-operators-pipeline.git do-not-delete-rehearsals https://github.com/redhat-openshift-ecosystem/operator-test-playbooks.git upstream-community 77 rehearsal
 
 set -e #fail in case of non zero return
+PLAYBOOK_REPO='https://github.com/redhat-openshift-ecosystem/operator-test-playbooks.git'
+PLAYBOOK_REPO_BRANCH='upstream-community'
 echo "OCP_CLUSTER_VERSION_SUFFIX=$OCP_CLUSTER_VERSION_SUFFIX"
 
 JQ_VERSION='1.6'
@@ -13,17 +15,15 @@ OC_DIR_CORE=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1)
 SUBDIR_ARG="-e work_subdir_name=oc-$OC_DIR_CORE"
 echo "SUBDIR_ARG = $SUBDIR_ARG"
 
-if [[ $TEST_MODE -ne 1  ]]; then
-    #CURRENT_PATH=/go/src/github.com/redhat-openshift-ecosystem/community-operators-pipeline/scripts/ci
-    CURRENT_PATH=$(pwd)
-    if [ $(echo $CURRENT_PATH|grep operator-framework)  ]; then
-         	TARGET_PATH=$(echo $CURRENT_PATH|sed -e 's/scripts\/ci/community-operators/g')
-    else
-                TARGET_PATH=$(echo $CURRENT_PATH|sed -e 's/scripts\/ci/operators/g')
-    fi
-fi
-echo "TARGET_PATH=$TARGET_PATH"
+# prod or test in rehearsal mode
+TARGET_PATH="$(dirname $(dirname $(dirname $(readlink -m $0))))/operators"
 export PR_TARGET_REPO=$(echo $TARGET_PATH|cut -d"/" -f 5-6)
+
+#local test only
+[[ $TEST_MODE -eq 1 ]] && TARGET_PATH='/tmp/oper-for-me-test/community-operators/operators'
+[[ $TEST_MODE -eq 1  ]] && [[ $REHEARSAL -ne 1 ]] && export PR_TARGET_REPO='redhat-openshift-ecosystem/community-operators-pipeline'
+
+echo "TARGET_PATH=$TARGET_PATH"
 echo "PR_TARGET_REPO=$PR_TARGET_REPO"
 
 #label start
@@ -32,23 +32,17 @@ echo "PR_TARGET_REPO=$PR_TARGET_REPO"
 -H "Accept: application/vnd.github.v3+json" \
 "https://api.github.com/repos/$PR_TARGET_REPO/dispatches" --data "{\"event_type\": \"openshift-test-status\", \"client_payload\": {\"source_pr\": \"$PULL_NUMBER\", \"remove_labels\": [\"installation-validated$OCP_CLUSTER_VERSION_SUFFIX\", \"openshift-started$OCP_CLUSTER_VERSION_SUFFIX\", \"installation-failed$OCP_CLUSTER_VERSION_SUFFIX\", \"installation-validated\"], \"add_labels\": [\"openshift-started$OCP_CLUSTER_VERSION_SUFFIX\"]}}"
 
-echo "OS"
-cat /etc/os-release
-
 pwd
 
 #[[ $TEST_MODE -ne 1 ]] && TARGET_PATH='/go/src/github.com/operator-framework/community-operators/community-operators'
 
-
-[[ $TEST_MODE -eq 1 ]] && TARGET_PATH='/tmp/oper-for-me-test/community-operators/community-operators'
-
 #temp test for development to test on a stable commit
 if [[ $TEST_MODE -eq 1 ]]; then
   echo "Need to clone test branch, cloning..."
   if [ -d /tmp/oper-for-me-test ]; then rm -Rf /tmp/oper-for-me-test; fi
   mkdir -p /tmp/oper-for-me-test
   cd /tmp/oper-for-me-test
-  git clone $TEST_COMMUNITY_REPO
+  git clone $TEST_COMMUNITY_REPO community-operators
   cd community-operators
   git checkout $TEST_COMMUNITY_BRANCH
   ls
@@ -74,7 +68,9 @@ chmod +x "/tmp/jq-$OC_DIR_CORE/bin/jq" && echo "Rights adjusted"
 -H "Accept: application/vnd.github.v3+json" \
 "https://api.github.com/repos/$PR_TARGET_REPO/issues/$PULL_NUMBER"|"/tmp/jq-$OC_DIR_CORE/bin/jq" '.labels[].name'|grep 'allow/longer-deployment' \
 && echo "Longer deployment detected" && EXTRA_ARGS='-e pod_start_retries=300'
+
 cd "$TARGET_PATH"
+echo "Checking PR $PULL_NUMBER on $PR_TARGET_REPO"
 tmpfile=$(mktemp /tmp/pr-details-XXXXXXX.json)
 curl -s https://api.github.com/repos/$PR_TARGET_REPO/pulls/$PULL_NUMBER -o $tmpfile
 cat $tmpfile
@@ -98,7 +94,7 @@ export OPRT=1
 echo "OPRT values set [OK]"
 [ -n "$OPRT_REPO" ] || { echo "Error: '\$OPRT_REPO' is empty !!!"; exit 1; }
 [ -n "$OPRT_SHA" ] || { echo "Error: '\$OPRT_SHA' is empty !!!"; exit 1; }
-echo "Going to clone"
+echo "Going to clone $REPO_FULL"
 git clone $REPO_FULL community-operators #> /dev/null 2>&1
 echo "Cloned  [OK]"
 cd community-operators
@@ -203,10 +199,12 @@ if [ -d /tmp/playbooks2 ]; then rm -Rf /tmp/playbooks2; fi
 mkdir -p /tmp/playbooks2
 cd /tmp/playbooks2
 echo "We are in dir"
-[[ $TEST_MODE -ne 1 ]] && git clone https://github.com/operator-framework/operator-test-playbooks.git
-[[ $TEST_MODE -eq 1 ]] && git clone $TEST_PB_REPO
+
+[[ $TEST_MODE -ne 1 ]] && git clone $PLAYBOOK_REPO operator-test-playbooks
+[[ $TEST_MODE -eq 1 ]] && git clone $TEST_PB_REPO operator-test-playbooks
 cd operator-test-playbooks
 [[ $TEST_MODE -eq 1 ]] && git checkout $TEST_PB_BRANCH
+[[ $TEST_MODE -ne 1 ]] && git checkout $PLAYBOOK_REPO_BRANCH
 cd upstream
 echo "Config ..."
 export ANSIBLE_CONFIG=/tmp/playbooks2/operator-test-playbooks/upstream/ansible.cfg

diff --git a/scripts/ci/openshift-deploy.sh → ci/prow/openshift-deploy.sh b/scripts/ci/openshift-deploy.sh → ci/prow/openshift-deploy.sh
@@ -1,16 +1,17 @@
 #!/usr/bin/env bash
-#This scripts is configured in https://github.com/openshift/release/tree/master/ci-operator/config/operator-framework/community-operators and executed from ci-operator/jobs
+#This scripts is configured in https://github.com/openshift/release/tree/master/ci-operator/config/redhat-openshift-ecosystem/community-operators-prod and executed from ci-operator/jobs
 if [ -z "$1" ]; then echo "Production mode"; else
   if [ $1 = "test-only" ]; then
     echo "Running in test mode"
     export TEST_MODE=1
-    if [ -z "${2+xxx}" ]; then echo "Please provide test repo and branch for community operators 'test-only https://github.com/operator-framework/community-operators.git community_branch_name https://github.com/operator-framework/operator-test-playbooks.git playbook_branch_name 3966'"
+    if [ -z "${2+xxx}" ]; then echo "Please provide test repo and branch for community operators 'test-only https://github.com/redhat-openshift-ecosystem/community-operators-prod.git community_branch_name https://github.com/redhat-openshift-ecosystem/operator-test-playbooks.git playbook_branch_name 3966'"
     else
       export TEST_COMMUNITY_REPO=$2
       export TEST_COMMUNITY_BRANCH=$3
       export TEST_PB_REPO=$4
       export TEST_PB_BRANCH=$5
       export PULL_NUMBER=$6
+      if [ $7 = "rehearsal" ]; then echo "Rehearsal mode detected..."; export REHEARSAL=1;fi
     fi
   fi
 fi

diff --git a/scripts/ci b/scripts/ci
@@ -0,0 +1 @@
+../ci/prow