DevRel CI based on Cloud Build

Author: danistrebel

.gcloudignore

@@ -0,0 +1,10 @@
+**/node_modules/**
+**/generated/**
+**/target/**
+/tools/hybrid-quickstart/**/tools
+/service-accounts
+.settings
+.project
+.classpath
+.vscode
+.git

.github/workflows/devrel-full.yml .github/workflows/devrel-linter.yml

@@ -12,18 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: DevRel All Projects Pipeline
+name: DevRel Linter Pipeline
 on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
   workflow_dispatch:
   schedule:
   - cron: "0 0 * * *"
-env:
-  APIGEE_USER: ${{ secrets.APIGEE_USER }}
-  APIGEE_PASS: ${{ secrets.APIGEE_PASS }}
-  APIGEE_ORG: ${{ secrets.APIGEE_ORG }}
-  APIGEE_ENV: ${{ secrets.APIGEE_ENV }}
-  # Component specific secrets
-  REF_GCP_SA_SF: ${{ secrets.REF_GCP_SA_SF }}
 jobs:
 
   static-analysis:
@@ -44,36 +43,11 @@ jobs:
           MARKDOWN_CONFIG_FILE: .markdown-lint.yml
 
       - name: Check all files are licensed
-        uses: ./tools/pipeline-runner
+        uses: ./tools/pipeline-linter
         with:
           pipeline-action: check-license.sh
 
       - name: Check README and CODEOWNER index
-        uses: ./tools/pipeline-runner
+        uses: ./tools/pipeline-linter
         with:
-          pipeline-action: check-readme.sh
-
-  project-pipelines:
-
-    name: Run Pipelines
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v2
-
-      - name: Turnstyle
-        uses: softprops/turnstyle@v1
-        with:
-          same-branch-only: false
-          poll-interval-seconds: 10
-          continue-after-seconds: 120
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Run Pipelines
-        uses: ./tools/pipeline-runner
-        with:
-          pipeline-action: run-pipelines.sh
-
-
+          pipeline-action: check-readme.sh

.github/workflows/devrel-single.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v2
       - name: generate-docs
-        uses: ./tools/pipeline-runner
+        uses: ./tools/pipeline-linter
         with:
           pipeline-action: generate-all-docs.sh
       - name: Archive Generated Docs
@@ -50,6 +50,6 @@ jobs:
         run: |
           git config --global user.name "DevRel GH Pages Publish Action"
           git config --global user.email "[email protected]"
-          git add . 
+          git add .
           git commit -m "Update GH Pages - $GITHUB_SHA" || echo "No Changes in docs"
           git push

.github/workflows/devrel-site.yaml

@@ -1,5 +1,6 @@
 **/node_modules/**
 **/generated/**
+/service-accounts
 
 .settings
 .project

.gitignore

@@ -14,7 +14,8 @@
 /references/southbound-mtls           @danistrebel
 /labs/best-practices-hackathon        @laughingbiscuit @sauliuz @seymen
 /tools/organization-cleanup           @laughingbiscuit
-/tools/pipeline-runner                @laughingbiscuit @seymen
+/tools/pipeline-runner                @laughingbiscuit @seymen @danistrebel
+/tools/pipeline-linter                @laughingbiscuit @seymen @danistrebel
 /tools/another-apigee-client          @laughingbiscuit
 /tools/hybrid-quickstart              @danistrebel
 /tools/decrypt-hybrid-assets          @yuriylesyuk

CODEOWNERS

@@ -51,7 +51,9 @@ Apigee products.
   A tool to cleanup proxies in an Apigee organization, leveraging
   [Another Apigee Client](tools/another-apigee-client)
 - [Pipeline Runner](tools/pipeline-runner) -
-  A tool to lint, build and test groups of Apigee projects
+  A tool to build and test groups of Apigee projects
+- [Pipeline Linter](tools/pipeline-linter) -
+  A tool to lint groups of Apigee projects
 - [Another Apigee Client](tools/another-apigee-client) -
   A lightweight Apigee Management CLI
 - [Apigee hybrid Quickstart GKE](tools/hybrid-quickstart) -

README.md

@@ -15,7 +15,6 @@
 # limitations under the License.
 
 set -e
-set -x
 
 # TODO figure out how to move this to apigee/devrel repo
 # See https://github.com/apigee/devrel/issues/76

references/cicd-jenkins/pipeline.sh

@@ -15,7 +15,7 @@
 
 # REQUIREMENT:
 # Populate a service account key into the environment variable "REF_GCP_SA_SF" e.g. by:
-# $ REF_GCP_SA_SF=$(cat /path/to/gcp-sa-key.json | jq '. | tostring')
+# $ c
 #
 # Hint:
 # Make sure that newline characters are properly escaped with `\\n` within
@@ -34,9 +34,9 @@ deleteKVM() {
 #clean up if the KVM and create cache if not already exists
 deleteKVM 'gcp-sa-devrel'
 
-curl -XPOST -u "$APIGEE_USER:$APIGEE_PASS" "https://api.enterprise.apigee.com/v1/o/$APIGEE_ORG/e/$APIGEE_ENV/caches" \
+curl -XPOST -s -u "$APIGEE_USER:$APIGEE_PASS" "https://api.enterprise.apigee.com/v1/o/$APIGEE_ORG/e/$APIGEE_ENV/caches" \
   -H 'Content-Type: application/json; charset=utf-8' \
-  --data-binary @- << EOF
+  --data-binary @- > /dev/null << EOF
 {
   "name":"gcp-tokens",
   "description":"GCP service account tokens",
@@ -51,16 +51,16 @@ EOF
 #don't continue on failure
 set -e
 
-curl -XPOST -u "$APIGEE_USER:$APIGEE_PASS" "https://api.enterprise.apigee.com/v1/o/$APIGEE_ORG/e/$APIGEE_ENV/keyvaluemaps" \
+curl -XPOST -s -u "$APIGEE_USER:$APIGEE_PASS" "https://api.enterprise.apigee.com/v1/o/$APIGEE_ORG/e/$APIGEE_ENV/keyvaluemaps" \
   -H 'Content-Type: application/json; charset=utf-8' \
-  --data-binary @- << EOF
+  --data-binary @- > /dev/null << EOF
 {
   "name": "gcp-sa-devrel",
   "encrypted": "true",
   "entry": [
     {
       "name": "[email protected]",
-      "value": $REF_GCP_SA_SF
+      "value": $(echo "$REF_GCP_SA_SF" | jq '. | tostring')
     }
   ]
 }

references/gcp-sa-auth-shared-flow/pipeline.sh

@@ -19,5 +19,5 @@ In this example, the Java code is packaged as a jar, and as such has a separate
 export APIGEE_USER=xxx
 export APIGEE_PASS=xxx
 export APIGEE_ORG=xxx
-mvn clean install -Ptest
+mvn clean install -Ptest -ntp
 ```

references/java-callout/README.md

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 set -e
-set -x 
+set -x
 
-mvn install -Ptest -B
+mvn install -Ptest -B -ntp
 npm test --prefix proxy-v1

references/java-callout/pipeline.sh

@@ -17,6 +17,6 @@
 set -e
 set -x
 
-npm i
+npm i --no-fund
 npm run deploy
 npm test

references/js-callout/pipeline.sh

@@ -17,5 +17,5 @@
 set -e
 set -x
 
-mvn install
+mvn install -ntp -B
 npm run test

references/kvm-admin-api/pipeline.sh

@@ -16,7 +16,7 @@
 set -e
 
 # deploy Apigee API Proxy
-mvn install -P"$APIGEE_ENV" -Dapigee.config.options=update
+mvn install -ntp -P"$APIGEE_ENV" -Dapigee.config.options=update
 
 # execute integration tests
 npm install

references/oidc-mock/pipeline.sh

@@ -20,7 +20,7 @@ set -x
 mkdir -p ./tmp
 curl https://badssl.com/certs/badssl.com-client.p12 -o ./tmp/badssl.com-client.p12
 
-mvn install -Ptest -Dapigee.config.options=update 
+mvn install -ntp -B -Ptest -Dapigee.config.options=update
 
 npm i
 npm run test

references/southbound-mtls/pipeline.sh

@@ -0,0 +1,26 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# <http://www.apache.org/licenses/LICENSE-2.0>
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:alpine3.12
+
+RUN apk add --no-cache git
+RUN go get github.com/google/addlicense
+RUN export GO111MODULE=on && go get github.com/googlecodelabs/tools/[email protected]
+
+# install our dependencies
+COPY *.sh /usr/bin/
+
+# Run script
+WORKDIR /home
+CMD ["check-licenses.sh"]

tools/pipeline-linter/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: "Apigee DevRel Pipeline Runner"
+name: "Apigee DevRel Pipeline Linter"
 inputs:
   pipeline-action:
     required: true

tools/pipeline-runner/action.yml tools/pipeline-linter/action.yml

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e 
+set -e
 
 addlicense -check "$PWD"

tools/pipeline-runner/check-license.sh tools/pipeline-linter/check-license.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# Copyright 2020 Google LLC
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# <http://www.apache.org/licenses/LICENSE-2.0>
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "no pipeline required"