Merge pull request #768 from apigee/feature/fix-go-pipeline

feat: base pipeline builder off alpine

Author: danistrebel

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,8 +12,19 @@ What's changed, or what was fixed?
 - [ ] I have run all the tests locally and they all pass.
 - [ ] I have followed the relevant style guide for my changes.
 
-## Full Repo Validation Required
-(please check all that apply [x], do not edit the text)
+## ADVANCED Settings
+
+For most PRs you most likely do not have to edit anything in this section.
+
+### Test Scope
+
+By default the PR pipeline tests all projects that were edited in the PR.
+
+Some PRs might have side-effects on other projects such as changes to the test pipeline or shared tooling and need tests beyond the projects edited in this PR.
+
+To customize the PR testing scope check [x] either option below and replace the sub-projects as needed.
+
 - [ ] PR requires full pipeline run (Run for changes only by default).
+- [ ] Test following projects (comma separated list): references/cicd-pipeline,tools/apigee-sackmesser
 
 **CC:** @apigee-devrel-reviewers

references/common-shared-flows/error-handling-v1/sharedflowbundle/resources/jsc/PopulateDefaultErrorVariables.js

@@ -98,6 +98,6 @@ switch (context.getVariable("fault.name")) {
     // }
 }
 
-if (context.getVariable("flow.error.code") == null || !context.getVariable("custom.error.code")) {
+if (!context.getVariable("custom.error.code")) {
   setFault(500, "Internal Server Error", "500.99", "Internal Server Error");
 }

references/identity-facade/test/integration/features/step_definitions/ui.js

@@ -36,7 +36,7 @@ Given('I navigate to the authorize page', async function() {
   this.browser = await puppeteer.launch({
     ignoreHTTPSErrors: true,
     headless: true,
-    args: ["--no-sandbox"]
+    args: ["--no-sandbox", "--disable-gpu"]
   })
   this.page = await this.browser.newPage()
   return await this.page.goto('https://' + hostname + basePath + '/authorize?client_id=' + this.apickli.scenarioVariables.clientId
@@ -49,7 +49,7 @@ Given('I navigate to the authorize page with query params that must be sanitized
   this.browser = await puppeteer.launch({
     ignoreHTTPSErrors: true,
     headless: true,
-    args: ["--no-sandbox"]
+    args: ["--no-sandbox", "--disable-gpu"]
   })
   this.page = await this.browser.newPage()
   return await this.page.goto('https://' + hostname + basePath + '/authorize?client_id=' + '    ' + this.apickli.scenarioVariables.clientId
@@ -62,7 +62,7 @@ Given('I navigate to the authorize page with an invalid response type', async fu
   this.browser = await puppeteer.launch({
     ignoreHTTPSErrors: true,
     headless: true,
-    args: ["--no-sandbox"]
+    args: ["--no-sandbox", "--disable-gpu"]
   })
   this.page = await this.browser.newPage()
   return await this.page.goto('https://' + hostname + basePath + '/authorize?client_id=' + this.apickli.scenarioVariables.clientId
@@ -75,7 +75,7 @@ Given('I navigate to the authorize page without a scope parameter', async functi
   this.browser = await puppeteer.launch({
     ignoreHTTPSErrors: true,
     headless: true,
-    args: ["--no-sandbox"]
+    args: ["--no-sandbox", "--disable-gpu"]
   })
   this.page = await this.browser.newPage()
   return await this.page.goto('https://' + hostname + basePath + '/authorize?client_id=' + this.apickli.scenarioVariables.clientId
@@ -88,7 +88,7 @@ Given('I navigate to the authorize page without a state parameter', async functi
   this.browser = await puppeteer.launch({
     ignoreHTTPSErrors: true,
     headless: true,
-    args: ["--no-sandbox"]
+    args: ["--no-sandbox", "--disable-gpu"]
   })
   this.page = await this.browser.newPage()
   return await this.page.goto('https://' + hostname + basePath + '/authorize?client_id=' + this.apickli.scenarioVariables.clientId
@@ -102,7 +102,7 @@ if (isPkceEnabled) {
     this.browser = await puppeteer.launch({
       ignoreHTTPSErrors: true,
       headless: true,
-      args: ["--no-sandbox"]
+      args: ["--no-sandbox", "--disable-gpu"]
     })
     this.page = await this.browser.newPage()
     return await this.page.goto('https://' + hostname + basePath + '/authorize?client_id=' + this.apickli.scenarioVariables.clientId
@@ -114,7 +114,7 @@ if (isPkceEnabled) {
     this.browser = await puppeteer.launch({
       ignoreHTTPSErrors: true,
       headless: true,
-      args: ["--no-sandbox"]
+      args: ["--no-sandbox", "--disable-gpu"]
     })
     this.page = await this.browser.newPage()
     return await this.page.goto('https://' + hostname + basePath + '/authorize?client_id=' + this.apickli.scenarioVariables.clientId
@@ -126,7 +126,7 @@ if (isPkceEnabled) {
     this.browser = await puppeteer.launch({
       ignoreHTTPSErrors: true,
       headless: true,
-      args: ["--no-sandbox"]
+      args: ["--no-sandbox", "--disable-gpu"]
     })
     this.page = await this.browser.newPage()
     return await this.page.goto('https://' + hostname + basePath + '/authorize?client_id=' + this.apickli.scenarioVariables.clientId

references/oidc-mock/test/integration/features/step_definitions/ui.js

@@ -31,7 +31,7 @@ Given('I navigate to the authorize page', async function() {
   this.browser = await puppeteer.launch({
     ignoreHTTPSErrors: true,
     headless: true,
-    args: ["--no-sandbox"]
+    args: ["--no-sandbox", "--disable-gpu"]
   })
   this.page = await this.browser.newPage()
   return await this.page.goto('https://' + hostname + '/v1/openid-connect/authorize?client_id=' + this.apickli.scenarioVariables.clientId
@@ -42,7 +42,7 @@ Given('I navigate to the authorize page with an invalid response type', async fu
   this.browser = await puppeteer.launch({
     ignoreHTTPSErrors: true,
     headless: true,
-    args: ["--no-sandbox"]
+    args: ["--no-sandbox", "--disable-gpu"]
   })
   this.page = await this.browser.newPage()
   return await this.page.goto('https://' + hostname + '/v1/openid-connect/authorize?client_id=' + this.apickli.scenarioVariables.clientId
@@ -53,7 +53,7 @@ Given('I navigate to the authorize page without a scope parameter', async functi
   this.browser = await puppeteer.launch({
     ignoreHTTPSErrors: true,
     headless: true,
-    args: ["--no-sandbox"]
+    args: ["--no-sandbox", "--disable-gpu"]
   })
   this.page = await this.browser.newPage()
   return await this.page.goto('https://' + hostname + '/v1/openid-connect/authorize?client_id=' + this.apickli.scenarioVariables.clientId
@@ -64,7 +64,7 @@ Given('I navigate to the authorize page without a state parameter', async functi
   this.browser = await puppeteer.launch({
     ignoreHTTPSErrors: true,
     headless: true,
-    args: ["--no-sandbox"]
+    args: ["--no-sandbox", "--disable-gpu"]
   })
   this.page = await this.browser.newPage()
   return await this.page.goto('https://' + hostname + '/v1/openid-connect/authorize?client_id=' + this.apickli.scenarioVariables.clientId

references/openapi-mock/README.md

@@ -60,7 +60,7 @@ apigeecli apis create bundle \
 
 If you just need to quickly generate a mock API proxy from your OpenAPI 3 spec, the [apigee-go-gen](https://apigee.github.io/apigee-go-gen/installation/) tool can help. 
 
-The tool's [mock oas](https://apigee.github.io/apigee-go-gen/mock/mock-openapi-spec/) command automates the process, saving you time and effort.
+The tool's [mock oas](https://apigee.github.io/apigee-go-gen/mock/mock-openapi-description/) command automates the process, saving you time and effort.
 
 Here is an example of how to generate a mock API proxy using the `apigee-go-gen` tool.
 

tools/endpoints-oas-importer/import-endpoints.sh

@@ -131,7 +131,7 @@ function client_authentication() {
 
 # normalizing yaml and json OAS file types
 if [[ $oas == *.yaml || $oas == *.yml ]]; then
-  oas_json_content=$(yq -o json "$oas")
+  oas_json_content=$(yq e -o json "$oas")
 elif [[ $oas == *.json ]]; then
   oas_json_content=$(cat "$oas")
 else

tools/pipeline-runner/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,10 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM google/cloud-sdk:alpine
+FROM google/cloud-sdk:alpine AS gcloud-sdk
+
+FROM alpine:3.21
+
+COPY --from=gcloud-sdk /google-cloud-sdk /google-cloud-sdk
+ENV PATH="/google-cloud-sdk/bin:${PATH}"
 
 # Install additional tools
 RUN apk add --no-cache \
+  bash \
+  docker \
+  git \
+  curl \
   jq \
   libxml2-utils \
   maven \
@@ -38,7 +47,9 @@ RUN apk add --no-cache \
   zip \
   make \
   go \
-  protobuf-dev
+  yq \
+  protobuf-dev \
+  dbus
 
 # Reduce nighly log (note: -ntp requires maven 3.6.1+)
 RUN mv /usr/bin/mvn /usr/bin/_mvn &&\
@@ -52,18 +63,11 @@ RUN chmod +x /usr/local/bin/claat
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
   PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
 
-RUN npm install --global [email protected]
+RUN npm install --global [email protected] [email protected]
 
 # add our tooling scripts
 COPY *.sh /usr/bin/
 
-# install apgieelint
-RUN npm install --global [email protected]
-
-# yq because the apk is too old
-RUN wget -q -O /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/v4.29.2/yq_linux_amd64"
-RUN chmod +x /usr/local/bin/yq
-
 # Run script
 WORKDIR /home
 CMD ["run-pipelines.sh"]

tools/pipeline-runner/cloudbuild.yaml

@@ -56,19 +56,37 @@ steps:
     args:
       - '-c'
       - |-
+        set -e
         max_duration=8760 # cloud build timeout minus 4m
         if [ -n "$_PR_NUMBER" ]; then
-          FULL_PIPELINE_REQUIRED=$(curl "https://api.github.com/repos/$_REPO_GH_ISSUE/issues/$_PR_NUMBER" | jq '.body |= ascii_downcase | .body | contains("[x] pr requires full pipeline run")' )
-          PROJECTS=$(list-repo-changes.sh)
+          PR_DATA="$(curl "https://api.github.com/repos/$_REPO_GH_ISSUE/issues/$_PR_NUMBER")"
+          FULL_PIPELINE_REQUIRED=$(printf '%s' "$$PR_DATA" | jq '.body |= ascii_downcase | .body | contains("[x] pr requires full pipeline run")' )
+          SELECTED_PROJECTS_FLAG=$(printf '%s' "$$PR_DATA" | jq '.body |= ascii_downcase | .body | contains("[x] test following projects (comma separated list):")')
+
+          # get a comma-separated list of folders of changed sub-projects
+          CHANGED_PROJECTS=$(list-repo-changes.sh)
+
+          # test only explicity subset of projects
+          if [ "$$SELECTED_PROJECTS_FLAG" = "true" ]; then
+            PROJECTS=$(printf "$$PR_DATA" | grep -o -i -E "test following projects \(comma separated list\): *.*" | cut -d ':' -f 2 | tr -d '[:space:]')
+            if [ -z "$$PROJECTS" ]; then
+              echo "SELECTED PROJECTS is empty. Please provide a list"
+              PROJECTS="$$CHANGED_PROJECTS"
+            fi
+          else
+            PROJECTS=$$CHANGED_PROJECTS
+          fi
+
           if [ "$$FULL_PIPELINE_REQUIRED" = "true" ]; then
             echo "Running Full Pipeline as required in the PR"
             timeout "$$max_duration" run-pipelines.sh || true
-          elif [ -z "$$PROJECTS"]; then
+          elif [ -z "$$PROJECTS" ]; then
             echo "TOTAL PIPELINE (no change);pass;0" > ./pipeline-result.txt
           else
-            echo "PR includes changes in $$PROJECTS"
+            echo "PR testing changes in the following projects: $$PROJECTS"
             timeout "$$max_duration" run-pipelines.sh "$$PROJECTS" || true
           fi
+
         elif [ "$_CI_PROJECT" = "all" ]; then
           timeout "$$max_duration" run-pipelines.sh || true
         else

tools/pipeline-runner/github-api.sh

@@ -18,7 +18,8 @@
 
 buildresult=$(cat)
 
-REPORT_ROWS=$(echo "$buildresult" | awk -F";" '$0="|"$1"|"$2"|"$3"|"' OFS="|")
+REPORT_ROWS=$(echo "$buildresult" | awk -F\; -v OFS='|' '{gsub(/pass/, "✅"); gsub(/fail/, "❌"); print "|" $1 "|" $2 "|" $3 "|"}')
+
 REPORT=$(cat <<EOF
 ### Pipeline Report