feat: release pipeline

concourse/pipeline/job_def.lib.yml

@@ -1,14 +1,26 @@
 #@ load("base.lib.yml", "add_res_by_conf", "add_res_by_name")
 #@ load("@ytt:template", "template")
 
+
+#@ def inter_bin_name(base_name, release_build):
+#@   if release_build:
+#@     return base_name + "_rel"
+#@   end
+#@
+#@   return base_name
+#@ end
+
 #! Use bin_gpdb_postfix="" to use a release version of gpdb binary
 #! Job config for centos7
 #@ def centos7_gpdb6_conf(release_build=False):
 res_test_image: centos7-gpdb6-image-test
 res_gpdb_bin: bin_gpdb6_centos7
 res_gpdb_src: gpdb6_src
 res_plpython3_bin: bin_plpython3_gpdb6_rhel7
+res_intermediates_bin: #@ inter_bin_name("bin_greenplumpython_rhel7_intermediates", release_build)
 os: rhel7
+release_build: #@ release_build
+release_bin: bin_greenplumpython_rhel7_release
 #@ end
 
 #! Job config for rhel8
@@ -17,7 +29,10 @@ res_test_image: rhel8-gpdb6-image-test
 res_gpdb_bin: bin_gpdb6_rhel8
 res_gpdb_src: gpdb6_src
 res_plpython3_bin: bin_plpython3_gpdb6_rhel8
+res_intermediates_bin: #@ inter_bin_name("bin_greenplumpython_rhel8_intermediates", release_build)
 os: rhel8
+release_build: #@ release_build
+release_bin: bin_greenplumpython_rhel8_release
 #@ end
 
 #! Job config for ubuntu18
@@ -26,7 +41,10 @@ res_test_image: ubuntu18-gpdb6-image-test
 res_gpdb_bin: bin_gpdb6_ubuntu18
 res_gpdb_src: gpdb6_src
 res_plpython3_bin: bin_plpython3_gpdb6_ubuntu18
+res_intermediates_bin: #@ inter_bin_name("bin_greenplumpython_ubuntu18_intermediates", release_build)
 os: ubuntu1804
+release_build: #@ release_build
+release_bin: bin_greenplumpython_ubuntu18_release
 #@ end
 
 #! The entry point of a pipeline. The job name must be 'entrance'.
@@ -83,20 +101,22 @@ plan:
 #@   end
 #@ end
 
-#@ def _test_task(conf):
-task: #@ "test_" + conf["os"]
+#@ def _test_and_build_task(conf):
+task: #@ "test_and_build" + conf["os"]
 timeout: 2h
 config:
   platform: linux
   run:
     path: greenplumpython_src/concourse/scripts/entry.sh
     args:
-      - test
+      - test_and_build
   inputs:
     - name: bin_gpdb
     - name: gpdb_src
     - name: greenplumpython_src
     - name: bin_plpython3
+  outputs:
+    - name: greenplumpython_artifacts
 image: image_test
 params:
 #@ end
@@ -128,5 +148,47 @@ plan:
     resource: #@ conf["res_gpdb_src"]
   - get: bin_plpython3
     resource: #@ conf["res_plpython3_bin"]
-- #@ _test_task(conf)
+- #@ _test_and_build_task(conf)
+- put: #@ conf["res_intermediates_bin"]
+  params:
+    file: greenplumpython_artifacts/greenplumpython.tar.gz
 #@ end
+
+#! The final release job
+#! 1. Push the artifacts to the release bucket
+#! 2. Push the git tag
+#@ def exit_release_job(param):
+#@   trigger = param["trigger"]
+#@   confs = param["confs"]
+#@   passed_jobs = []
+#@   res_map = param["res_map"]
+#@   for conf in confs:
+#@     passed_jobs.append(build_test_job_name(conf))
+#@     add_res_by_name(res_map, conf["release_bin"])
+#@   end
+name: exit_release
+on_failure: #@ trigger["on_failure"]
+on_error: #@ trigger["on_error"]
+on_success: #@ trigger["on_success"]
+plan:
+#@   for to_get in trigger["to_get"]:
+- passed: #@ passed_jobs
+  _: #@ template.replace(to_get)
+#@   end
+- in_parallel:
+    steps:
+#@   for i, conf in enumerate(confs):
+      - do:
+        - get: #@ conf["res_intermediates_bin"]
+          passed:
+            - #@ passed_jobs[i]
+          params:
+            unpack: true
+        - put: #@ conf["release_bin"]
+          params:
+            file: #@ conf["res_intermediates_bin"] + "/greenplumpython_*_*.tar.gz"
+#@   end
+#@   for to_put in trigger["to_put_post"]:
+- #@ to_put
+#@   end
+#@ end

concourse/pipeline/release.yml

@@ -2,7 +2,6 @@
 #@   "entrance_job",
 #@   "build_test_job",
 #@   "exit_release_job",
-#@   "centos6_gpdb6_conf",
 #@   "centos7_gpdb6_conf",
 #@   "rhel8_gpdb6_conf",
 #@   "ubuntu18_gpdb6_conf")
@@ -15,7 +14,6 @@
 #@ res_type_map = {}
 #@ trigger = release_trigger(res_map)
 #@ confs = [
-#@   centos6_gpdb6_conf(release_build=True),
 #@   centos7_gpdb6_conf(release_build=True),
 #@   rhel8_gpdb6_conf(release_build=True),
 #@   ubuntu18_gpdb6_conf(release_build=True)

concourse/pipeline/res_def.yml

@@ -103,6 +103,72 @@ resources:
     json_key: ((concourse-gcs-resources-service-account-key))
     regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64.tar.gz
 
+
+# For uploading to the dev bucket
+# For uploading every build to gcs
+# Dev
+- name: bin_greenplumpython_rhel7_intermediates
+  type: gcs
+  source:
+    bucket: gpdb-extensions-concourse-resources
+    json_key: ((extension/extensions-gcs-service-account-key))
+    versioned_file: intermediates/greenplumpython/greenplumpython_rhel7_gpdb6.tar.gz
+
+- name: bin_greenplumpython_rhel8_intermediates
+  type: gcs
+  source:
+    bucket: gpdb-extensions-concourse-resources
+    json_key: ((extension/extensions-gcs-service-account-key))
+    versioned_file: intermediates/greenplumpython/greenplumpython_rhel8_gpdb6.tar.gz
+
+- name: bin_greenplumpython_ubuntu18_intermediates
+  type: gcs
+  source:
+    bucket: gpdb-extensions-concourse-resources
+    json_key: ((extension/extensions-gcs-service-account-key))
+    versioned_file: intermediates/greenplumpython/greenplumpython_ubuntu18_gpdb6.tar.gz
+
+# Release
+- name: bin_greenplumpython_rhel7_intermediates_rel
+  type: gcs
+  source:
+    bucket: gpdb-extensions-concourse-resources
+    json_key: ((extension/extensions-gcs-service-account-key))
+    versioned_file: intermediates_release/greenplumpython/greenplumpython_rhel7_gpdb6.tar.gz
+- name: bin_greenplumpython_rhel8_intermediates_rel
+  type: gcs
+  source:
+    bucket: gpdb-extensions-concourse-resources
+    json_key: ((extension/extensions-gcs-service-account-key))
+    versioned_file: intermediates_release/greenplumpython/greenplumpython_rhel8_gpdb6.tar.gz
+- name: bin_greenplumpython_ubuntu18_intermediates_rel
+  type: gcs
+  source:
+    bucket: gpdb-extensions-concourse-resources
+    json_key: ((extension/extensions-gcs-service-account-key))
+    versioned_file: intermediates_release/greenplumpython/greenplumpython_ubuntu18.04_gpdb6.tar.gz
+
+
+# For uploading to the release bucket
+- name: bin_greenplumpython_rhel7_release
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: greenplumpython/released/greenplumpython-(.*).tar.gz
+- name: bin_greenplumpython_rhel8_release
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: greenplumpython/released/greenplumpython-(.*).tar.gz
+- name: bin_greenplumpython_ubuntu18_release
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: greenplumpython/released/greenplumpython-(.*).tar.gz
+
 # plpython3 for gpdb6
 - name: bin_plpython3_gpdb6_rhel7
   type: gcs

concourse/pipeline/trigger_def.lib.yml

@@ -56,6 +56,27 @@ on_error:
     alert_type: errored
 #@ end
 
+#! Release trigger. For dev pipelines. No webhook
+#@ def release_trigger(res_map):
+#@   add_res_by_name(res_map, "greenplumpython_commit")
+#@   add_res_by_name(res_map, "greenplumpython_release")
+auto_trigger: true
+to_get:
+- get: greenplumpython_src
+  resource: greenplumpython_commit
+to_put_pre: #@ []
+to_put_post:
+- put: greenplumpython_commit
+  params:
+    repository: greenplumpython_src
+    tag: greenplumpython_src/VERSION
+#! To set the github commit status, https://github.com/Pix4D/cogito is a good choice.
+#! Unfortunately it doesn't work with Concourse 5.
+on_success:
+on_failure:
+on_error:
+#@ end
+
 #! Commit trigger. For dev pipelines. No webhook
 #@ def commit_dev_trigger(res_map):
 #@   add_res_by_name(res_map, "greenplumpython_commit_dev")

concourse/scripts/entry.sh

@@ -161,13 +161,12 @@ setup_gpadmin_bashrc
 # Do the special setup with root permission for the each task, then run the real task script with
 # gpadmin. bashrc won't be read by 'su', it needs to be sourced explicitly.
 case "$1" in
-    test)
-
+    test_and_build)
         # To make fly debug easier
         su gpadmin -c \
             "source /home/gpadmin/.bashrc &&\
-            /home/gpadmin/greenplumpython_src/concourse/scripts/test.sh"
-        ;;
+            /home/gpadmin/greenplumpython_src/concourse/scripts/test_and_build.sh"
+        ;;   
     *)
         echo "Unknown target task $1"
         exit 1

concourse/scripts/test_and_build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -l
+
+set -exo pipefail
+
+function _main() {
+    # FIXME: The test db and extension creation should be handled by python code.
+    createdb gpadmin
+    psql gpadmin -c "CREATE LANGUAGE plpython3u;"
+
+    # Run testing
+    pushd /home/gpadmin/greenplumpython_src
+    unset PYTHONPATH
+    unset PYTHONHOME
+    tox -e test_py39
+    # build wheel
+    pip3 wheel . -w ../greenplumpython_artifacts
+    popd
+    tar -czf greenplumpython_artifacts/greenplumpython.tar.gz greenplumpython_artifacts/*.whl
+}
+
+_main