Set up Bazel build system to outsource parameterization (#642)

.gitignore

@@ -4,3 +4,7 @@
 
 # Ignore Python cache files after running tests
 */__pycache__/
+
+# Ignore all bazel-* symlinks. There is no full list since this can change
+# based on the name of the directory bazel is cloned into.
+/bazel-*

BUILD

@@ -0,0 +1,91 @@
+package(default_visibility = ["//visibility:public"])
+
+test_suite(
+    name = "DataprocInitActionsTestSuite",
+    tests = [
+        ":test_cloud_sql_proxy",
+        ":test_dr_elephant",
+        ":test_hive_hcatalog",
+        ":test_starburst_presto",
+        "//bigtable:test_bigtable",
+        "//conda:test_conda",
+        "//connectors:test_connectors",
+        "//datalab:test_datalab",
+        "//drill:test_drill",
+        "//flink:test_flink",
+        "//ganglia:test_ganglia",
+        "//gpu:test_gpu",
+        "//hbase:test_hbase",
+        "//hue:test_hue",
+        "//jupyter:test_jupyter",
+        "//kafka:test_kafka",
+        "//livy:test_livy",
+        "//oozie:test_oozie",
+        "//presto:test_presto",
+        "//ranger:test_ranger",
+        "//rapids:test_rapids",
+        "//rstudio:test_rstudio",
+        "//solr:test_solr",
+        "//tez:test_tez",
+        "//tony:test_tony",
+    ],
+)
+
+py_test(
+    name = "test_cloud_sql_proxy",
+    size = "enormous",
+    srcs = ["cloud-sql-proxy/test_cloud_sql_proxy.py"],
+    data = ["cloud-sql-proxy/cloud-sql-proxy.sh"],
+    local = True,
+    shard_count = 3,
+    deps = [
+        ":pyspark_metastore_test",
+        "//integration_tests:dataproc_test_case",
+        "@io_abseil_py//absl/testing:parameterized",
+    ],
+)
+
+py_test(
+    name = "test_dr_elephant",
+    size = "enormous",
+    srcs = ["dr-elephant/test_dr_elephant.py"],
+    data = ["dr-elephant/dr-elephant.sh"],
+    local = True,
+    shard_count = 2,
+    deps = [
+        "//integration_tests:dataproc_test_case",
+        "@io_abseil_py//absl/testing:parameterized",
+    ],
+)
+
+py_test(
+    name = "test_hive_hcatalog",
+    size = "enormous",
+    srcs = ["hive-hcatalog/test_hive_hcatalog.py"],
+    data = ["hive-hcatalog/hive-hcatalog.sh"],
+    local = True,
+    shard_count = 6,
+    deps = [
+        "//integration_tests:dataproc_test_case",
+        "@io_abseil_py//absl/testing:parameterized",
+    ],
+)
+
+py_test(
+    name = "test_starburst_presto",
+    size = "enormous",
+    srcs = ["starburst-presto/test_starburst_presto.py"],
+    data = ["starburst-presto/presto.sh"],
+    local = True,
+    shard_count = 4,
+    deps = [
+        "//integration_tests:dataproc_test_case",
+        "@io_abseil_py//absl/testing:parameterized",
+    ],
+)
+
+py_library(
+    name = "pyspark_metastore_test",
+    testonly = True,
+    srcs = ["cloud-sql-proxy/pyspark_metastore_test.py"],
+)

WORKSPACE

@@ -0,0 +1,25 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rules_python",
+    sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
+)
+
+http_archive(
+    name = "io_abseil_py",
+    sha256 = "9b324bc96587819a1420c592cf5d54424456645719f1de6bb22f7045bb6dbc6b",
+    strip_prefix = "abseil-py-master",
+    url = "https://github.com/abseil/abseil-py/archive/master.zip",
+)
+
+http_archive(
+    name = "six_archive",
+    build_file = "@io_abseil_py//third_party:six.BUILD",
+    sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
+    strip_prefix = "six-1.10.0",
+    urls = [
+        "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
+        "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
+    ],
+)

bigtable/BUILD

@@ -0,0 +1,20 @@
+package(default_visibility = ["//visibility:public"])
+
+py_test(
+    name = "test_bigtable",
+    size = "enormous",
+    srcs = ["test_bigtable.py"],
+    data = ["bigtable.sh"],
+    local = True,
+    shard_count = 3,
+    deps = [
+        ":run_hbase_commands",
+        "//integration_tests:dataproc_test_case",
+        "@io_abseil_py//absl/testing:parameterized",
+    ],
+)
+
+py_library(
+    name = "run_hbase_commands",
+    srcs = ["run_hbase_commands.py"],
+)

bigtable/test_bigtable.py

@@ -10,9 +10,9 @@
     See: https://cloud.google.com/bigtable/docs/cbt-overview
 """
 import os
-import unittest
 
-from parameterized import parameterized
+from absl.testing import absltest
+from absl.testing import parameterized
 
 from integration_tests.dataproc_test_case import DataprocTestCase
 
@@ -48,13 +48,14 @@ def tearDown(self):
     def _validate_bigtable(self):
         _, stdout, _ = self.assert_command(
             'cbt -instance {} count test-bigtable '.format(self.db_name))
-        self.assertEqual(int(float(stdout)), 4,
-                         "Invalid BigTable instance count")
+        self.assertEqual(
+            int(float(stdout)), 4, "Invalid BigTable instance count")
 
     def verify_instance(self, name):
         self.upload_test_file(
-            os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                         self.TEST_SCRIPT_FILE_NAME), name)
+            os.path.join(
+                os.path.dirname(os.path.abspath(__file__)),
+                self.TEST_SCRIPT_FILE_NAME), name)
         self.assert_instance_command(
             name, "python {}".format(self.TEST_SCRIPT_FILE_NAME))
         self._validate_bigtable()
@@ -64,26 +65,19 @@ def verify_instance(self, name):
     admin commands are provided from text file.
     """
 
-    @parameterized.expand(
-        [
-            ("SINGLE", "1.2", ["m"]),
-            ("STANDARD", "1.2", ["m"]),
-            ("HA", "1.2", ["m-0"]),
-            ("SINGLE", "1.3", ["m"]),
-            ("STANDARD", "1.3", ["m"]),
-            ("HA", "1.3", ["m-0"]),
-        ],
-        testcase_func_name=DataprocTestCase.generate_verbose_test_name)
-    def test_bigtable(self, configuration, dataproc_version, machine_suffixes):
-        self.createCluster(configuration,
-                           self.INIT_ACTIONS,
-                           dataproc_version,
-                           metadata=self.metadata)
+    @parameterized.parameters(
+        ("SINGLE", ["m"]),
+        ("STANDARD", ["m"]),
+        ("HA", ["m-0"]),
+    )
+    def test_bigtable(self, configuration, machine_suffixes):
+        self.createCluster(
+            configuration, self.INIT_ACTIONS, metadata=self.metadata)
 
         for machine_suffix in machine_suffixes:
             self.verify_instance("{}-{}".format(self.getClusterName(),
                                                 machine_suffix))
 
 
 if __name__ == '__main__':
-    unittest.main()
+    absltest.main()

cloud-sql-proxy/test_cloud_sql.py → cloud-sql-proxy/test_cloud_sql_proxy.py

@@ -1,8 +1,8 @@
 import json
 import logging
-import unittest
 
-from parameterized import parameterized
+from absl.testing import absltest
+from absl.testing import parameterized
 
 from integration_tests.dataproc_test_case import DataprocTestCase
 
@@ -50,31 +50,23 @@ def __submit_pyspark_job(self, cluster_name):
             cluster_name, 'pyspark',
             '{}/{}'.format(self.INIT_ACTIONS_REPO, self.TEST_SCRIPT_FILE_NAME))
 
-    @parameterized.expand(
-        [
-            ("SINGLE", "1.2"),
-            ("STANDARD", "1.2"),
-            ("HA", "1.2"),
-            ("SINGLE", "1.3"),
-            ("STANDARD", "1.3"),
-            ("HA", "1.3"),
-            ("SINGLE", "1.4"),
-            ("STANDARD", "1.4"),
-            ("HA", "1.4"),
-        ],
-        testcase_func_name=DataprocTestCase.generate_verbose_test_name)
-    def test_cloud_sql_proxy(self, configuration, dataproc_version):
+    @parameterized.parameters(
+        "SINGLE",
+        "STANDARD",
+        "HA",
+    )
+    def test_cloud_sql_proxy(self, configuration):
         metadata = 'hive-metastore-instance={}:{}'.format(
             self.PROJECT_METADATA, self.DB_NAME)
-        self.createCluster(configuration,
-                           self.INIT_ACTIONS,
-                           dataproc_version,
-                           machine_type="n1-standard-2",
-                           metadata=metadata,
-                           scopes='sql-admin')
+        self.createCluster(
+            configuration,
+            self.INIT_ACTIONS,
+            machine_type="n1-standard-2",
+            metadata=metadata,
+            scopes='sql-admin')
 
         self.verify_cluster(self.getClusterName())
 
 
 if __name__ == '__main__':
-    unittest.main()
+    absltest.main()

cloudbuild/Dockerfile

@@ -7,13 +7,10 @@ RUN useradd -m -d /home/ia-tests -s /bin/bash ia-tests
 
 COPY --chown=ia-tests:ia-tests . /init-actions
 
-# Install Pip
-RUN apt-get -y update
-RUN apt-get -y install python3-pip
-
-# Install fastunit
-WORKDIR /tmp/fastunit
-RUN git clone https://github.com/ityoung/python3-fastunit.git .
-RUN python3 setup.py install
+# Install Bazel:
+# https://docs.bazel.build/versions/master/install-ubuntu.html
+RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
+RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
+RUN apt-get update && apt-get install -y openjdk-8-jdk python3-setuptools bazel
 
 USER ia-tests

cloudbuild/cloudbuild.yaml

@@ -1,15 +1,42 @@
 steps:
   # Create Docker image from regular Dockerfile
   - name: 'gcr.io/cloud-builders/docker'
+    id: 'docker-build'
     args: ['build', '--tag=gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '-f', 'cloudbuild/Dockerfile', '.']
+
   # Push Docker image to GCR
   - name: 'gcr.io/cloud-builders/docker'
+    id: 'gcr-push'
     args: ['push', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID']
 
-  # Run presubmit script
+  # Run presubmit tests in parallel for 1.2 image
+  - name: 'gcr.io/cloud-builders/kubectl'
+    id: 'dataproc-1.2-tests'
+    waitFor: ['gcr-push']
+    entrypoint: 'bash'
+    args: ['cloudbuild/run-presubmit-on-k8s.sh', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '$BUILD_ID', '1.2']
+    env:
+      - 'COMMIT_SHA=$COMMIT_SHA'
+      - 'CLOUDSDK_COMPUTE_ZONE=us-central1-f'
+      - 'CLOUDSDK_CONTAINER_CLUSTER=init-actions-presubmit'
+
+  # Run presubmit tests in parallel for 1.3 image
+  - name: 'gcr.io/cloud-builders/kubectl'
+    id: 'dataproc-1.3-tests'
+    waitFor: ['gcr-push']
+    entrypoint: 'bash'
+    args: ['cloudbuild/run-presubmit-on-k8s.sh', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '$BUILD_ID', '1.3']
+    env:
+      - 'COMMIT_SHA=$COMMIT_SHA'
+      - 'CLOUDSDK_COMPUTE_ZONE=us-central1-f'
+      - 'CLOUDSDK_CONTAINER_CLUSTER=init-actions-presubmit'
+
+  # Run presubmit tests in parallel for 1.4 image
   - name: 'gcr.io/cloud-builders/kubectl'
+    id: 'dataproc-1.4-tests'
+    waitFor: ['gcr-push']
     entrypoint: 'bash'
-    args: ['cloudbuild/run-presubmit-on-k8s.sh', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '$BUILD_ID']
+    args: ['cloudbuild/run-presubmit-on-k8s.sh', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '$BUILD_ID', '1.4']
     env:
       - 'COMMIT_SHA=$COMMIT_SHA'
       - 'CLOUDSDK_COMPUTE_ZONE=us-central1-f'