enable kerberos support for non GPU Driver dependent init action tests (

#1246)

* enable kerberos support for init action tests

* removing kerberos parameter from not supported init actions

* removing kerberos parameter from not supported init actions

* removing kerberos parameter from not supported init actions

* removing kerberos parameter from not supported init actions

* unpinning libsystemd0 as the fix is patched in open source

* revert the changes for BUILD file

BUILD

@@ -6,7 +6,6 @@ test_suite(
         ":test_cloud_sql_proxy",
         ":test_dr_elephant",
         ":test_hive_hcatalog",
-        ":test_hive_llap",
         ":test_starburst_presto",
         ":test_spark_rapids",
         "//alluxio:test_alluxio",
@@ -63,7 +62,7 @@ py_test(
     srcs = ["cloud-sql-proxy/test_cloud_sql_proxy.py"],
     data = ["cloud-sql-proxy/cloud-sql-proxy.sh", "cloud-sql-proxy/hivetest.hive"],
     local = True,
-    shard_count = 3,
+    shard_count = 4,
     deps = [
         ":pyspark_metastore_test",
         "//integration_tests:dataproc_test_case",
@@ -77,7 +76,7 @@ py_test(
     srcs = ["dr-elephant/test_dr_elephant.py"],
     data = ["dr-elephant/dr-elephant.sh"],
     local = True,
-    shard_count = 2,
+    shard_count = 3,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",
@@ -90,7 +89,7 @@ py_test(
     srcs = ["hive-hcatalog/test_hive_hcatalog.py"],
     data = ["hive-hcatalog/hive-hcatalog.sh"],
     local = True,
-    shard_count = 6,
+    shard_count = 8,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",
@@ -103,7 +102,7 @@ py_test(
     srcs = ["starburst-presto/test_starburst_presto.py"],
     data = ["starburst-presto/presto.sh"],
     local = True,
-    shard_count = 4,
+    shard_count = 6,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",

alluxio/BUILD

@@ -8,7 +8,7 @@ py_test(
         "alluxio.sh",
     ],
     local = True,
-    shard_count = 2,
+    shard_count = 4,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",

alluxio/test_alluxio.py

@@ -16,7 +16,9 @@ def verify_instance(self, name):
     self.assert_instance_command(name, "alluxio fs leader")
 
   @parameterized.parameters(
-      ("STANDARD", ["m"]),)
+      ("STANDARD", ["m"]),
+      ("KERBEROS", ["m"]),
+  )
   def test_alluxio(self, configuration, machine_suffixes):
     if self.getImageOs() == 'rocky':
       self.skipTest("Not supported in Rocky Linux-based images")
@@ -32,7 +34,9 @@ def test_alluxio(self, configuration, machine_suffixes):
                                           machine_suffix))
 
   @parameterized.parameters(
-      ("STANDARD", ["m"]),)
+      ("STANDARD", ["m"]),
+      ("KERBEROS", ["m"]),
+  )
   def test_alluxio_with_presto(self, configuration, machine_suffixes):
     if self.getImageOs() == 'rocky':
       self.skipTest("Not supported in Rocky Linux-based images")

atlas/BUILD

@@ -10,7 +10,7 @@ py_test(
         "//kafka:kafka.sh",
     ],
     local = True,
-    shard_count = 7,
+    shard_count = 9,
     deps = [
         ":validate_atlas",
         "//integration_tests:dataproc_test_case",

atlas/test_atlas.py

@@ -94,6 +94,7 @@ def verify_instance(self, instance, username='admin', password='admin'):
     @parameterized.parameters(
         ("SINGLE", ["m"]),
         ("STANDARD", ["m"]),
+        ("KERBEROS", ["m"]),
     )
     def test_atlas(self, configuration, machine_suffixes):
         if self.getImageOs() == 'rocky':
@@ -145,7 +146,10 @@ def test_atlas(self, configuration, machine_suffixes):
         if configuration == "HA":
             self.assertEqual(2, atlas_statuses.count("PASSIVE"))
 
-    @parameterized.parameters(("SINGLE", ["m"]))
+    @parameterized.parameters(
+        ("SINGLE", ["m"]),
+        ("KERBEROS", ["m"]),
+    )
     def test_atlas_overrides_admin_credentials(self, configuration,
                                                machine_suffixes):
       if self.getImageOs() == 'rocky':

bigtable/BUILD

@@ -6,7 +6,7 @@ py_test(
     srcs = ["test_bigtable.py"],
     data = ["bigtable.sh"],
     local = True,
-    shard_count = 3,
+    shard_count = 4,
     deps = [
         ":run_hbase_commands",
         "//integration_tests:dataproc_test_case",

bigtable/test_bigtable.py

@@ -67,6 +67,7 @@ def verify_instance(self, name):
         ("SINGLE", ["m"]),
         ("STANDARD", ["m"]),
         ("HA", ["m-0"]),
+        ("KERBEROS", ["m"]),
     )
     def test_bigtable(self, configuration, machine_suffixes):
         self.createCluster(

cloud-sql-proxy/test_cloud_sql_proxy.py

@@ -59,6 +59,7 @@ def __submit_pyspark_job(self, cluster_name):
       'SINGLE',
       'STANDARD',
       'HA',
+      'KERBEROS',
   )
   def test_cloud_sql_proxy(self, configuration):
     metadata = 'hive-metastore-instance={}:{},hive-metastore-db=metastore'.format(self.PROJECT_METADATA,

conda/BUILD

@@ -11,7 +11,7 @@ py_test(
         "install-conda-env.sh",
     ],
     local = True,
-    shard_count = 2,
+    shard_count = 4,
     deps = [
         ":get_sys_exec",
         "//integration_tests:dataproc_test_case",

conda/test_conda.py

@@ -56,6 +56,8 @@ def _parse_packages(stdout):
     @parameterized.parameters(
         ("STANDARD", [], []),
         ("STANDARD", CONDA_PKGS, PIP_PKGS),
+        ("KERBEROS", [], []),
+        ("KERBEROS", CONDA_PKGS, PIP_PKGS),
     )
     def test_conda(self, configuration, conda_packages, pip_packages):
         if self.getImageOs() == 'rocky':

connectors/BUILD

@@ -6,7 +6,7 @@ py_test(
     srcs = ["test_connectors.py"],
     data = ["connectors.sh"],
     local = True,
-    shard_count = 10,
+    shard_count = 12,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",

connectors/test_connectors.py

@@ -50,8 +50,11 @@ def _hadoop_version(self):
     def _scala_version(self):
         return "2.12"
 
-    @parameterized.parameters(("SINGLE", ["m"]),
-                              ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]))
+    @parameterized.parameters(
+        ("SINGLE", ["m"]),
+        ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]),
+        ("KERBEROS", ["m"]),
+    )
     def test_bq_connector_version(self, configuration, instances):
         if self.getImageOs() == 'rocky':
           self.skipTest("Not supported in Rocky Linux-based images")
@@ -63,8 +66,11 @@ def test_bq_connector_version(self, configuration, instances):
         self.verify_instances(self.getClusterName(), instances,
                               "bigquery-connector", self.BQ_CONNECTOR_VERSION)
 
-    @parameterized.parameters(("SINGLE", ["m"]),
-                              ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]))
+    @parameterized.parameters(
+        ("SINGLE", ["m"]),
+        ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]),
+        ("KERBEROS", ["m"]),
+    )
     def test_spark_bq_connector_version(self, configuration, instances):
         if self.getImageVersion() < pkg_resources.parse_version("1.5"):
             self.SPARK_BQ_CONNECTOR_VERSION = "0.29.0"
@@ -83,8 +89,11 @@ def test_spark_bq_connector_version(self, configuration, instances):
                               "spark-bigquery-connector",
                               self.SPARK_BQ_CONNECTOR_VERSION)
 
-    @parameterized.parameters(("SINGLE", ["m"]),
-                              ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]))
+    @parameterized.parameters(
+        ("SINGLE", ["m"]),
+        ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]),
+        ("KERBEROS", ["m"]),
+    )
     def test_bq_connector_url(self, configuration, instances):
         if self.getImageVersion() < pkg_resources.parse_version("1.5"):
             self.SPARK_BQ_CONNECTOR_VERSION = "0.29.0"
@@ -102,8 +111,11 @@ def test_bq_connector_url(self, configuration, instances):
         self.verify_instances(self.getClusterName(), instances,
                               "bigquery-connector", self.BQ_CONNECTOR_VERSION)
 
-    @parameterized.parameters(("SINGLE", ["m"]),
-                              ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]))
+    @parameterized.parameters(
+        ("SINGLE", ["m"]),
+        ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]),
+        ("KERBEROS", ["m"]),
+    )
     def test_spark_bq_connector_url(self, configuration, instances):
         if self.getImageVersion() < pkg_resources.parse_version("1.5"):
             self.SPARK_BQ_CONNECTOR_VERSION = "0.29.0"

dask/BUILD

@@ -12,7 +12,7 @@ py_test(
         "verify_dask_standalone.py",
     ],
     local = True,
-    shard_count = 3,
+    shard_count = 4,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",

dask/test_dask.py

@@ -39,7 +39,9 @@ def _run_dask_test_script(self, name, script):
 
     @parameterized.parameters(
         ("STANDARD", ["m", "w-0"], "yarn"),
-        ("STANDARD", ["m"], "standalone"))
+        ("STANDARD", ["m"], "standalone"),
+        ("KERBEROS", ["m"], "standalone"),
+    )
     def test_dask(self, configuration, instances, runtime):
 
         if self.getImageVersion() < pkg_resources.parse_version("2.0"):

dr-elephant/test_dr_elephant.py

@@ -30,6 +30,7 @@ def verify_instance(self, instance_name):
     @parameterized.parameters(
         ("STANDARD", ["m"]),
         ("HA", ["m-0"]),
+        ("KERBEROS", ["m"]),
     )
     def test_dr_elephant(self, configuration, machine_suffixes):
         if self.getImageVersion() >= pkg_resources.parse_version("1.3"):

drill/BUILD

@@ -10,7 +10,7 @@ py_test(
         "//zookeeper:zookeeper.sh",
     ],
     local = True,
-    shard_count = 3,
+    shard_count = 4,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",

drill/test_drill.py

@@ -30,6 +30,7 @@ def __run_bash_test_file(self, name, drill_mode, target_node):
         ("SINGLE", [("m", "m")]),
         ("STANDARD", [("m", "w-0"), ("m", "m")]),
         ("HA", [("m-0", "w-0"), ("w-0", "m-1")]),
+        ("KERBEROS", ["m"]),
     )
     def test_drill(self, configuration, verify_options):
         if self.getImageOs() == 'rocky':

flink/BUILD

@@ -9,7 +9,7 @@ py_test(
         "validate.sh",
     ],
     local = True,
-    shard_count = 5,
+    shard_count = 7,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",

flink/test_flink.py

@@ -28,6 +28,7 @@ def __run_test_file(self, name, yarn_session):
   @parameterized.parameters(
       ("STANDARD", ["m"]),
       ("HA", ["m-0", "m-1", "m-2"]),
+      ("KERBEROS", ["m"]),
   )
   def test_flink(self, configuration, machine_suffixes):
     if self.getImageOs() == 'rocky':
@@ -49,6 +50,7 @@ def test_flink(self, configuration, machine_suffixes):
       ("SINGLE", ["m"]),
       ("STANDARD", ["m"]),
       ("HA", ["m-0", "m-1", "m-2"]),
+      ("KERBEROS", ["m"]),
   )
   def test_flink_with_optional_metadata(self, configuration, machine_suffixes):
     if self.getImageOs() == 'rocky':

ganglia/BUILD

@@ -8,7 +8,7 @@ py_test(
         "ganglia.sh",
     ],
     local = True,
-    shard_count = 3,
+    shard_count = 4,
     deps = [
         ":verify_ganglia_running",
         "//integration_tests:dataproc_test_case",

ganglia/test_ganglia.py

@@ -31,6 +31,7 @@ def verify_instance(self, name):
         ("SINGLE", ["m"]),
         ("STANDARD", ["m", "w-0"]),
         ("HA", ["m-0", "m-1", "m-2", "w-0"]),
+        ("KERBEROS", ["m"]),
     )
     def test_ganglia(self, configuration, machine_suffixes):
         if self.getImageOs() == 'rocky':

h2o/BUILD

@@ -10,7 +10,7 @@ py_test(
         "//conda:bootstrap-conda.sh",
     ],
     local = True,
-    shard_count = 2,
+    shard_count = 3,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",

h2o/test_h2o.py

@@ -10,7 +10,7 @@ class H2OTestCase(DataprocTestCase):
     INIT_ACTIONS = ["h2o/h2o.sh"]
     SAMPLE_H2O_JOB_PATH = "h2o/sample-script.py"
 
-    @parameterized.parameters("STANDARD", "HA")
+    @parameterized.parameters("STANDARD", "HA", "KERBEROS")
     def test_h2o(self, configuration):
         if self.getImageVersion() < pkg_resources.parse_version("2.0"):
             self.skipTest("Not supported in pre-2.0 images")

hbase/BUILD

@@ -9,7 +9,7 @@ py_test(
         "//zookeeper:zookeeper.sh",
     ],
     local = True,
-    shard_count = 6,
+    shard_count = 8,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",

hbase/test_hbase.py

@@ -32,6 +32,7 @@ def verify_instance(self, name):
         ("SINGLE", ["m"]),
         ("STANDARD", ["m"]),
         ("HA", ["m-0"]),
+        ("KERBEROS", ["m"]),
     )
     def test_hbase(self, configuration, machine_suffixes):
         if self.getImageOs() == 'rocky':
@@ -54,6 +55,7 @@ def test_hbase(self, configuration, machine_suffixes):
         ("SINGLE", ["m"]),
         ("STANDARD", ["m"]),
         ("HA", ["m-0"]),
+        ("KERBEROS", ["m"]),
     )
     def test_hbase_on_gcs(self, configuration, machine_suffixes):
         if self.getImageOs() == 'rocky':

hive-hcatalog/test_hive_hcatalog.py

@@ -61,9 +61,11 @@ def __submit_hive_job(self, cluster_name, job, should_repeat_job=False):
         ("SINGLE", False),
         ("STANDARD", False),
         ("HA", False),
+        ("KERBEROS", False),
         ("SINGLE", True),
         ("STANDARD", True),
         ("HA", True),
+        ("KERBEROS", True),
     )
     def test_hive_hcatalog(self, configuration, should_repeat_job):
         if self.getImageOs() == 'rocky':

hue/BUILD

@@ -10,7 +10,7 @@ py_test(
             "run_queries.py"],
     data = ["hue.sh"],
     local = True,
-    shard_count = 3,
+    shard_count = 7,
     deps = [
         "//integration_tests:dataproc_test_case",
         "@io_abseil_py//absl/testing:parameterized",

integration_tests/dataproc_test_case.py

@@ -38,6 +38,9 @@ class DataprocTestCase(parameterized.TestCase):
         "HA": [
             "--num-masters=3",
             "--num-workers=2",
+        ],
+        "KERBEROS": [
+            "--enable-kerberos"
         ]
     }
 

kafka/test_kafka.py

@@ -46,6 +46,7 @@ def __submit_pyspark_job(self, cluster_name):
     @parameterized.parameters(
         'STANDARD',
         'HA',
+        'KERBEROS',
     )
     def test_kafka_job(self, configuration):
         if self.getImageOs() == 'rocky':
@@ -60,6 +61,7 @@ def test_kafka_job(self, configuration):
     @parameterized.parameters(
         'STANDARD',
         'HA',
+        'KERBEROS',
     )
     def test_kafka_cruise_control_job(self, configuration):
         if self.getImageOs() == 'rocky':
@@ -74,6 +76,7 @@ def test_kafka_cruise_control_job(self, configuration):
     @parameterized.parameters(
         'STANDARD',
         'HA',
+        'KERBEROS',
     )
     def test_kafka_manager_job(self, configuration):
         if self.getImageOs() == 'rocky':