diff --git a/BUILD b/BUILD index c8efdeac0..dfb599b4d 100644 --- a/BUILD +++ b/BUILD @@ -6,7 +6,6 @@ test_suite( ":test_cloud_sql_proxy", ":test_dr_elephant", ":test_hive_hcatalog", - ":test_hive_llap", ":test_starburst_presto", ":test_spark_rapids", "//alluxio:test_alluxio", @@ -63,7 +62,7 @@ py_test( srcs = ["cloud-sql-proxy/test_cloud_sql_proxy.py"], data = ["cloud-sql-proxy/cloud-sql-proxy.sh", "cloud-sql-proxy/hivetest.hive"], local = True, - shard_count = 3, + shard_count = 4, deps = [ ":pyspark_metastore_test", "//integration_tests:dataproc_test_case", @@ -77,7 +76,7 @@ py_test( srcs = ["dr-elephant/test_dr_elephant.py"], data = ["dr-elephant/dr-elephant.sh"], local = True, - shard_count = 2, + shard_count = 3, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", @@ -90,7 +89,7 @@ py_test( srcs = ["hive-hcatalog/test_hive_hcatalog.py"], data = ["hive-hcatalog/hive-hcatalog.sh"], local = True, - shard_count = 6, + shard_count = 8, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", @@ -103,7 +102,7 @@ py_test( srcs = ["starburst-presto/test_starburst_presto.py"], data = ["starburst-presto/presto.sh"], local = True, - shard_count = 4, + shard_count = 6, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/alluxio/BUILD b/alluxio/BUILD index ae77b412f..46424653d 100644 --- a/alluxio/BUILD +++ b/alluxio/BUILD @@ -8,7 +8,7 @@ py_test( "alluxio.sh", ], local = True, - shard_count = 2, + shard_count = 4, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/alluxio/test_alluxio.py b/alluxio/test_alluxio.py index 93c34a707..458168f2a 100644 --- a/alluxio/test_alluxio.py +++ b/alluxio/test_alluxio.py @@ -16,7 +16,9 @@ def verify_instance(self, name): self.assert_instance_command(name, "alluxio fs leader") @parameterized.parameters( - ("STANDARD", ["m"]),) + ("STANDARD", ["m"]), + ("KERBEROS", ["m"]), + ) def test_alluxio(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': self.skipTest("Not supported in Rocky Linux-based images") @@ -32,7 +34,9 @@ def test_alluxio(self, configuration, machine_suffixes): machine_suffix)) @parameterized.parameters( - ("STANDARD", ["m"]),) + ("STANDARD", ["m"]), + ("KERBEROS", ["m"]), + ) def test_alluxio_with_presto(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': self.skipTest("Not supported in Rocky Linux-based images") diff --git a/atlas/BUILD b/atlas/BUILD index 032e5e234..4ddbde1eb 100644 --- a/atlas/BUILD +++ b/atlas/BUILD @@ -10,7 +10,7 @@ py_test( "//kafka:kafka.sh", ], local = True, - shard_count = 7, + shard_count = 9, deps = [ ":validate_atlas", "//integration_tests:dataproc_test_case", diff --git a/atlas/test_atlas.py b/atlas/test_atlas.py index 5ea895ce3..71afae757 100644 --- a/atlas/test_atlas.py +++ b/atlas/test_atlas.py @@ -94,6 +94,7 @@ def verify_instance(self, instance, username='admin', password='admin'): @parameterized.parameters( ("SINGLE", ["m"]), ("STANDARD", ["m"]), + ("KERBEROS", ["m"]), ) def test_atlas(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': @@ -145,7 +146,10 @@ def test_atlas(self, configuration, machine_suffixes): if configuration == "HA": self.assertEqual(2, atlas_statuses.count("PASSIVE")) - @parameterized.parameters(("SINGLE", ["m"])) + @parameterized.parameters( + ("SINGLE", ["m"]), + ("KERBEROS", ["m"]), + ) def test_atlas_overrides_admin_credentials(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': diff --git a/bigtable/BUILD b/bigtable/BUILD index 50ca95e83..4310ce2e0 100644 --- a/bigtable/BUILD +++ b/bigtable/BUILD @@ -6,7 +6,7 @@ py_test( srcs = ["test_bigtable.py"], data = ["bigtable.sh"], local = True, - shard_count = 3, + shard_count = 4, deps = [ ":run_hbase_commands", "//integration_tests:dataproc_test_case", diff --git a/bigtable/test_bigtable.py b/bigtable/test_bigtable.py index 1acc14c2a..8a0061864 100644 --- a/bigtable/test_bigtable.py +++ b/bigtable/test_bigtable.py @@ -67,6 +67,7 @@ def verify_instance(self, name): ("SINGLE", ["m"]), ("STANDARD", ["m"]), ("HA", ["m-0"]), + ("KERBEROS", ["m"]), ) def test_bigtable(self, configuration, machine_suffixes): self.createCluster( diff --git a/cloud-sql-proxy/test_cloud_sql_proxy.py b/cloud-sql-proxy/test_cloud_sql_proxy.py index d9226a23c..b9923b578 100644 --- a/cloud-sql-proxy/test_cloud_sql_proxy.py +++ b/cloud-sql-proxy/test_cloud_sql_proxy.py @@ -59,6 +59,7 @@ def __submit_pyspark_job(self, cluster_name): 'SINGLE', 'STANDARD', 'HA', + 'KERBEROS', ) def test_cloud_sql_proxy(self, configuration): metadata = 'hive-metastore-instance={}:{},hive-metastore-db=metastore'.format(self.PROJECT_METADATA, diff --git a/conda/BUILD b/conda/BUILD index b85acff40..1e4e3779c 100644 --- a/conda/BUILD +++ b/conda/BUILD @@ -11,7 +11,7 @@ py_test( "install-conda-env.sh", ], local = True, - shard_count = 2, + shard_count = 4, deps = [ ":get_sys_exec", "//integration_tests:dataproc_test_case", diff --git a/conda/test_conda.py b/conda/test_conda.py index a9c20731b..cd20f047d 100644 --- a/conda/test_conda.py +++ b/conda/test_conda.py @@ -56,6 +56,8 @@ def _parse_packages(stdout): @parameterized.parameters( ("STANDARD", [], []), ("STANDARD", CONDA_PKGS, PIP_PKGS), + ("KERBEROS", [], []), + ("KERBEROS", CONDA_PKGS, PIP_PKGS), ) def test_conda(self, configuration, conda_packages, pip_packages): if self.getImageOs() == 'rocky': diff --git a/connectors/BUILD b/connectors/BUILD index be0df9f14..614786f30 100644 --- a/connectors/BUILD +++ b/connectors/BUILD @@ -6,7 +6,7 @@ py_test( srcs = ["test_connectors.py"], data = ["connectors.sh"], local = True, - shard_count = 10, + shard_count = 12, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/connectors/test_connectors.py b/connectors/test_connectors.py index ad6d7b05d..966ea6cef 100644 --- a/connectors/test_connectors.py +++ b/connectors/test_connectors.py @@ -50,8 +50,11 @@ def _hadoop_version(self): def _scala_version(self): return "2.12" - @parameterized.parameters(("SINGLE", ["m"]), - ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"])) + @parameterized.parameters( + ("SINGLE", ["m"]), + ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]), + ("KERBEROS", ["m"]), + ) def test_bq_connector_version(self, configuration, instances): if self.getImageOs() == 'rocky': self.skipTest("Not supported in Rocky Linux-based images") @@ -63,8 +66,11 @@ def test_bq_connector_version(self, configuration, instances): self.verify_instances(self.getClusterName(), instances, "bigquery-connector", self.BQ_CONNECTOR_VERSION) - @parameterized.parameters(("SINGLE", ["m"]), - ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"])) + @parameterized.parameters( + ("SINGLE", ["m"]), + ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]), + ("KERBEROS", ["m"]), + ) def test_spark_bq_connector_version(self, configuration, instances): if self.getImageVersion() < pkg_resources.parse_version("1.5"): self.SPARK_BQ_CONNECTOR_VERSION = "0.29.0" @@ -83,8 +89,11 @@ def test_spark_bq_connector_version(self, configuration, instances): "spark-bigquery-connector", self.SPARK_BQ_CONNECTOR_VERSION) - @parameterized.parameters(("SINGLE", ["m"]), - ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"])) + @parameterized.parameters( + ("SINGLE", ["m"]), + ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]), + ("KERBEROS", ["m"]), + ) def test_bq_connector_url(self, configuration, instances): if self.getImageVersion() < pkg_resources.parse_version("1.5"): self.SPARK_BQ_CONNECTOR_VERSION = "0.29.0" @@ -102,8 +111,11 @@ def test_bq_connector_url(self, configuration, instances): self.verify_instances(self.getClusterName(), instances, "bigquery-connector", self.BQ_CONNECTOR_VERSION) - @parameterized.parameters(("SINGLE", ["m"]), - ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"])) + @parameterized.parameters( + ("SINGLE", ["m"]), + ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]), + ("KERBEROS", ["m"]), + ) def test_spark_bq_connector_url(self, configuration, instances): if self.getImageVersion() < pkg_resources.parse_version("1.5"): self.SPARK_BQ_CONNECTOR_VERSION = "0.29.0" diff --git a/dask/BUILD b/dask/BUILD index 97cd7d3c9..bb6aca492 100644 --- a/dask/BUILD +++ b/dask/BUILD @@ -12,7 +12,7 @@ py_test( "verify_dask_standalone.py", ], local = True, - shard_count = 3, + shard_count = 4, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/dask/test_dask.py b/dask/test_dask.py index 8671dfac3..a5d4314e4 100644 --- a/dask/test_dask.py +++ b/dask/test_dask.py @@ -39,7 +39,9 @@ def _run_dask_test_script(self, name, script): @parameterized.parameters( ("STANDARD", ["m", "w-0"], "yarn"), - ("STANDARD", ["m"], "standalone")) + ("STANDARD", ["m"], "standalone"), + ("KERBEROS", ["m"], "standalone"), + ) def test_dask(self, configuration, instances, runtime): if self.getImageVersion() < pkg_resources.parse_version("2.0"): diff --git a/dr-elephant/test_dr_elephant.py b/dr-elephant/test_dr_elephant.py index 742094ef5..a3e6a7eae 100644 --- a/dr-elephant/test_dr_elephant.py +++ b/dr-elephant/test_dr_elephant.py @@ -30,6 +30,7 @@ def verify_instance(self, instance_name): @parameterized.parameters( ("STANDARD", ["m"]), ("HA", ["m-0"]), + ("KERBEROS", ["m"]), ) def test_dr_elephant(self, configuration, machine_suffixes): if self.getImageVersion() >= pkg_resources.parse_version("1.3"): diff --git a/drill/BUILD b/drill/BUILD index afada39f5..a8d6fe3ec 100644 --- a/drill/BUILD +++ b/drill/BUILD @@ -10,7 +10,7 @@ py_test( "//zookeeper:zookeeper.sh", ], local = True, - shard_count = 3, + shard_count = 4, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/drill/test_drill.py b/drill/test_drill.py index 3080a349c..b78a92602 100644 --- a/drill/test_drill.py +++ b/drill/test_drill.py @@ -30,6 +30,7 @@ def __run_bash_test_file(self, name, drill_mode, target_node): ("SINGLE", [("m", "m")]), ("STANDARD", [("m", "w-0"), ("m", "m")]), ("HA", [("m-0", "w-0"), ("w-0", "m-1")]), + ("KERBEROS", ["m"]), ) def test_drill(self, configuration, verify_options): if self.getImageOs() == 'rocky': diff --git a/flink/BUILD b/flink/BUILD index d2617b252..aa8532216 100644 --- a/flink/BUILD +++ b/flink/BUILD @@ -9,7 +9,7 @@ py_test( "validate.sh", ], local = True, - shard_count = 5, + shard_count = 7, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/flink/test_flink.py b/flink/test_flink.py index 1748d3754..4a0a0e125 100644 --- a/flink/test_flink.py +++ b/flink/test_flink.py @@ -28,6 +28,7 @@ def __run_test_file(self, name, yarn_session): @parameterized.parameters( ("STANDARD", ["m"]), ("HA", ["m-0", "m-1", "m-2"]), + ("KERBEROS", ["m"]), ) def test_flink(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': @@ -49,6 +50,7 @@ def test_flink(self, configuration, machine_suffixes): ("SINGLE", ["m"]), ("STANDARD", ["m"]), ("HA", ["m-0", "m-1", "m-2"]), + ("KERBEROS", ["m"]), ) def test_flink_with_optional_metadata(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': diff --git a/ganglia/BUILD b/ganglia/BUILD index 5b6f3bd52..c25124265 100644 --- a/ganglia/BUILD +++ b/ganglia/BUILD @@ -8,7 +8,7 @@ py_test( "ganglia.sh", ], local = True, - shard_count = 3, + shard_count = 4, deps = [ ":verify_ganglia_running", "//integration_tests:dataproc_test_case", diff --git a/ganglia/test_ganglia.py b/ganglia/test_ganglia.py index c265b4da1..ce29380d6 100644 --- a/ganglia/test_ganglia.py +++ b/ganglia/test_ganglia.py @@ -31,6 +31,7 @@ def verify_instance(self, name): ("SINGLE", ["m"]), ("STANDARD", ["m", "w-0"]), ("HA", ["m-0", "m-1", "m-2", "w-0"]), + ("KERBEROS", ["m"]), ) def test_ganglia(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': diff --git a/h2o/BUILD b/h2o/BUILD index 7dcd89eb4..621e07706 100644 --- a/h2o/BUILD +++ b/h2o/BUILD @@ -10,7 +10,7 @@ py_test( "//conda:bootstrap-conda.sh", ], local = True, - shard_count = 2, + shard_count = 3, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/h2o/test_h2o.py b/h2o/test_h2o.py index e98e54ecb..9ecd66fd9 100644 --- a/h2o/test_h2o.py +++ b/h2o/test_h2o.py @@ -10,7 +10,7 @@ class H2OTestCase(DataprocTestCase): INIT_ACTIONS = ["h2o/h2o.sh"] SAMPLE_H2O_JOB_PATH = "h2o/sample-script.py" - @parameterized.parameters("STANDARD", "HA") + @parameterized.parameters("STANDARD", "HA", "KERBEROS") def test_h2o(self, configuration): if self.getImageVersion() < pkg_resources.parse_version("2.0"): self.skipTest("Not supported in pre-2.0 images") diff --git a/hbase/BUILD b/hbase/BUILD index 854f02c60..630076f67 100644 --- a/hbase/BUILD +++ b/hbase/BUILD @@ -9,7 +9,7 @@ py_test( "//zookeeper:zookeeper.sh", ], local = True, - shard_count = 6, + shard_count = 8, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/hbase/test_hbase.py b/hbase/test_hbase.py index e6b55788b..2d8d07f19 100644 --- a/hbase/test_hbase.py +++ b/hbase/test_hbase.py @@ -32,6 +32,7 @@ def verify_instance(self, name): ("SINGLE", ["m"]), ("STANDARD", ["m"]), ("HA", ["m-0"]), + ("KERBEROS", ["m"]), ) def test_hbase(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': @@ -54,6 +55,7 @@ def test_hbase(self, configuration, machine_suffixes): ("SINGLE", ["m"]), ("STANDARD", ["m"]), ("HA", ["m-0"]), + ("KERBEROS", ["m"]), ) def test_hbase_on_gcs(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': diff --git a/hive-hcatalog/test_hive_hcatalog.py b/hive-hcatalog/test_hive_hcatalog.py index 29606bac3..904b0843e 100644 --- a/hive-hcatalog/test_hive_hcatalog.py +++ b/hive-hcatalog/test_hive_hcatalog.py @@ -61,9 +61,11 @@ def __submit_hive_job(self, cluster_name, job, should_repeat_job=False): ("SINGLE", False), ("STANDARD", False), ("HA", False), + ("KERBEROS", False), ("SINGLE", True), ("STANDARD", True), ("HA", True), + ("KERBEROS", True), ) def test_hive_hcatalog(self, configuration, should_repeat_job): if self.getImageOs() == 'rocky': diff --git a/hue/BUILD b/hue/BUILD index d1c08c2a1..d9b733f81 100644 --- a/hue/BUILD +++ b/hue/BUILD @@ -10,7 +10,7 @@ py_test( "run_queries.py"], data = ["hue.sh"], local = True, - shard_count = 3, + shard_count = 7, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/integration_tests/dataproc_test_case.py b/integration_tests/dataproc_test_case.py index 6f2bd9f28..c2e6577b1 100644 --- a/integration_tests/dataproc_test_case.py +++ b/integration_tests/dataproc_test_case.py @@ -38,6 +38,9 @@ class DataprocTestCase(parameterized.TestCase): "HA": [ "--num-masters=3", "--num-workers=2", + ], + "KERBEROS": [ + "--enable-kerberos" ] } diff --git a/kafka/test_kafka.py b/kafka/test_kafka.py index 2f3049844..97a843026 100644 --- a/kafka/test_kafka.py +++ b/kafka/test_kafka.py @@ -46,6 +46,7 @@ def __submit_pyspark_job(self, cluster_name): @parameterized.parameters( 'STANDARD', 'HA', + 'KERBEROS', ) def test_kafka_job(self, configuration): if self.getImageOs() == 'rocky': @@ -60,6 +61,7 @@ def test_kafka_job(self, configuration): @parameterized.parameters( 'STANDARD', 'HA', + 'KERBEROS', ) def test_kafka_cruise_control_job(self, configuration): if self.getImageOs() == 'rocky': @@ -74,6 +76,7 @@ def test_kafka_cruise_control_job(self, configuration): @parameterized.parameters( 'STANDARD', 'HA', + 'KERBEROS', ) def test_kafka_manager_job(self, configuration): if self.getImageOs() == 'rocky': diff --git a/knox/BUILD b/knox/BUILD index 675695b31..fe09ad7cf 100644 --- a/knox/BUILD +++ b/knox/BUILD @@ -16,7 +16,7 @@ py_test( "topologies/*", ]), local = True, - shard_count = 6, + shard_count = 8, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/knox/test_knox.py b/knox/test_knox.py index 8503be4c0..70376b236 100644 --- a/knox/test_knox.py +++ b/knox/test_knox.py @@ -27,7 +27,7 @@ def _run_test_script(self, name, cert_type): @parameterized.parameters(("SINGLE", ["m"]), ("SINGLE", ["m"]), ("STANDARD", ["m", "w-0"]), - ("HA", ["m-2", "w-0"])) + ("HA", ["m-2", "w-0"]),) def test_knox_localhost_cert(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': self.skipTest("Not supported in Rocky Linux-based images") @@ -49,7 +49,7 @@ def test_knox_localhost_cert(self, configuration, machine_suffixes): "localhost") @parameterized.parameters(("STANDARD", ["w-0", "m"]), - ("HA", ["m-1", "m-0"])) + ("HA", ["m-1", "m-0"]),) def test_knox_hostname_cert(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': self.skipTest("Not supported in Rocky Linux-based images") diff --git a/livy/BUILD b/livy/BUILD index 8d0f73735..dc7930ecb 100644 --- a/livy/BUILD +++ b/livy/BUILD @@ -6,7 +6,7 @@ py_test( srcs = ["test_livy.py"], data = ["livy.sh"], local = True, - shard_count = 3, + shard_count = 8, deps = [ ":verify_livy_running", "//integration_tests:dataproc_test_case", diff --git a/oozie/BUILD b/oozie/BUILD index 6edef0aba..40ce84623 100644 --- a/oozie/BUILD +++ b/oozie/BUILD @@ -10,7 +10,7 @@ py_test( "verify_oozie_running.py", ], local = True, - shard_count = 3, + shard_count = 4, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/otel/test_otel.py b/otel/test_otel.py index b18d4f270..f15dc8297 100644 --- a/otel/test_otel.py +++ b/otel/test_otel.py @@ -15,6 +15,7 @@ def verify_service_status(self): @parameterized.parameters( 'SINGLE', 'STANDARD', + 'KERBEROS', ) def test_otel(self, configuration): self.createCluster(configuration, self.INIT_ACTIONS) diff --git a/presto/BUILD b/presto/BUILD index 30130934a..c2f616727 100644 --- a/presto/BUILD +++ b/presto/BUILD @@ -8,7 +8,7 @@ py_test( srcs = ["test_presto.py"], data = ["presto.sh"], local = True, - shard_count = 4, + shard_count = 6, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/presto/test_presto.py b/presto/test_presto.py index b22e25928..d4683e1f2 100644 --- a/presto/test_presto.py +++ b/presto/test_presto.py @@ -99,7 +99,7 @@ def test_presto(self, configuration, machine_suffixes, coordinators, "{}-{}".format(self.getClusterName(), machine_suffix), coordinators, workers) - @parameterized.parameters(("SINGLE", ["m"], 1, 0)) + @parameterized.parameters(("SINGLE", ["m"], 1, 0),) def test_presto_custom_port(self, configuration, machine_suffixes, coordinators, workers): if self.getImageOs() == 'rocky': diff --git a/ranger/BUILD b/ranger/BUILD index 8d311c444..d615342bc 100644 --- a/ranger/BUILD +++ b/ranger/BUILD @@ -9,7 +9,7 @@ py_test( "//solr:solr.sh", ], local = True, - shard_count = 3, + shard_count = 4, deps = [ ":verify_ranger", "//integration_tests:dataproc_test_case", diff --git a/ranger/test_ranger.py b/ranger/test_ranger.py index db621517e..2275c1e3e 100644 --- a/ranger/test_ranger.py +++ b/ranger/test_ranger.py @@ -30,6 +30,7 @@ def __run_test_script(self, name): ("SINGLE", ["m"]), ("STANDARD", ["m"]), ("HA", ["m-0"]), + ("KERBEROS", ["m"]), ) def test_ranger(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': diff --git a/rstudio/BUILD b/rstudio/BUILD index 85b4e43b2..dec75ecfe 100644 --- a/rstudio/BUILD +++ b/rstudio/BUILD @@ -6,7 +6,7 @@ py_test( srcs = ["test_rstudio.py"], data = ["rstudio.sh"], local = True, - shard_count = 4, + shard_count = 5, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/rstudio/rstudio.sh b/rstudio/rstudio.sh index a321a22e7..c78eac168 100755 --- a/rstudio/rstudio.sh +++ b/rstudio/rstudio.sh @@ -138,11 +138,7 @@ if [[ "${ROLE}" == 'Master' ]]; then if [[ "${OS_ID}" == "ubuntu" ]]; then curl -fsSL --retry-connrefused --retry 10 --retry-max-time 30 https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc fi - if [[ ${OS_ID} == debian ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION}" == "2.2" | bc -l) == 1 ]]; then - apt-get install -y software-properties-common libsystemd0=252.26-1~deb12u2 - else - apt-get install -y software-properties-common - fi + apt-get install -y software-properties-common add-apt-repository "deb http://cran.r-project.org/bin/linux/${OS_ID} ${OS_CODE}-cran40/" if [[ ${OS_ID} == ubuntu ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION} < 2.1" | bc -l) == 1 ]]; then install_package diff --git a/rstudio/test_rstudio.py b/rstudio/test_rstudio.py index f9730cfbd..abd233f4c 100644 --- a/rstudio/test_rstudio.py +++ b/rstudio/test_rstudio.py @@ -13,6 +13,7 @@ class RStudioTestCase(DataprocTestCase): ("SINGLE", "", "password"), # default username ("SINGLE", "rstudio", ""), # no auth ("SINGLE", "", ""), # default username and no auth + ("KERBEROS", "", ""), ) def test_rstudio(self, configuration, user, password): if self.getImageOs() == 'rocky': diff --git a/solr/BUILD b/solr/BUILD index bddcfa963..e93f1f0c3 100644 --- a/solr/BUILD +++ b/solr/BUILD @@ -8,7 +8,7 @@ py_test( srcs = ["test_solr.py"], data = ["solr.sh"], local = True, - shard_count = 3, + shard_count = 4, deps = [ ":verify_solr", "//integration_tests:dataproc_test_case", diff --git a/solr/test_solr.py b/solr/test_solr.py index 500e98350..a51e7835a 100644 --- a/solr/test_solr.py +++ b/solr/test_solr.py @@ -32,6 +32,7 @@ def __run_test_script(self, name): ("SINGLE", ["m"]), ("STANDARD", ["m"]), ("HA", ["m-0"]), + ("KERBEROS", ["m"]), ) def test_solr(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': diff --git a/sqoop/BUILD b/sqoop/BUILD index a48161464..afce46a14 100644 --- a/sqoop/BUILD +++ b/sqoop/BUILD @@ -6,7 +6,7 @@ py_test( srcs = ["test_sqoop.py"], data = ["sqoop.sh", "test_sql_db_dump.gz", "sqoop_sql.sh"], local = True, - shard_count = 4, + shard_count = 6, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/sqoop/test_sqoop.py b/sqoop/test_sqoop.py index ddcc9713f..fa688f3ad 100644 --- a/sqoop/test_sqoop.py +++ b/sqoop/test_sqoop.py @@ -58,6 +58,7 @@ def import_mysql_db(self, instance): ("SINGLE", ["m"]), ("STANDARD", ["m"]), ("HA", ["m-0", "m-1", "m-2"]), + ("KERBEROS", ["m"]), ) def test_sqoop(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': @@ -68,7 +69,7 @@ def test_sqoop(self, configuration, machine_suffixes): self.verify_instance("{}-{}".format(self.getClusterName(), machine_suffix)) - @parameterized.parameters(("SINGLE", ["m"])) + @parameterized.parameters(("SINGLE", ["m"]),) def test_sqoop_import_from_local_mysql_to_hdfs(self, configuration, machine_suffixes): if self.getImageOs() == 'rocky': diff --git a/starburst-presto/test_starburst_presto.py b/starburst-presto/test_starburst_presto.py index 1fc77f07a..854ad7f99 100644 --- a/starburst-presto/test_starburst_presto.py +++ b/starburst-presto/test_starburst_presto.py @@ -100,7 +100,8 @@ def test_starburst_presto(self, configuration, machine_suffixes, coordinators, workers) @parameterized.parameters( - ("SINGLE", ["m"], 1, 0), ) + ("SINGLE", ["m"], 1, 0), + ) def test_starburst_presto_custom_port( self, configuration, machine_suffixes, coordinators, workers): if self.getImageOs() == 'rocky': diff --git a/toree/BUILD b/toree/BUILD index f02586480..422b0ee0a 100644 --- a/toree/BUILD +++ b/toree/BUILD @@ -6,7 +6,7 @@ py_test( srcs = ["test_toree.py"], data = ["toree.sh"], local = True, - shard_count = 3, + shard_count = 5, deps = [ "//integration_tests:dataproc_test_case", "@io_abseil_py//absl/testing:parameterized", diff --git a/toree/test_toree.py b/toree/test_toree.py index f42521660..b857aebb5 100644 --- a/toree/test_toree.py +++ b/toree/test_toree.py @@ -29,6 +29,7 @@ class ToreeTestCase(DataprocTestCase): ("STANDARD", "m", True), ("HA", "m-0", True), ("SINGLE", "m", False), + ("KERBEROS", "m", False), ) def test_toree(self, configuration, machine_suffix, install_explicit): properties = "dataproc:jupyter.port=12345"