diff --git a/alluxio/test_alluxio.py b/alluxio/test_alluxio.py index 1e8a18c6f..e710fce27 100644 --- a/alluxio/test_alluxio.py +++ b/alluxio/test_alluxio.py @@ -16,8 +16,8 @@ def verify_instance(self, name): @parameterized.parameters( ("STANDARD", ["m"]),) def test_alluxio(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster( configuration, @@ -32,8 +32,8 @@ def test_alluxio(self, configuration, machine_suffixes): @parameterized.parameters( ("STANDARD", ["m"]),) def test_alluxio_with_presto(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster( configuration, diff --git a/atlas/test_atlas.py b/atlas/test_atlas.py index b7011ac64..924114dad 100644 --- a/atlas/test_atlas.py +++ b/atlas/test_atlas.py @@ -93,8 +93,8 @@ def verify_instance(self, instance, username='admin', password='admin'): ("HA", ["m-0", "m-1", "m-2"]), ) def test_atlas(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") image_version = self.getImageVersion() if image_version < pkg_resources.parse_version("1.5") or \ @@ -147,8 +147,8 @@ def test_atlas(self, configuration, machine_suffixes): @parameterized.parameters(("SINGLE", ["m"])) def test_atlas_overrides_admin_credentials(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") image_version = self.getImageVersion() if image_version < pkg_resources.parse_version("1.5") or \ @@ -175,8 +175,8 @@ def test_atlas_overrides_admin_credentials(self, configuration, @parameterized.parameters("ZOOKEEPER", "HBASE", "SOLR") def test_atlas_fails_without_component(self, component): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") image_version = self.getImageVersion() if image_version < pkg_resources.parse_version("1.5") or \ @@ -193,8 +193,8 @@ def test_atlas_fails_without_component(self, component): optional_components=self.OPTIONAL_COMPONENTS.remove(component)) def test_atlas_ha_fails_without_kafka(self): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageVersion() < pkg_resources.parse_version("1.5"): self.skipTest("Not supported in pre 1.5 images") diff --git a/cloud-sql-proxy/test_cloud_sql_proxy.py b/cloud-sql-proxy/test_cloud_sql_proxy.py index cca4fe28d..d9bba09a5 100644 --- a/cloud-sql-proxy/test_cloud_sql_proxy.py +++ b/cloud-sql-proxy/test_cloud_sql_proxy.py @@ -56,8 +56,8 @@ def __submit_pyspark_job(self, cluster_name): 'HA', ) def test_cloud_sql_proxy(self, configuration): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") metadata = 'hive-metastore-instance={}:{}'.format(self.PROJECT_METADATA, self.DB_NAME) diff --git a/conda/test_conda.py b/conda/test_conda.py index d6b3e2659..a9c20731b 100644 --- a/conda/test_conda.py +++ b/conda/test_conda.py @@ -58,8 +58,8 @@ def _parse_packages(stdout): ("STANDARD", CONDA_PKGS, PIP_PKGS), ) def test_conda(self, configuration, conda_packages, pip_packages): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): diff --git a/connectors/test_connectors.py b/connectors/test_connectors.py index 85e33830f..32f1ba3b5 100644 --- a/connectors/test_connectors.py +++ b/connectors/test_connectors.py @@ -55,8 +55,8 @@ def _scala_version(self): @parameterized.parameters(("SINGLE", ["m"]), ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"])) def test_bq_connector_version(self, configuration, instances): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster(configuration, self.INIT_ACTIONS, @@ -68,8 +68,8 @@ def test_bq_connector_version(self, configuration, instances): @parameterized.parameters(("SINGLE", ["m"]), ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"])) def test_spark_bq_connector_version(self, configuration, instances): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster( configuration, @@ -83,8 +83,8 @@ def test_spark_bq_connector_version(self, configuration, instances): @parameterized.parameters(("SINGLE", ["m"]), ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"])) def test_bq_connector_url(self, configuration, instances): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster(configuration, self.INIT_ACTIONS, @@ -97,8 +97,8 @@ def test_bq_connector_url(self, configuration, instances): @parameterized.parameters(("SINGLE", ["m"]), ("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"])) def test_spark_bq_connector_url(self, configuration, instances): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster(configuration, self.INIT_ACTIONS, diff --git a/dask/test_dask.py b/dask/test_dask.py index d939b79fd..4986c1df9 100644 --- a/dask/test_dask.py +++ b/dask/test_dask.py @@ -35,8 +35,8 @@ def _run_dask_test_script(self, name, script): ("STANDARD", ["m", "w-0"], "yarn"), ("STANDARD", ["m"], "standalone")) def test_dask(self, configuration, instances, runtime): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageVersion() <= pkg_resources.parse_version("1.4"): self.skipTest("Not supported in pre 1.5 images") diff --git a/datalab/test_datalab.py b/datalab/test_datalab.py index b58b0ff81..40394d854 100644 --- a/datalab/test_datalab.py +++ b/datalab/test_datalab.py @@ -20,8 +20,8 @@ def verify_instance(self, name): ("STANDARD", ["m"], "python3"), ) def test_datalab(self, configuration, machine_suffixes, python): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): diff --git a/drill/test_drill.py b/drill/test_drill.py index 5b0d9a3c5..4fe17be74 100644 --- a/drill/test_drill.py +++ b/drill/test_drill.py @@ -32,8 +32,8 @@ def __run_bash_test_file(self, name, drill_mode, target_node): ("HA", [("m-0", "w-0"), ("w-0", "m-1")]), ) def test_drill(self, configuration, verify_options): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageVersion() >= pkg_resources.parse_version("2.0"): self.skipTest("Not supported in the 2.0+ images") diff --git a/flink/test_flink.py b/flink/test_flink.py index 995d3fb2b..1748d3754 100644 --- a/flink/test_flink.py +++ b/flink/test_flink.py @@ -30,8 +30,8 @@ def __run_test_file(self, name, yarn_session): ("HA", ["m-0", "m-1", "m-2"]), ) def test_flink(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): @@ -51,8 +51,8 @@ def test_flink(self, configuration, machine_suffixes): ("HA", ["m-0", "m-1", "m-2"]), ) def test_flink_with_optional_metadata(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): diff --git a/ganglia/test_ganglia.py b/ganglia/test_ganglia.py index 4d9f61e7c..bcffe22b8 100644 --- a/ganglia/test_ganglia.py +++ b/ganglia/test_ganglia.py @@ -29,8 +29,8 @@ def verify_instance(self, name): ("HA", ["m-0", "m-1", "m-2", "w-0"]), ) def test_ganglia(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster(configuration, self.INIT_ACTIONS) for machine_suffix in machine_suffixes: diff --git a/gpu/install_gpu_driver.sh b/gpu/install_gpu_driver.sh index d44faa1a0..6f64bb5a3 100755 --- a/gpu/install_gpu_driver.sh +++ b/gpu/install_gpu_driver.sh @@ -47,10 +47,10 @@ NCCL_REPO_URL=$(get_metadata_attribute 'nccl-repo-url' "${DEFAULT_NCCL_REPO_URL} readonly NCCL_REPO_URL readonly DEFAULT_NCCL_VERSION="2.8.3" -readonly DEFAULT_NCCL_VERSION_CENTOS="2.8.4" +readonly DEFAULT_NCCL_VERSION_ROCKY="2.8.4" -if [[ ${OS_NAME} == centos ]]; then - NCCL_VERSION=$(get_metadata_attribute 'nccl-version' ${DEFAULT_NCCL_VERSION_CENTOS}) +if [[ ${OS_NAME} == rocky ]]; then + NCCL_VERSION=$(get_metadata_attribute 'nccl-version' ${DEFAULT_NCCL_VERSION_ROCKY}) else NCCL_VERSION=$(get_metadata_attribute 'nccl-version' ${DEFAULT_NCCL_VERSION}) fi @@ -71,8 +71,8 @@ readonly NVIDIA_UBUNTU_REPOSITORY_URL="${NVIDIA_BASE_DL_URL}/cuda/repos/ubuntu18 readonly NVIDIA_UBUNTU_REPOSITORY_KEY="${NVIDIA_UBUNTU_REPOSITORY_URL}/7fa2af80.pub" readonly NVIDIA_UBUNTU_REPOSITORY_CUDA_PIN="${NVIDIA_UBUNTU_REPOSITORY_URL}/cuda-ubuntu1804.pin" -# Parameter for NVIDIA-provided Centos GPU driver -readonly NVIDIA_CENTOS_REPOSITORY_URL="${NVIDIA_BASE_DL_URL}/cuda/repos/rhel8/x86_64/cuda-rhel8.repo" +# Parameter for NVIDIA-provided Rocky Linux GPU driver +readonly NVIDIA_ROCKY_REPOSITORY_URL="${NVIDIA_BASE_DL_URL}/cuda/repos/rhel8/x86_64/cuda-rhel8.repo" # Parameters for NVIDIA-provided CUDNN library readonly CUDNN_VERSION=$(get_metadata_attribute 'cudnn-version' '') @@ -108,7 +108,7 @@ function execute_with_retries() { function install_nvidia_nccl() { local -r nccl_version="${NCCL_VERSION}-1+cuda${CUDA_VERSION}" - if [[ ${OS_NAME} == centos ]]; then + if [[ ${OS_NAME} == rocky ]]; then execute_with_retries "dnf -y -q install libnccl-${nccl_version} libnccl-devel-${nccl_version} libnccl-static-${nccl_version}" elif [[ ${OS_NAME} == ubuntu ]] || [[ ${OS_NAME} == debian ]]; then local tmp_dir @@ -134,7 +134,7 @@ function install_nvidia_cudnn() { local cudnn_pkg_version cudnn_pkg_version="${CUDNN_VERSION}-1+cuda${CUDA_VERSION}" - if [[ ${OS_NAME} == centos ]]; then + if [[ ${OS_NAME} == rocky ]]; then if [[ ${major_version} == 8 ]]; then execute_with_retries "dnf -y -q install libcudnn8-${cudnn_pkg_version} libcudnn8-devel-${cudnn_pkg_version}" else @@ -196,8 +196,8 @@ function install_nvidia_gpu_driver() { # Without --no-install-recommends this takes a very long time. execute_with_retries "apt-get install -y -q --no-install-recommends cuda-drivers-460" execute_with_retries "apt-get install -y -q --no-install-recommends ${cuda_package}" - elif [[ ${OS_NAME} == centos ]]; then - execute_with_retries "dnf config-manager --add-repo ${NVIDIA_CENTOS_REPOSITORY_URL}" + elif [[ ${OS_NAME} == rocky ]]; then + execute_with_retries "dnf config-manager --add-repo ${NVIDIA_ROCKY_REPOSITORY_URL}" execute_with_retries "dnf clean all" execute_with_retries "dnf -y -q module install nvidia-driver:460-dkms" execute_with_retries "dnf -y -q install cuda-${CUDA_VERSION//./-}" @@ -334,7 +334,7 @@ EOF } function main() { - if [[ ${OS_NAME} != debian ]] && [[ ${OS_NAME} != ubuntu ]] && [[ ${OS_NAME} != centos ]]; then + if [[ ${OS_NAME} != debian ]] && [[ ${OS_NAME} != ubuntu ]] && [[ ${OS_NAME} != rocky ]]; then echo "Unsupported OS: '${OS_NAME}'" exit 1 fi @@ -343,7 +343,7 @@ function main() { export DEBIAN_FRONTEND=noninteractive execute_with_retries "apt-get update" execute_with_retries "apt-get install -y -q pciutils" - elif [[ ${OS_NAME} == centos ]] ; then + elif [[ ${OS_NAME} == rocky ]] ; then execute_with_retries "dnf -y -q update" execute_with_retries "dnf -y -q install pciutils" execute_with_retries "dnf -y -q install kernel-devel" diff --git a/gpu/test_gpu.py b/gpu/test_gpu.py index e357aed0c..fec3bbdf9 100644 --- a/gpu/test_gpu.py +++ b/gpu/test_gpu.py @@ -30,8 +30,8 @@ def verify_instance_cudnn(self, name): def test_install_gpu_default_agent(self, configuration, machine_suffixes, master_accelerator, worker_accelerator, driver_provider): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") metadata = None if driver_provider is not None: @@ -56,8 +56,8 @@ def test_install_gpu_default_agent(self, configuration, machine_suffixes, def test_install_gpu_without_agent(self, configuration, machine_suffixes, master_accelerator, worker_accelerator, driver_provider): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") metadata = "install-gpu-agent=false" if driver_provider is not None: @@ -82,8 +82,8 @@ def test_install_gpu_without_agent(self, configuration, machine_suffixes, def test_install_gpu_with_agent(self, configuration, machine_suffixes, master_accelerator, worker_accelerator, driver_provider): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # GPU agent supported on Dataproc 1.4+ if self.getImageVersion() < pkg_resources.parse_version("1.4"): @@ -117,8 +117,8 @@ def test_install_gpu_with_agent(self, configuration, machine_suffixes, def test_install_gpu_cuda_nvidia(self, configuration, machine_suffixes, master_accelerator, worker_accelerator, cuda_version): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") metadata = "gpu-driver-provider=NVIDIA,cuda-version={}".format(cuda_version) self.createCluster( @@ -139,8 +139,8 @@ def test_install_gpu_cuda_nvidia(self, configuration, machine_suffixes, def test_install_gpu_with_cudnn(self, configuration, machine_suffixes, master_accelerator, worker_accelerator, driver_provider, cudnn_version): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") metadata = "cudnn-version={}".format(cudnn_version) if driver_provider is not None: diff --git a/hbase/test_hbase.py b/hbase/test_hbase.py index 632813cc4..0498efbff 100644 --- a/hbase/test_hbase.py +++ b/hbase/test_hbase.py @@ -34,8 +34,8 @@ def verify_instance(self, name): ("HA", ["m-0"]), ) def test_hbase(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): @@ -56,8 +56,8 @@ def test_hbase(self, configuration, machine_suffixes): ("HA", ["m-0"]), ) def test_hbase_on_gcs(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): diff --git a/hive-hcatalog/test_hive_hcatalog.py b/hive-hcatalog/test_hive_hcatalog.py index 03b55147d..29606bac3 100644 --- a/hive-hcatalog/test_hive_hcatalog.py +++ b/hive-hcatalog/test_hive_hcatalog.py @@ -66,8 +66,8 @@ def __submit_hive_job(self, cluster_name, job, should_repeat_job=False): ("HA", True), ) def test_hive_hcatalog(self, configuration, should_repeat_job): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster(configuration, self.INIT_ACTIONS) self.verify_instance(self.getClusterName(), should_repeat_job) diff --git a/hive-llap/test_hive_llap.py b/hive-llap/test_hive_llap.py index bf2933dbe..8d658f569 100644 --- a/hive-llap/test_hive_llap.py +++ b/hive-llap/test_hive_llap.py @@ -32,8 +32,8 @@ def __run_test_script(self, name): ("HA", ["m-0"]), ("STANDARD", ["m"])) def test_llap(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageVersion() < pkg_resources.parse_version("2.0"): self.skipTest("Hive LLAP only supported on Dataproc 2.0+") @@ -52,8 +52,8 @@ def test_llap(self, configuration, machine_suffixes): ("STANDARD", ["m"]), ("HA", ["m-0"])) def test_llap_num_exec(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageVersion() < pkg_resources.parse_version("2.0"): self.skipTest("Hive LLAP only supported on Dataproc 2.0+") @@ -72,8 +72,8 @@ def test_llap_num_exec(self, configuration, machine_suffixes): ("STANDARD", ["m"]), ("HA", ["m-0"])) def test_llap_exec_size(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageVersion() < pkg_resources.parse_version("2.0"): self.skipTest("Hive LLAP only supported on Dataproc 2.0+") diff --git a/horovod/test_horovod.py b/horovod/test_horovod.py index 3257ef6fc..cd84422d5 100644 --- a/horovod/test_horovod.py +++ b/horovod/test_horovod.py @@ -25,8 +25,8 @@ def _submit_spark_job(self, script): ("STANDARD", "gloo"), ) def test_horovod_cpu(self, configuration, controller): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Init action supported on Dataproc 1.4+ if self.getImageVersion() <= pkg_resources.parse_version("1.3"): @@ -47,8 +47,8 @@ def test_horovod_cpu(self, configuration, controller): ("STANDARD", "gloo"), ) def test_horovod_gpu(self, configuration, controller): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Init action supported on Dataproc 1.4+ if self.getImageVersion() <= pkg_resources.parse_version("1.3"): diff --git a/jupyter/test_jupyter.py b/jupyter/test_jupyter.py index 9aaafd59b..5ef223e1e 100644 --- a/jupyter/test_jupyter.py +++ b/jupyter/test_jupyter.py @@ -20,8 +20,8 @@ def verify_instance(self, name, jupyter_port): ("STANDARD", ["m"]), ) def test_jupyter(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): @@ -43,8 +43,8 @@ def test_jupyter(self, configuration, machine_suffixes): ("STANDARD", ["m"]), ) def test_jupyter_with_metadata(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): diff --git a/jupyter_sparkmonitor/test_jupyter_sparkmonitor.py b/jupyter_sparkmonitor/test_jupyter_sparkmonitor.py index 4ac60dc46..dcc91aefb 100644 --- a/jupyter_sparkmonitor/test_jupyter_sparkmonitor.py +++ b/jupyter_sparkmonitor/test_jupyter_sparkmonitor.py @@ -23,8 +23,8 @@ def verify_instance(self, name, jupyter_port): ("STANDARD", ["m"]), ) def test_sparkmonitor(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Use 1.4+ version of Dataproc to test because it requires Python 3 if self.getImageVersion() <= pkg_resources.parse_version("1.3"): diff --git a/kafka/test_kafka.py b/kafka/test_kafka.py index 4a119f8b8..a1569cd34 100644 --- a/kafka/test_kafka.py +++ b/kafka/test_kafka.py @@ -26,8 +26,8 @@ def __run_test_script(self, name): @parameterized.parameters( ("HA", ["m-0", "m-1", "m-2"]), ) def test_kafka(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster(configuration, self.INIT_ACTIONS) for machine_suffix in machine_suffixes: diff --git a/knox/test_knox.py b/knox/test_knox.py index 210709767..d048a3758 100644 --- a/knox/test_knox.py +++ b/knox/test_knox.py @@ -29,8 +29,8 @@ def _run_test_script(self, name, cert_type): ("STANDARD", ["m", "w-0"]), ("HA", ["m-2", "w-0"])) def test_knox_localhost_cert(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageOs() == 'debian': if self.getImageVersion() <= pkg_resources.parse_version("1.4"): @@ -55,8 +55,8 @@ def test_knox_localhost_cert(self, configuration, machine_suffixes): @parameterized.parameters(("STANDARD", ["w-0", "m"]), ("HA", ["m-1", "m-0"])) def test_knox_hostname_cert(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageOs() == 'debian': if self.getImageVersion() < pkg_resources.parse_version("1.5"): diff --git a/mlvm/test_mlvm.py b/mlvm/test_mlvm.py index 387b5c517..121bec6d0 100644 --- a/mlvm/test_mlvm.py +++ b/mlvm/test_mlvm.py @@ -87,8 +87,8 @@ def verify_all(self): ("STANDARD", "standalone"), ) def test_mlvm(self, configuration, dask_runtime): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Supported on Dataproc 2.0+ if self.getImageVersion() < pkg_resources.parse_version("2.0"): @@ -116,8 +116,8 @@ def test_mlvm(self, configuration, dask_runtime): ("STANDARD", "standalone", "DASK"), ) def test_mlvm_gpu(self, configuration, dask_runtime, rapids_runtime): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Supported on Dataproc 2.0+ if self.getImageVersion() < pkg_resources.parse_version("2.0"): diff --git a/oozie/test_oozie.py b/oozie/test_oozie.py index 3c3e13a5e..2fd5de855 100644 --- a/oozie/test_oozie.py +++ b/oozie/test_oozie.py @@ -29,8 +29,8 @@ def __run_test_file(self, name): ("HA", ["m-0", "m-1", "m-2"]), ) def test_oozie(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster( configuration, diff --git a/presto/test_presto.py b/presto/test_presto.py index 3f64e4039..b22e25928 100644 --- a/presto/test_presto.py +++ b/presto/test_presto.py @@ -86,8 +86,8 @@ def __verify_workers_count(self, name, workers, server_param=""): ) def test_presto(self, configuration, machine_suffixes, coordinators, workers): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): @@ -102,8 +102,8 @@ def test_presto(self, configuration, machine_suffixes, coordinators, @parameterized.parameters(("SINGLE", ["m"], 1, 0)) def test_presto_custom_port(self, configuration, machine_suffixes, coordinators, workers): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): diff --git a/ranger/test_ranger.py b/ranger/test_ranger.py index 4e27e34b9..9dcc326b3 100644 --- a/ranger/test_ranger.py +++ b/ranger/test_ranger.py @@ -32,8 +32,8 @@ def __run_test_script(self, name): ("HA", ["m-0"]), ) def test_ranger(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): diff --git a/rapids/README.md b/rapids/README.md index 3b98762dd..2993fa179 100644 --- a/rapids/README.md +++ b/rapids/README.md @@ -33,7 +33,7 @@ To use RAPIDS Accelerator For Apache Spark, XGBoost4j with Spark 3 * NVIDIA GPU driver 440.33+ * CUDA v11.0/v10.2/v10.1 * NCCL 2.4.7+ - * Ubuntu 18.04, Ubuntu 20.04 or CentOS 7, CentOS8 + * Ubuntu 18.04, Ubuntu 20.04 or Rocky Linux 7, Rocky Linux8 This section describes how to create [Google Cloud Dataproc](https://cloud.google.com/dataproc) cluster with diff --git a/rapids/test_rapids.py b/rapids/test_rapids.py index 0a0f358b4..a2bdc35b4 100644 --- a/rapids/test_rapids.py +++ b/rapids/test_rapids.py @@ -57,8 +57,8 @@ def verify_spark_job(self): ("STANDARD", ["m", "w-0"], GPU_P100, "yarn"), ("STANDARD", ["m"], GPU_P100, "standalone")) def test_rapids_dask(self, configuration, machine_suffixes, accelerator, dask_runtime): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageVersion() < pkg_resources.parse_version("2.0"): self.skipTest("Not supported in pre 2.0 images") @@ -84,8 +84,8 @@ def test_rapids_dask(self, configuration, machine_suffixes, accelerator, dask_ru ("SINGLE", ["m"], GPU_P100), ("STANDARD", ["w-0"], GPU_P100)) def test_rapids_spark(self, configuration, machine_suffixes, accelerator): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageVersion() <= pkg_resources.parse_version("1.4"): self.skipTest("Not supported in pre 1.5 images") @@ -115,8 +115,8 @@ def test_rapids_spark(self, configuration, machine_suffixes, accelerator): @parameterized.parameters( ("STANDARD", ["m", "w-0"], GPU_P100, "11.2")) def test_non_default_cuda_versions(self, configuration, machine_suffixes, accelerator, cuda_version): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") if self.getImageVersion() <= pkg_resources.parse_version("2.0"): self.skipTest("Not supported in pre 2.0 images") diff --git a/rstudio/test_rstudio.py b/rstudio/test_rstudio.py index c855991a4..f9730cfbd 100644 --- a/rstudio/test_rstudio.py +++ b/rstudio/test_rstudio.py @@ -15,8 +15,8 @@ class RStudioTestCase(DataprocTestCase): ("SINGLE", "", ""), # default username and no auth ) def test_rstudio(self, configuration, user, password): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") metadata = "rstudio-password={}".format(password) if user: diff --git a/solr/test_solr.py b/solr/test_solr.py index 741582fdf..609801de8 100644 --- a/solr/test_solr.py +++ b/solr/test_solr.py @@ -34,8 +34,8 @@ def __run_test_script(self, name): ("HA", ["m-0"]), ) def test_solr(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"): diff --git a/sqoop/test_sqoop.py b/sqoop/test_sqoop.py index 28bc3d6b5..e730e633a 100644 --- a/sqoop/test_sqoop.py +++ b/sqoop/test_sqoop.py @@ -44,8 +44,8 @@ def import_mysql_db(self, instance): ("HA", ["m-0", "m-1", "m-2"]), ) def test_sqoop(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster(configuration, self.INIT_ACTIONS) for machine_suffix in machine_suffixes: @@ -55,8 +55,8 @@ def test_sqoop(self, configuration, machine_suffixes): @parameterized.parameters(("SINGLE", ["m"])) def test_sqoop_import_from_local_mysql_to_hdfs(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster(configuration, self.INIT_ACTIONS) for machine_suffix in machine_suffixes: diff --git a/starburst-presto/test_starburst_presto.py b/starburst-presto/test_starburst_presto.py index b3da01498..8039167d2 100644 --- a/starburst-presto/test_starburst_presto.py +++ b/starburst-presto/test_starburst_presto.py @@ -85,8 +85,8 @@ def __verify_workers_count(self, name, workers, server_param=""): ) def test_starburst_presto(self, configuration, machine_suffixes, coordinators, workers): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster(configuration, self.INIT_ACTIONS) for machine_suffix in machine_suffixes: @@ -98,8 +98,8 @@ def test_starburst_presto(self, configuration, machine_suffixes, ("SINGLE", ["m"], 1, 0), ) def test_starburst_presto_custom_port( self, configuration, machine_suffixes, coordinators, workers): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") self.createCluster( configuration, diff --git a/tez/test_tez.py b/tez/test_tez.py index 8b30f6abe..9c0d0041b 100644 --- a/tez/test_tez.py +++ b/tez/test_tez.py @@ -39,8 +39,8 @@ def __run_test_script(self, name): ("HA", ["m-0", "m-1", "m-2"]), ) def test_tez(self, configuration, machine_suffixes): - if self.getImageOs() == 'centos': - self.skipTest("Not supported in CentOS-based images") + if self.getImageOs() == 'rocky': + self.skipTest("Not supported in Rocky Linux-based images") # Skip on 2.0+ version of Dataproc because it's not supported if self.getImageVersion() >= pkg_resources.parse_version("2.0"):