Skip to content

Commit

Permalink
Migrate all init actions to Rocky Linux from deprecated CentOS (#989)
Browse files Browse the repository at this point in the history
  • Loading branch information
@medb
medb authored May 2, 2022
1 parent 0860e1e commit da698e2
Show file tree
Hide file tree
Showing 31 changed files with 118 additions and 118 deletions.
8 changes: 4 additions & 4 deletions alluxio/test_alluxio.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ def verify_instance(self, name):
@parameterized.parameters(
("STANDARD", ["m"]),)
def test_alluxio(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

self.createCluster(
configuration,
Expand All @@ -32,8 +32,8 @@ def test_alluxio(self, configuration, machine_suffixes):
@parameterized.parameters(
("STANDARD", ["m"]),)
def test_alluxio_with_presto(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

self.createCluster(
configuration,
Expand Down
16 changes: 8 additions & 8 deletions atlas/test_atlas.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ def verify_instance(self, instance, username='admin', password='admin'):
("HA", ["m-0", "m-1", "m-2"]),
)
def test_atlas(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

image_version = self.getImageVersion()
if image_version < pkg_resources.parse_version("1.5") or \
Expand Down Expand Up @@ -147,8 +147,8 @@ def test_atlas(self, configuration, machine_suffixes):
@parameterized.parameters(("SINGLE", ["m"]))
def test_atlas_overrides_admin_credentials(self, configuration,
machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

image_version = self.getImageVersion()
if image_version < pkg_resources.parse_version("1.5") or \
Expand All @@ -175,8 +175,8 @@ def test_atlas_overrides_admin_credentials(self, configuration,

@parameterized.parameters("ZOOKEEPER", "HBASE", "SOLR")
def test_atlas_fails_without_component(self, component):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

image_version = self.getImageVersion()
if image_version < pkg_resources.parse_version("1.5") or \
Expand All @@ -193,8 +193,8 @@ def test_atlas_fails_without_component(self, component):
optional_components=self.OPTIONAL_COMPONENTS.remove(component))

def test_atlas_ha_fails_without_kafka(self):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

if self.getImageVersion() < pkg_resources.parse_version("1.5"):
self.skipTest("Not supported in pre 1.5 images")
Expand Down
4 changes: 2 additions & 2 deletions cloud-sql-proxy/test_cloud_sql_proxy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ def __submit_pyspark_job(self, cluster_name):
'HA',
)
def test_cloud_sql_proxy(self, configuration):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

metadata = 'hive-metastore-instance={}:{}'.format(self.PROJECT_METADATA,
self.DB_NAME)
Expand Down
4 changes: 2 additions & 2 deletions conda/test_conda.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ def _parse_packages(stdout):
("STANDARD", CONDA_PKGS, PIP_PKGS),
)
def test_conda(self, configuration, conda_packages, pip_packages):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

# Skip on 2.0+ version of Dataproc because it's not supported
if self.getImageVersion() >= pkg_resources.parse_version("2.0"):
Expand Down
16 changes: 8 additions & 8 deletions connectors/test_connectors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ def _scala_version(self):
@parameterized.parameters(("SINGLE", ["m"]),
("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]))
def test_bq_connector_version(self, configuration, instances):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

self.createCluster(configuration,
self.INIT_ACTIONS,
Expand All @@ -68,8 +68,8 @@ def test_bq_connector_version(self, configuration, instances):
@parameterized.parameters(("SINGLE", ["m"]),
("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]))
def test_spark_bq_connector_version(self, configuration, instances):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

self.createCluster(
configuration,
Expand All @@ -83,8 +83,8 @@ def test_spark_bq_connector_version(self, configuration, instances):
@parameterized.parameters(("SINGLE", ["m"]),
("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]))
def test_bq_connector_url(self, configuration, instances):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

self.createCluster(configuration,
self.INIT_ACTIONS,
Expand All @@ -97,8 +97,8 @@ def test_bq_connector_url(self, configuration, instances):
@parameterized.parameters(("SINGLE", ["m"]),
("HA", ["m-0", "m-1", "m-2", "w-0", "w-1"]))
def test_spark_bq_connector_url(self, configuration, instances):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

self.createCluster(configuration,
self.INIT_ACTIONS,
Expand Down
4 changes: 2 additions & 2 deletions dask/test_dask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ def _run_dask_test_script(self, name, script):
("STANDARD", ["m", "w-0"], "yarn"),
("STANDARD", ["m"], "standalone"))
def test_dask(self, configuration, instances, runtime):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

if self.getImageVersion() <= pkg_resources.parse_version("1.4"):
self.skipTest("Not supported in pre 1.5 images")
Expand Down
4 changes: 2 additions & 2 deletions datalab/test_datalab.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ def verify_instance(self, name):
("STANDARD", ["m"], "python3"),
)
def test_datalab(self, configuration, machine_suffixes, python):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

# Skip on 2.0+ version of Dataproc because it's not supported
if self.getImageVersion() >= pkg_resources.parse_version("2.0"):
Expand Down
4 changes: 2 additions & 2 deletions drill/test_drill.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ def __run_bash_test_file(self, name, drill_mode, target_node):
("HA", [("m-0", "w-0"), ("w-0", "m-1")]),
)
def test_drill(self, configuration, verify_options):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

if self.getImageVersion() >= pkg_resources.parse_version("2.0"):
self.skipTest("Not supported in the 2.0+ images")
Expand Down
8 changes: 4 additions & 4 deletions flink/test_flink.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ def __run_test_file(self, name, yarn_session):
("HA", ["m-0", "m-1", "m-2"]),
)
def test_flink(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

# Skip on 2.0+ version of Dataproc because it's not supported
if self.getImageVersion() >= pkg_resources.parse_version("2.0"):
Expand All @@ -51,8 +51,8 @@ def test_flink(self, configuration, machine_suffixes):
("HA", ["m-0", "m-1", "m-2"]),
)
def test_flink_with_optional_metadata(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

# Skip on 2.0+ version of Dataproc because it's not supported
if self.getImageVersion() >= pkg_resources.parse_version("2.0"):
Expand Down
4 changes: 2 additions & 2 deletions ganglia/test_ganglia.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ def verify_instance(self, name):
("HA", ["m-0", "m-1", "m-2", "w-0"]),
)
def test_ganglia(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

self.createCluster(configuration, self.INIT_ACTIONS)
for machine_suffix in machine_suffixes:
Expand Down
22 changes: 11 additions & 11 deletions gpu/install_gpu_driver.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ NCCL_REPO_URL=$(get_metadata_attribute 'nccl-repo-url' "${DEFAULT_NCCL_REPO_URL}
readonly NCCL_REPO_URL

readonly DEFAULT_NCCL_VERSION="2.8.3"
readonly DEFAULT_NCCL_VERSION_CENTOS="2.8.4"
readonly DEFAULT_NCCL_VERSION_ROCKY="2.8.4"

if [[ ${OS_NAME} == centos ]]; then
NCCL_VERSION=$(get_metadata_attribute 'nccl-version' ${DEFAULT_NCCL_VERSION_CENTOS})
if [[ ${OS_NAME} == rocky ]]; then
NCCL_VERSION=$(get_metadata_attribute 'nccl-version' ${DEFAULT_NCCL_VERSION_ROCKY})
else
NCCL_VERSION=$(get_metadata_attribute 'nccl-version' ${DEFAULT_NCCL_VERSION})
fi
Expand All @@ -71,8 +71,8 @@ readonly NVIDIA_UBUNTU_REPOSITORY_URL="${NVIDIA_BASE_DL_URL}/cuda/repos/ubuntu18
readonly NVIDIA_UBUNTU_REPOSITORY_KEY="${NVIDIA_UBUNTU_REPOSITORY_URL}/7fa2af80.pub"
readonly NVIDIA_UBUNTU_REPOSITORY_CUDA_PIN="${NVIDIA_UBUNTU_REPOSITORY_URL}/cuda-ubuntu1804.pin"

# Parameter for NVIDIA-provided Centos GPU driver
readonly NVIDIA_CENTOS_REPOSITORY_URL="${NVIDIA_BASE_DL_URL}/cuda/repos/rhel8/x86_64/cuda-rhel8.repo"
# Parameter for NVIDIA-provided Rocky Linux GPU driver
readonly NVIDIA_ROCKY_REPOSITORY_URL="${NVIDIA_BASE_DL_URL}/cuda/repos/rhel8/x86_64/cuda-rhel8.repo"

# Parameters for NVIDIA-provided CUDNN library
readonly CUDNN_VERSION=$(get_metadata_attribute 'cudnn-version' '')
Expand Down Expand Up @@ -108,7 +108,7 @@ function execute_with_retries() {
function install_nvidia_nccl() {
local -r nccl_version="${NCCL_VERSION}-1+cuda${CUDA_VERSION}"

if [[ ${OS_NAME} == centos ]]; then
if [[ ${OS_NAME} == rocky ]]; then
execute_with_retries "dnf -y -q install libnccl-${nccl_version} libnccl-devel-${nccl_version} libnccl-static-${nccl_version}"
elif [[ ${OS_NAME} == ubuntu ]] || [[ ${OS_NAME} == debian ]]; then
local tmp_dir
Expand All @@ -134,7 +134,7 @@ function install_nvidia_cudnn() {
local cudnn_pkg_version
cudnn_pkg_version="${CUDNN_VERSION}-1+cuda${CUDA_VERSION}"

if [[ ${OS_NAME} == centos ]]; then
if [[ ${OS_NAME} == rocky ]]; then
if [[ ${major_version} == 8 ]]; then
execute_with_retries "dnf -y -q install libcudnn8-${cudnn_pkg_version} libcudnn8-devel-${cudnn_pkg_version}"
else
Expand Down Expand Up @@ -196,8 +196,8 @@ function install_nvidia_gpu_driver() {
# Without --no-install-recommends this takes a very long time.
execute_with_retries "apt-get install -y -q --no-install-recommends cuda-drivers-460"
execute_with_retries "apt-get install -y -q --no-install-recommends ${cuda_package}"
elif [[ ${OS_NAME} == centos ]]; then
execute_with_retries "dnf config-manager --add-repo ${NVIDIA_CENTOS_REPOSITORY_URL}"
elif [[ ${OS_NAME} == rocky ]]; then
execute_with_retries "dnf config-manager --add-repo ${NVIDIA_ROCKY_REPOSITORY_URL}"
execute_with_retries "dnf clean all"
execute_with_retries "dnf -y -q module install nvidia-driver:460-dkms"
execute_with_retries "dnf -y -q install cuda-${CUDA_VERSION//./-}"
Expand Down Expand Up @@ -334,7 +334,7 @@ EOF
}

function main() {
if [[ ${OS_NAME} != debian ]] && [[ ${OS_NAME} != ubuntu ]] && [[ ${OS_NAME} != centos ]]; then
if [[ ${OS_NAME} != debian ]] && [[ ${OS_NAME} != ubuntu ]] && [[ ${OS_NAME} != rocky ]]; then
echo "Unsupported OS: '${OS_NAME}'"
exit 1
fi
Expand All @@ -343,7 +343,7 @@ function main() {
export DEBIAN_FRONTEND=noninteractive
execute_with_retries "apt-get update"
execute_with_retries "apt-get install -y -q pciutils"
elif [[ ${OS_NAME} == centos ]] ; then
elif [[ ${OS_NAME} == rocky ]] ; then
execute_with_retries "dnf -y -q update"
execute_with_retries "dnf -y -q install pciutils"
execute_with_retries "dnf -y -q install kernel-devel"
Expand Down
20 changes: 10 additions & 10 deletions gpu/test_gpu.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ def verify_instance_cudnn(self, name):
def test_install_gpu_default_agent(self, configuration, machine_suffixes,
master_accelerator, worker_accelerator,
driver_provider):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

metadata = None
if driver_provider is not None:
Expand All @@ -56,8 +56,8 @@ def test_install_gpu_default_agent(self, configuration, machine_suffixes,
def test_install_gpu_without_agent(self, configuration, machine_suffixes,
master_accelerator, worker_accelerator,
driver_provider):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

metadata = "install-gpu-agent=false"
if driver_provider is not None:
Expand All @@ -82,8 +82,8 @@ def test_install_gpu_without_agent(self, configuration, machine_suffixes,
def test_install_gpu_with_agent(self, configuration, machine_suffixes,
master_accelerator, worker_accelerator,
driver_provider):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

# GPU agent supported on Dataproc 1.4+
if self.getImageVersion() < pkg_resources.parse_version("1.4"):
Expand Down Expand Up @@ -117,8 +117,8 @@ def test_install_gpu_with_agent(self, configuration, machine_suffixes,
def test_install_gpu_cuda_nvidia(self, configuration, machine_suffixes,
master_accelerator, worker_accelerator,
cuda_version):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

metadata = "gpu-driver-provider=NVIDIA,cuda-version={}".format(cuda_version)
self.createCluster(
Expand All @@ -139,8 +139,8 @@ def test_install_gpu_cuda_nvidia(self, configuration, machine_suffixes,
def test_install_gpu_with_cudnn(self, configuration, machine_suffixes,
master_accelerator, worker_accelerator,
driver_provider, cudnn_version):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

metadata = "cudnn-version={}".format(cudnn_version)
if driver_provider is not None:
Expand Down
8 changes: 4 additions & 4 deletions hbase/test_hbase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ def verify_instance(self, name):
("HA", ["m-0"]),
)
def test_hbase(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

# Skip on 2.0+ version of Dataproc because it's not supported
if self.getImageVersion() >= pkg_resources.parse_version("2.0"):
Expand All @@ -56,8 +56,8 @@ def test_hbase(self, configuration, machine_suffixes):
("HA", ["m-0"]),
)
def test_hbase_on_gcs(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

# Skip on 2.0+ version of Dataproc because it's not supported
if self.getImageVersion() >= pkg_resources.parse_version("2.0"):
Expand Down
4 changes: 2 additions & 2 deletions hive-hcatalog/test_hive_hcatalog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,8 @@ def __submit_hive_job(self, cluster_name, job, should_repeat_job=False):
("HA", True),
)
def test_hive_hcatalog(self, configuration, should_repeat_job):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

self.createCluster(configuration, self.INIT_ACTIONS)
self.verify_instance(self.getClusterName(), should_repeat_job)
Expand Down
12 changes: 6 additions & 6 deletions hive-llap/test_hive_llap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ def __run_test_script(self, name):
("HA", ["m-0"]),
("STANDARD", ["m"]))
def test_llap(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

if self.getImageVersion() < pkg_resources.parse_version("2.0"):
self.skipTest("Hive LLAP only supported on Dataproc 2.0+")
Expand All @@ -52,8 +52,8 @@ def test_llap(self, configuration, machine_suffixes):
("STANDARD", ["m"]),
("HA", ["m-0"]))
def test_llap_num_exec(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

if self.getImageVersion() < pkg_resources.parse_version("2.0"):
self.skipTest("Hive LLAP only supported on Dataproc 2.0+")
Expand All @@ -72,8 +72,8 @@ def test_llap_num_exec(self, configuration, machine_suffixes):
("STANDARD", ["m"]),
("HA", ["m-0"]))
def test_llap_exec_size(self, configuration, machine_suffixes):
if self.getImageOs() == 'centos':
self.skipTest("Not supported in CentOS-based images")
if self.getImageOs() == 'rocky':
self.skipTest("Not supported in Rocky Linux-based images")

if self.getImageVersion() < pkg_resources.parse_version("2.0"):
self.skipTest("Hive LLAP only supported on Dataproc 2.0+")
Expand Down
Loading

0 comments on commit da698e2

Please sign in to comment.