Fail deprecated initialization actions on Dataproc 2.0+ (#805)

GoogleCloudDataproc · Oct 20, 2020 · 28a3d7e · 28a3d7e
1 parent 4a29da5
commit 28a3d7e
Show file tree

Hide file tree

Showing 7 changed files with 44 additions and 2 deletions.
diff --git a/docker/README.md b/docker/README.md
@@ -1,3 +1,15 @@
+--------------------------------------------------------------------------------
+
+# NOTE: *The Docker initialization action has been deprecated. Please use the Docker Component*
+
+**The
+[Docker Component](https://cloud.google.com/dataproc/docs/concepts/components/docker)
+is the best way to use Docker with Cloud Dataproc. To learn more about
+Dataproc Components see
+[here](https://cloud.google.com/dataproc/docs/concepts/components/overview).**
+
+--------------------------------------------------------------------------------
+
 # Docker Initialization Action
 
 This initialization action installs a binary release of

diff --git a/docker/docker.sh b/docker/docker.sh
@@ -2,6 +2,10 @@
 
 set -euxo pipefail
 
+readonly NOT_SUPPORTED_MESSAGE="Docker initialization action is not supported on Dataproc ${DATAPROC_VERSION}.
+Use Docker Component instead: https://cloud.google.com/dataproc/docs/concepts/components/docker"
+[[ $DATAPROC_VERSION != 1.* ]] && echo "$NOT_SUPPORTED_MESSAGE" && exit 1
+
 readonly OS_ID=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
 readonly OS_CODE=$(lsb_release -cs)
 # TODO: Allow this to be configured by metadata.

diff --git a/flink/README.md b/flink/README.md
@@ -1,3 +1,15 @@
+--------------------------------------------------------------------------------
+
+# NOTE: *The Flink initialization action has been deprecated. Please use the Flink Component*
+
+**The
+[Flink Component](https://cloud.google.com/dataproc/docs/concepts/components/flink)
+is the best way to use Flink with Cloud Dataproc. To learn more about
+Dataproc Components see
+[here](https://cloud.google.com/dataproc/docs/concepts/components/overview).**
+
+--------------------------------------------------------------------------------
+
 # Apache Flink Initialization Action
 
 This initialization action installs a binary release of

diff --git a/flink/flink.sh b/flink/flink.sh
@@ -24,6 +24,10 @@
 
 set -euxo pipefail
 
+readonly NOT_SUPPORTED_MESSAGE="Flink initialization action is not supported on Dataproc ${DATAPROC_VERSION}.
+Use Flink Component instead: https://cloud.google.com/dataproc/docs/concepts/components/flink"
+[[ $DATAPROC_VERSION != 1.* ]] && echo "$NOT_SUPPORTED_MESSAGE" && exit 1
+
 # Use Python from /usr/bin instead of /opt/conda.
 export PATH=/usr/bin:$PATH
 

diff --git a/flink/test_flink.py b/flink/test_flink.py
@@ -1,4 +1,5 @@
 import os
+import pkg_resources
 
 from absl.testing import absltest
 from absl.testing import parameterized
@@ -29,6 +30,10 @@ def __run_test_file(self, name, yarn_session):
       ("HA", ["m-0", "m-1", "m-2"]),
   )
   def test_flink(self, configuration, machine_suffixes):
+    # Skip on 2.0+ version of Dataproc because it's not supported
+    if self.getImageVersion() >= pkg_resources.parse_version("2.0"):
+      self.skipTest("Not supported in 2.0+ images")
+
     self.createCluster(
         configuration,
         self.INIT_ACTIONS,
@@ -43,6 +48,10 @@ def test_flink(self, configuration, machine_suffixes):
       ("HA", ["m-0", "m-1", "m-2"]),
   )
   def test_flink_with_optional_metadata(self, configuration, machine_suffixes):
+    # Skip on 2.0+ version of Dataproc because it's not supported
+    if self.getImageVersion() >= pkg_resources.parse_version("2.0"):
+      self.skipTest("Not supported in 2.0+ images")
+
     self.createCluster(
         configuration,
         self.INIT_ACTIONS,

diff --git a/jupyter_sparkmonitor/README.md b/jupyter_sparkmonitor/README.md
@@ -4,7 +4,7 @@
 
 **The
 [Jupyter Component](https://cloud.google.com/dataproc/docs/concepts/components/jupyter)
-is the best way to use Jupyter with Cloud Dataproc. To learn more about
+is the best way to use Jupyter Spark Monitor with Cloud Dataproc. To learn more about
 Dataproc Components see
 [here](https://cloud.google.com/dataproc/docs/concepts/components/overview).**
 

diff --git a/jupyter_sparkmonitor/sparkmonitor.sh b/jupyter_sparkmonitor/sparkmonitor.sh
@@ -20,7 +20,8 @@
 
 set -euxo pipefail
 
-readonly NOT_SUPPORTED_MESSAGE="Jupyter Spark Monitor initialization action is not supported on Dataproc ${DATAPROC_VERSION}."
+readonly NOT_SUPPORTED_MESSAGE="Jupyter Spark Monitor initialization action is not supported on Dataproc ${DATAPROC_VERSION}.
+Use Jupyter Component instead: https://cloud.google.com/dataproc/docs/concepts/components/jupyter"
 [[ $DATAPROC_VERSION != 1.* ]] && echo "$NOT_SUPPORTED_MESSAGE" && exit 1
 
 source '/usr/local/share/google/dataproc/bdutil/bdutil_helpers.sh'