Skip to content

Commit

Permalink
[datalab] Also mount Spark metrics listener jar (#968)
Browse files Browse the repository at this point in the history
  • Loading branch information
Deependra-Patel authored Feb 9, 2022
1 parent c4fe482 commit fa8fd3e
Showing 1 changed file with 12 additions and 9 deletions.
21 changes: 12 additions & 9 deletions datalab/datalab.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,19 +41,22 @@ readonly INIT_ACTIONS_BRANCH="$(/usr/share/google/get_metadata_value attributes/
# Expose every possible spark configuration to the container.
VOLUMES="$(echo /etc/{hadoop*,hive*,*spark*})"

CONNECTORS_LIB=/usr/lib/hadoop/lib
DATAPROC_LIB=/usr/lib/hadoop/lib
if [[ -d /usr/local/share/google/dataproc/lib ]]; then
CONNECTORS_LIB="/usr/local/share/google/dataproc/lib"
DATAPROC_LIB="/usr/local/share/google/dataproc/lib"
fi
if [[ -L ${CONNECTORS_LIB}/gcs-connector.jar ]]; then
VOLUMES+=" ${CONNECTORS_LIB}/gcs-connector.jar"
if [[ -L ${DATAPROC_LIB}/gcs-connector.jar ]]; then
VOLUMES+=" ${DATAPROC_LIB}/gcs-connector.jar"
else
VOLUMES+=" $(compgen -G ${CONNECTORS_LIB}/gcs*)"
VOLUMES+=" $(compgen -G ${DATAPROC_LIB}/gcs*)"
fi
if [[ -L ${CONNECTORS_LIB}/bigquery-connector.jar ]]; then
VOLUMES+=" ${CONNECTORS_LIB}/bigquery-connector.jar"
elif compgen -G "${CONNECTORS_LIB}/bigquery*" >/dev/null; then
VOLUMES+=" $(compgen -G ${CONNECTORS_LIB}/bigquery*)"
if [[ -L ${DATAPROC_LIB}/bigquery-connector.jar ]]; then
VOLUMES+=" ${DATAPROC_LIB}/bigquery-connector.jar"
elif compgen -G "${DATAPROC_LIB}/bigquery*" >/dev/null; then
VOLUMES+=" $(compgen -G ${DATAPROC_LIB}/bigquery*)"
fi
if [[ -L ${DATAPROC_LIB}/spark-metrics-listener.jar ]]; then
VOLUMES+=" ${DATAPROC_LIB}/spark-metrics-listener.jar"
fi

readonly VOLUMES
Expand Down

0 comments on commit fa8fd3e

Please sign in to comment.