Standardize wget command parameters (#659)

GoogleCloudDataproc · Oct 22, 2019 · 55c6fbd · 55c6fbd
1 parent f3ab2d7
commit 55c6fbd
Show file tree

Hide file tree

Showing 15 changed files with 52 additions and 36 deletions.
diff --git a/bigdl/bigdl.sh b/bigdl/bigdl.sh
@@ -14,7 +14,7 @@ BIGDL_DOWNLOAD_URL=$(/usr/share/google/get_metadata_value attributes/bigdl-downl
 mkdir -p /opt/intel-bigdl
 cd /opt/intel-bigdl
 
-wget "${BIGDL_DOWNLOAD_URL}"
+wget -nv --timeout=30 --tries=5 --retry-connrefused "${BIGDL_DOWNLOAD_URL}"
 unzip *.zip
 
 JAR=`realpath lib/*.jar`

diff --git a/bigtable/bigtable.sh b/bigtable/bigtable.sh
@@ -54,15 +54,15 @@ function err() {
 
 function install_big_table_client() {
   local out="${HBASE_HOME}/lib/${BIGTABLE_HBASE_CLIENT}"
-  wget --progress=dot:mega -q "${BIGTABLE_HBASE_DL_LINK}" -O "${out}" ||
-    err 'Unable to install BigTable client libs.'
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    "${BIGTABLE_HBASE_DL_LINK}" -O "${out}"
 }
 
 function install_shc() {
   mkdir -p "/usr/lib/spark/external"
   local out="/usr/lib/spark/external/${SPARK_HBASE_CLIENT}"
-  wget --progress=dot:mega -q "${SPARK_HBASE_CLIENT_DL_LINK}" -O ${out} ||
-    err 'Unable to install shc.'
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    "${SPARK_HBASE_CLIENT_DL_LINK}" -O ${out}
   ln -s "${out}" "/usr/lib/spark/external/shc-core.jar"
 }
 

diff --git a/cloud-sql-proxy/cloud-sql-proxy.sh b/cloud-sql-proxy/cloud-sql-proxy.sh
@@ -189,8 +189,8 @@ function configure_proxy_flags() {
 
 function install_cloud_sql_proxy() {
   # Install proxy.
-  wget -q https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
-    || err 'Unable to download cloud-sql-proxy binary'
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64
   mv cloud_sql_proxy.linux.amd64 ${PROXY_BIN}
   chmod +x ${PROXY_BIN}
 

diff --git a/conda/bootstrap-conda.sh b/conda/bootstrap-conda.sh
@@ -84,7 +84,8 @@ else
       echo "Found existing Miniconda script at: $MINICONDA_SCRIPT_PATH"
     else
       echo "Downloading Miniconda script to: $MINICONDA_SCRIPT_PATH ..."
-      wget https://repo.continuum.io/miniconda/$MINICONDA_FULL_NAME -P "$PROJ_DIR"
+      wget -nv --timeout=30 --tries=5 --retry-connrefused \
+        https://repo.continuum.io/miniconda/$MINICONDA_FULL_NAME -P "$PROJ_DIR"
       echo "Downloaded $MINICONDA_FULL_NAME!"
       ls -al $MINICONDA_SCRIPT_PATH
       chmod 755 $MINICONDA_SCRIPT_PATH

diff --git a/dr-elephant/dr-elephant.sh b/dr-elephant/dr-elephant.sh
@@ -17,7 +17,8 @@ function err() {
 
 function build() {
   # Download and install Typesafe Activator
-  wget --progress=dot:mega -O /tmp/typesafe-activator.zip ${TYPESAFE_ACTIVATOR_URL}
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    ${TYPESAFE_ACTIVATOR_URL} -O /tmp/typesafe-activator.zip
   unzip -q /tmp/typesafe-activator.zip -d /tmp/
   mv /tmp/activator-dist-* /tmp/typesafe-activator
   export PATH=${PATH}:/tmp/typesafe-activator/bin/

diff --git a/drill/drill.sh b/drill/drill.sh
@@ -242,8 +242,8 @@ function main() {
   mkdir -p ${DRILL_HOME} && chown ${DRILL_USER}:${DRILL_USER} ${DRILL_HOME}
 
   # Download and unpack Drill as the pseudo-user.
-  wget https://archive.apache.org/dist/drill/drill-${DRILL_VERSION}/apache-drill-${DRILL_VERSION}.tar.gz || \
-    err "Unable to download archive"
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    https://archive.apache.org/dist/drill/drill-${DRILL_VERSION}/apache-drill-${DRILL_VERSION}.tar.gz
   tar -xvzf apache-drill-${DRILL_VERSION}.tar.gz -C ${DRILL_HOME} --strip 1
 
   # Replace default configuration with cluster-specific.

diff --git a/gpu/install_gpu_driver.sh b/gpu/install_gpu_driver.sh
@@ -93,8 +93,10 @@ function install_gpu_agent_service() {
   fi
   local install_dir=/opt/gpu_utilization_agent
   mkdir "${install_dir}"
-  wget -O "${install_dir}/requirements.txt" "${GPU_AGENT_REPO_URL}/requirements.txt"
-  wget -O "${install_dir}/report_gpu_metrics.py" "${GPU_AGENT_REPO_URL}/report_gpu_metrics.py"
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    "${GPU_AGENT_REPO_URL}/requirements.txt" -P "${install_dir}"
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    "${GPU_AGENT_REPO_URL}/report_gpu_metrics.py" -P "${install_dir}"
   pip install -r "${install_dir}/requirements.txt"
 
   # Generate GPU service.

diff --git a/livy/livy.sh b/livy/livy.sh
@@ -73,8 +73,8 @@ function main() {
   # Download Livy binary.
   local temp
   temp=$(mktemp -d)
-  wget --progress=dot:mega --timeout=30 --tries=5 --retry-connrefused \
-    -O "${temp}/livy.zip" "${APACHE_MIRROR}=${PKG_PATH}"
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    "${APACHE_MIRROR}=${PKG_PATH}" -O "${temp}/livy.zip"
   unzip -q "${temp}/livy.zip" -d "${temp}"
 
   # Create Livy user.

diff --git a/oozie/oozie.sh b/oozie/oozie.sh
@@ -77,7 +77,8 @@ function install_oozie(){
 
   if [[ ${node_name} == ${master_node} ]]; then
     # The ext library is needed to enable the Oozie web console
-    wget http://archive.cloudera.com/gplextras/misc/ext-2.2.zip || err 'Unable to download ext-2.2.zip'
+    wget -nv --timeout=30 --tries=5 --retry-connrefused \
+      http://archive.cloudera.com/gplextras/misc/ext-2.2.zip
     unzip ext-2.2.zip -d /var/lib/oozie
     # Install share lib
     install -d /usr/lib/oozie

diff --git a/presto/presto.sh b/presto/presto.sh
@@ -56,7 +56,8 @@ function wait_for_presto_cluster_ready() {
 
 function get_presto() {
   # Download and unpack Presto server
-  wget ${PRESTO_BASE_URL}/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    ${PRESTO_BASE_URL}/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz
   tar -zxvf presto-server-${PRESTO_VERSION}.tar.gz
   mkdir -p /var/presto/data
 }
@@ -190,7 +191,8 @@ discovery.uri=http://${PRESTO_MASTER_FQDN}:${HTTP_PORT}
 EOF
 
   # Install cli
-  wget ${PRESTO_BASE_URL}/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar -O /usr/bin/presto
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    ${PRESTO_BASE_URL}/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar -O /usr/bin/presto
   chmod a+x /usr/bin/presto
 }
 

diff --git a/prometheus/prometheus.sh b/prometheus/prometheus.sh
@@ -42,12 +42,13 @@ function is_kafka_installed {
 }
 
 function install_prometheus {
-  mkdir -p /etc/prometheus /var/lib/prometheus 
-  wget -q "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VER}/prometheus-${PROMETHEUS_VER}.linux-amd64.tar.gz" && \
-  tar -xf "prometheus-${PROMETHEUS_VER}.linux-amd64.tar.gz" && \
-  cp "prometheus-${PROMETHEUS_VER}.linux-amd64/prometheus" /usr/local/bin/ && \
-  cp "prometheus-${PROMETHEUS_VER}.linux-amd64/promtool" /usr/local/bin/ && \
-  cp -r "prometheus-${PROMETHEUS_VER}.linux-amd64/consoles" /etc/prometheus && \
+  mkdir -p /etc/prometheus /var/lib/prometheus
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VER}/prometheus-${PROMETHEUS_VER}.linux-amd64.tar.gz"
+  tar -xf "prometheus-${PROMETHEUS_VER}.linux-amd64.tar.gz"
+  cp "prometheus-${PROMETHEUS_VER}.linux-amd64/prometheus" /usr/local/bin/
+  cp "prometheus-${PROMETHEUS_VER}.linux-amd64/promtool" /usr/local/bin/
+  cp -r "prometheus-${PROMETHEUS_VER}.linux-amd64/consoles" /etc/prometheus
   cp -r "prometheus-${PROMETHEUS_VER}.linux-amd64/console_libraries" /etc/prometheus
 
   rm -rf "prometheus-${PROMETHEUS_VER}.linux-amd64.tar.gz" "prometheus-${PROMETHEUS_VER}.linux-amd64"
@@ -100,12 +101,13 @@ EOF
 
 function install_statsd_exporter {
   mkdir -p /var/lib/statsd
-  wget -q "https://github.com/prometheus/statsd_exporter/releases/download/v${STATSD_EXPORTER_VER}/statsd_exporter-${STATSD_EXPORTER_VER}.linux-amd64.tar.gz" && \
-    tar -xf "statsd_exporter-${STATSD_EXPORTER_VER}.linux-amd64.tar.gz" && \
-    cp "statsd_exporter-${STATSD_EXPORTER_VER}.linux-amd64/statsd_exporter" /var/lib/statsd/statsd_exporter
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    "https://github.com/prometheus/statsd_exporter/releases/download/v${STATSD_EXPORTER_VER}/statsd_exporter-${STATSD_EXPORTER_VER}.linux-amd64.tar.gz"
+  tar -xf "statsd_exporter-${STATSD_EXPORTER_VER}.linux-amd64.tar.gz"
+  cp "statsd_exporter-${STATSD_EXPORTER_VER}.linux-amd64/statsd_exporter" /var/lib/statsd/statsd_exporter
 
   rm -rf "statsd_exporter-${STATSD_EXPORTER_VER}.linux-amd64.tar.gz" "statsd_exporter-${STATSD_EXPORTER_VER}.linux-amd64"
- 
+
 cat << EOF > /etc/systemd/system/statsd-exporter.service
 [Unit]
 Description=Statsd
@@ -117,16 +119,18 @@ User=root
 Group=root
 Type=simple
 ExecStart=/var/lib/statsd/statsd_exporter \
-    --web.listen-address=:9102 
+    --web.listen-address=:9102
 
 [Install]
 WantedBy=multi-user.target
 EOF
 }
 
 function install_jmx_exporter {
-  wget "${KAFKA_JMX_JAVAAGENT_URI}" -P "${KAFKA_LIBS_DIR}" -nv
-  wget "${KAFKA_JMX_EXPORTER_CONFIG_URI}" -P "${KAFKA_CONFIG_DIR}" -nv
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+     "${KAFKA_JMX_JAVAAGENT_URI}" -P "${KAFKA_LIBS_DIR}"
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+     "${KAFKA_JMX_EXPORTER_CONFIG_URI}" -P "${KAFKA_CONFIG_DIR}"
   sed -i "/kafka-run-class.sh/i export KAFKA_OPTS=\"\${KAFKA_OPTS} -javaagent:${KAFKA_LIBS_DIR}/${KAFKA_JMX_JAVAAGENT_NAME}=${KAFKA_JMX_EXPORTER_PORT}:${KAFKA_CONFIG_DIR}/${KAFKA_JMX_EXPORTER_CONFIG_NAME}\"" \
         /usr/lib/kafka/bin/kafka-server-start.sh
 }

diff --git a/rapids/internal/install-gpu-driver.sh b/rapids/internal/install-gpu-driver.sh
@@ -9,7 +9,8 @@ readonly GPU_DRIVER_URL=$(/usr/share/google/get_metadata_value attributes/gpu-dr
 apt-get update
 DEBIAN_FRONTEND=noninteractive apt-get install -y pciutils "linux-headers-$(uname -r)"
 
-wget --progress=dot:mega -O driver.run "${GPU_DRIVER_URL}"
+wget -nv --timeout=30 --tries=5 --retry-connrefused \
+  "${GPU_DRIVER_URL}" -O driver.run
 chmod +x "./driver.run"
 
 "./driver.run" --silent
diff --git a/rstudio/rstudio.sh b/rstudio/rstudio.sh
@@ -95,7 +95,8 @@ if [[ "${ROLE}" == 'Master' ]]; then
   update_apt_get
   apt install -y r-base r-base-dev gdebi-core
 
-  cd /tmp; wget https://download2.rstudio.org/${rstudio_server_package}
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    https://download2.rstudio.org/${rstudio_server_package} -P /tmp
   gdebi -n /tmp/${rstudio_server_package}
 
   if ! [ $(getent group "${USER_NAME}") ]; then

diff --git a/solr/verify_solr.py b/solr/verify_solr.py
@@ -61,7 +61,8 @@ def use_api_to_create_catch_all_rule():
 
 def post_test_data():
     ret_code, stdout, stderr = run_command(
-        "wget -q {} -O /tmp/films.json".format(SOLR_EXAMPLE_DOC))
+        "wget -nv --timeout=30 --tries=5 --retry-connrefused {} -O /tmp/films.json"
+        .format(SOLR_EXAMPLE_DOC))
     assert ret_code == 0, "Failed to get test data. Error: {}".format(stderr)
 
     ret_code, stdout, stderr = run_command(

diff --git a/starburst-presto/presto.sh b/starburst-presto/presto.sh
@@ -49,7 +49,8 @@ function wait_for_presto_cluster_ready() {
 
 function get_presto(){
   # Download and unpack Presto server
-  wget "https://storage.googleapis.com/starburstdata/presto/${PRESTO_MAJOR_VERSION}e/${STARBURST_PRESTO_VERSION}/presto-server-${STARBURST_PRESTO_VERSION}.tar.gz" -O "presto-server-${STARBURST_PRESTO_VERSION}.tar.gz"
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    "https://storage.googleapis.com/starburstdata/presto/${PRESTO_MAJOR_VERSION}e/${STARBURST_PRESTO_VERSION}/presto-server-${STARBURST_PRESTO_VERSION}.tar.gz"
   tar -zxvf presto-server-${STARBURST_PRESTO_VERSION}.tar.gz
   ln -s "presto-server-${STARBURST_PRESTO_VERSION}" "presto-server"
   mkdir -p /var/presto/data
@@ -182,7 +183,8 @@ discovery-server.enabled=true
 discovery.uri=http://${PRESTO_MASTER_FQDN}:${HTTP_PORT}
 EOF
   # Install cli
-  wget "https://storage.googleapis.com/starburstdata/presto/${PRESTO_MAJOR_VERSION}e/${STARBURST_PRESTO_VERSION}/presto-cli-${STARBURST_PRESTO_VERSION}-executable.jar" -O /usr/bin/presto
+  wget -nv --timeout=30 --tries=5 --retry-connrefused \
+    "https://storage.googleapis.com/starburstdata/presto/${PRESTO_MAJOR_VERSION}e/${STARBURST_PRESTO_VERSION}/presto-cli-${STARBURST_PRESTO_VERSION}-executable.jar" -O /usr/bin/presto
   chmod a+x /usr/bin/presto
 }