Remove ambiguous SPARK_HADOOP_VERSION

apache · Feb 21, 2022 · 3b33ee5 · 3b33ee5
1 parent 0e6367a
commit 3b33ee5
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 51 deletions.
diff --git a/bin/load-kyuubi-env.sh b/bin/load-kyuubi-env.sh
@@ -71,32 +71,11 @@ fi
 export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.12"}"
 
 if [[ -f ${KYUUBI_HOME}/RELEASE ]]; then
-  KYUUBI_VERSION="$(grep "Kyuubi " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $2}')"
-  FLINK_VERSION_BUILD="$(grep "Flink " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $2}')"
-  SPARK_VERSION_BUILD="$(grep "Spark " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $2}' | grep -v 'Hadoop')"
-  HADOOP_VERSION_BUILD="$(grep "Spark Hadoop " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $3}')"
-  FLINK_BUILTIN="${KYUUBI_HOME}/externals/flink-$FLINK_VERSION_BUILD"
-  SPARK_BUILTIN="${KYUUBI_HOME}/externals/spark-$SPARK_VERSION_BUILD-bin-hadoop${HADOOP_VERSION_BUILD:0:3}"
+  FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals" -name 'flink-*' -type d | head -n 1)"
+  SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals" -name 'spark-*' -type d | head -n 1)"
 else
-  MVN="${MVN:-"${KYUUBI_HOME}/build/mvn"}"
-  KYUUBI_VERSION=$("$MVN" help:evaluate -Dexpression=project.version 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-  FLINK_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=flink.version 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-  SPARK_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=spark.version 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-  HADOOP_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=hadoop.binary.version 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-  FLINK_BUILTIN="${KYUUBI_HOME}/externals/kyuubi-download/target/flink-$FLINK_VERSION_BUILD"
-  SPARK_BUILTIN="${KYUUBI_HOME}/externals/kyuubi-download/target/spark-$SPARK_VERSION_BUILD-bin-hadoop${HADOOP_VERSION_BUILD}"
+  FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'flink-*' -type d | head -n 1)"
+  SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'spark-*' -type d | head -n 1)"
 fi
 
 export FLINK_HOME="${FLINK_HOME:-"${FLINK_BUILTIN}"}"
@@ -109,7 +88,6 @@ if [ $silent -eq 0 ]; then
   echo "JAVA_HOME: ${JAVA_HOME}"
 
   echo "KYUUBI_HOME: ${KYUUBI_HOME}"
-  echo "KYUUBI_VERSION: ${KYUUBI_VERSION}"
   echo "KYUUBI_CONF_DIR: ${KYUUBI_CONF_DIR}"
   echo "KYUUBI_LOG_DIR: ${KYUUBI_LOG_DIR}"
   echo "KYUUBI_PID_DIR: ${KYUUBI_PID_DIR}"

diff --git a/build/dist b/build/dist
@@ -153,11 +153,6 @@ SPARK_VERSION=$("$MVN" help:evaluate -Dexpression=spark.version $@ 2>/dev/null\
     | grep -v "WARNING"\
     | tail -n 1)
 
-SPARK_HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.binary.version $@ 2>/dev/null\
-    | grep -v "INFO"\
-    | grep -v "WARNING"\
-    | tail -n 1)
-
 HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\
     | grep -v "INFO"\
     | grep -v "WARNING"\
@@ -175,7 +170,7 @@ if [[ "$NAME" == "none" ]]; then
   if [[ "$SPARK_PROVIDED" == "true" ]]; then
     SUFFIX=""
   else
-    SUFFIX="-spark-${SPARK_VERSION:0:3}-hadoop${SPARK_HADOOP_VERSION}"
+    SUFFIX="-spark-${SPARK_VERSION:0:3}"
   fi
 fi
 
@@ -219,7 +214,6 @@ echo "Java $JAVA_VERSION" >> "$DISTDIR/RELEASE"
 echo "Scala $SCALA_VERSION" >> "$DISTDIR/RELEASE"
 echo "Flink $FLINK_VERSION" >> "$DISTDIR/RELEASE"
 echo "Spark $SPARK_VERSION" >> "$DISTDIR/RELEASE"
-echo "Spark Hadoop $SPARK_HADOOP_VERSION" >> "$DISTDIR/RELEASE"
 echo "Kyuubi Hadoop $HADOOP_VERSION" >> "$DISTDIR/RELEASE"
 echo "Hive $HIVE_VERSION" >> "$DISTDIR/RELEASE"
 echo "Build flags: $@" >> "$DISTDIR/RELEASE"
@@ -281,8 +275,8 @@ fi
 
 if [[ "$SPARK_PROVIDED" != "true" ]]; then
   # Copy spark binary dist
-  cp -r "$KYUUBI_HOME/externals/kyuubi-download/target/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/" \
-        "$DISTDIR/externals/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/"
+  SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'spark-*' -type d)"
+  cp -r "$SPARK_BUILTIN" "$DISTDIR/externals/"
 fi
 
 # Copy license files

diff --git a/pom.xml b/pom.xml
@@ -110,7 +110,6 @@
         <guava.version>30.1-jre</guava.version>
         <guava.failureaccess.version>1.0.1</guava.failureaccess.version>
         <hadoop.version>3.3.1</hadoop.version>
-        <hadoop.binary.version>3.2</hadoop.binary.version>
         <hive.version>2.3.9</hive.version>
         <hudi.version>0.10.0</hudi.version>
         <iceberg.name>iceberg-spark-runtime-3.2_${scala.binary.version}</iceberg.name>
@@ -138,7 +137,7 @@
           `delta.version`, `iceberg.name`
           -->
         <spark.version>3.2.1</spark.version>
-        <spark.archive.name>spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz</spark.archive.name>
+        <spark.archive.name>spark-${spark.version}-bin-hadoop3.2.tgz</spark.archive.name>
         <spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}</spark.archive.mirror>
         <spark.archive.download.skip>false</spark.archive.download.skip>
         <swagger.version>2.1.11</swagger.version>
@@ -1901,20 +1900,6 @@
             </properties>
         </profile>
 
-        <profile>
-            <id>spark-hadoop-2.7</id>
-            <properties>
-                <hadoop.binary.version>2.7</hadoop.binary.version>
-            </properties>
-        </profile>
-
-        <profile>
-            <id>spark-hadoop-3.2</id>
-            <properties>
-                <hadoop.binary.version>3.2</hadoop.binary.version>
-            </properties>
-        </profile>
-
         <profile>
             <id>spark-provided</id>
             <properties>