diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index d939e9122cb..f4ce59d8248 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -161,6 +161,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.engine.jdbc.connection.provider | <undefined> | A JDBC connection provider plugin for the Kyuubi Server to establish a connection to the JDBC URL. The configuration value should be a subclass of `org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider`. Kyuubi provides the following built-in implementations:
  • doris: For establishing Doris connections.
  • mysql: For establishing MySQL connections.
  • phoenix: For establishing Phoenix connections.
  • postgresql: For establishing PostgreSQL connections.
  • starrocks: For establishing StarRocks connections.
  • impala: For establishing Impala connections.
  • clickhouse: For establishing clickhouse connections.
    • | string | 1.6.0 | | kyuubi.engine.jdbc.connection.url | <undefined> | The server url that engine will connect to | string | 1.6.0 | | kyuubi.engine.jdbc.connection.user | <undefined> | The user is used for connecting to server | string | 1.6.0 | +| kyuubi.engine.jdbc.deploy.mode | LOCAL | Configures the jdbc engine deploy mode, The value can be 'local', 'yarn'. In local mode, the engine operates on the same node as the KyuubiServer. In YARN mode, the engine runs within the Application Master (AM) container of YARN. | string | 1.10.0 | | kyuubi.engine.jdbc.driver.class | <undefined> | The driver class for JDBC engine connection | string | 1.6.0 | | kyuubi.engine.jdbc.extra.classpath | <undefined> | The extra classpath for the JDBC query engine, for configuring the location of the JDBC driver and etc. | string | 1.6.0 | | kyuubi.engine.jdbc.fetch.size | 1000 | The fetch size of JDBC engine | int | 1.9.0 | diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala index a717f9dfd01..793fcdc7a4a 100644 --- a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala @@ -26,6 +26,7 @@ import org.apache.kyuubi.Utils import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.ENGINE_HIVE_EXTRA_CLASSPATH import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter +import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter.KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY import org.apache.kyuubi.engine.hive.HiveSQLEngine object HiveYarnModeSubmitter extends EngineYarnModeSubmitter { @@ -60,18 +61,10 @@ object HiveYarnModeSubmitter extends EngineYarnModeSubmitter { jars.toSeq } - private[hive] def parseClasspath(classpath: String, jars: ListBuffer[File]): Unit = { - classpath.split(":").filter(_.nonEmpty).foreach { cp => - if (cp.endsWith("/*")) { - val dir = cp.substring(0, cp.length - 2) - new File(dir) match { - case f if f.isDirectory => - f.listFiles().filter(_.getName.endsWith(".jar")).foreach(jars += _) - case _ => - } - } else { - jars += new File(cp) - } - } + override def listConfFiles(): Seq[File] = { + // respect the following priority loading configuration, and distinct files + // hive configuration -> hadoop configuration -> yarn configuration + val hiveConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY) + listDistinctFiles(hiveConf.get) ++ super.listConfFiles() } } diff --git a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitterSuite.scala b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitterSuite.scala deleted file mode 100644 index b04fefa0afd..00000000000 --- a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitterSuite.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kyuubi.engine.hive.deploy - -import java.io.File - -import scala.collection.mutable.ListBuffer - -import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION} -import org.apache.kyuubi.util.JavaUtils - -class HiveYarnModeSubmitterSuite extends KyuubiFunSuite { - val hiveEngineHome: String = JavaUtils.getCodeSourceLocation(getClass).split("/target")(0) - - test("hadoop class path") { - val jars = new ListBuffer[File] - val classpath = - s"$hiveEngineHome/target/scala-$SCALA_COMPILE_VERSION/jars/*:" + - s"$hiveEngineHome/target/kyuubi-hive-sql-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" - HiveYarnModeSubmitter.parseClasspath(classpath, jars) - assert(jars.nonEmpty) - assert(jars.exists( - _.getName == s"kyuubi-hive-sql-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar")) - } - -} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala index 6e0647f6c7a..ca620207c11 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala @@ -16,10 +16,15 @@ */ package org.apache.kyuubi.engine.jdbc +import java.security.PrivilegedExceptionAction + +import org.apache.hadoop.security.UserGroupInformation + import org.apache.kyuubi.{Logging, Utils} import org.apache.kyuubi.Utils.{addShutdownHook, JDBC_ENGINE_SHUTDOWN_PRIORITY} -import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.ENGINE_JDBC_INITIALIZE_SQL +import org.apache.kyuubi.config.{KyuubiConf, KyuubiReservedKeys} +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_DEPLOY_MODE, ENGINE_JDBC_INITIALIZE_SQL, ENGINE_KEYTAB, ENGINE_PRINCIPAL} +import org.apache.kyuubi.engine.deploy.DeployMode import org.apache.kyuubi.engine.jdbc.JdbcSQLEngine.currentEngine import org.apache.kyuubi.engine.jdbc.util.KyuubiJdbcUtils import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ZK_CONN_RETRY_POLICY @@ -38,6 +43,7 @@ class JdbcSQLEngine extends Serverable("JdbcSQLEngine") { // Start engine self-terminating checker after all services are ready and it can be reached by // all servers in engine spaces. backendService.sessionManager.startTerminatingChecker(() => { + selfExited = true currentEngine.foreach(_.stop()) }) } @@ -71,10 +77,32 @@ object JdbcSQLEngine extends Logging { Utils.fromCommandLineArgs(args, kyuubiConf) kyuubiConf.setIfMissing(KyuubiConf.FRONTEND_THRIFT_BINARY_BIND_PORT, 0) kyuubiConf.setIfMissing(HA_ZK_CONN_RETRY_POLICY, RetryPolicies.N_TIME.toString) + val proxyUser = kyuubiConf.getOption(KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY) + require(proxyUser.isDefined, s"${KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY} is not set") + val realUser = UserGroupInformation.getLoginUser + val principal = kyuubiConf.get(ENGINE_PRINCIPAL) + val keytab = kyuubiConf.get(ENGINE_KEYTAB) - startEngine() + val ugi = DeployMode.withName(kyuubiConf.get(ENGINE_JDBC_DEPLOY_MODE)) match { + case DeployMode.LOCAL + if UserGroupInformation.isSecurityEnabled && principal.isDefined && keytab.isDefined => + UserGroupInformation.loginUserFromKeytab(principal.get, keytab.get) + UserGroupInformation.getCurrentUser + case DeployMode.LOCAL if proxyUser.get != realUser.getShortUserName => + kyuubiConf.unset(KyuubiReservedKeys.KYUUBI_ENGINE_CREDENTIALS_KEY) + UserGroupInformation.createProxyUser(proxyUser.get, realUser) + case _ => + UserGroupInformation.getCurrentUser + } - KyuubiJdbcUtils.initializeJdbcSession(kyuubiConf, kyuubiConf.get(ENGINE_JDBC_INITIALIZE_SQL)) + ugi.doAs(new PrivilegedExceptionAction[Unit] { + override def run(): Unit = { + startEngine() + KyuubiJdbcUtils.initializeJdbcSession( + kyuubiConf, + kyuubiConf.get(ENGINE_JDBC_INITIALIZE_SQL)) + } + }) } catch { case t: Throwable if currentEngine.isDefined => currentEngine.foreach { engine => diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala new file mode 100644 index 00000000000..a585a9a6f46 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.deploy + +import java.io.File + +import scala.collection.mutable.ListBuffer + +import org.apache.hadoop.security.UserGroupInformation + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.ENGINE_JDBC_EXTRA_CLASSPATH +import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter +import org.apache.kyuubi.engine.jdbc.JdbcSQLEngine + +object JdbcYarnModeSubmitter extends EngineYarnModeSubmitter { + + def main(args: Array[String]): Unit = { + Utils.fromCommandLineArgs(args, kyuubiConf) + + if (UserGroupInformation.isSecurityEnabled) { + require( + kyuubiConf.get(KyuubiConf.ENGINE_PRINCIPAL).isDefined + && kyuubiConf.get(KyuubiConf.ENGINE_KEYTAB).isDefined, + s"${KyuubiConf.ENGINE_PRINCIPAL.key} and " + + s"${KyuubiConf.ENGINE_KEYTAB.key} must be set when submit " + + s"${JdbcSQLEngine.getClass.getSimpleName.stripSuffix("$")} to YARN") + } + run() + } + + override var engineType: String = "jdbc" + + override def engineMainClass(): String = JdbcSQLEngine.getClass.getName + + /** + * Jar list for the JDBC engine. + */ + override def engineExtraJars(): Seq[File] = { + val extraCp = kyuubiConf.get(ENGINE_JDBC_EXTRA_CLASSPATH) + val jars = new ListBuffer[File] + extraCp.foreach(cp => parseClasspath(cp, jars)) + jars.toSeq + } +} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala index 513e61303fd..101a06e7183 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala @@ -61,6 +61,9 @@ class JdbcSessionManager(name: String) } private def stopSession(): Unit = { - JdbcSQLEngine.currentEngine.foreach(_.stop()) + JdbcSQLEngine.currentEngine.foreach { engine => + engine.selfExited = true + engine.stop() + } } } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/clickhouse/WithClickHouseEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/clickhouse/WithClickHouseEngine.scala index 2cf695d28d3..bb834cd70af 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/clickhouse/WithClickHouseEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/clickhouse/WithClickHouseEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.clickhouse import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithClickHouseEngine extends WithJdbcEngine with WithClickHouseContainer { @@ -29,6 +30,7 @@ trait WithClickHouseEngine extends WithJdbcEngine with WithClickHouseContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> container.password, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "clickhouse", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> container.driverClassName) } } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala index 692f37b9515..47e5a4ab96e 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.doris import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithDorisEngine extends WithJdbcEngine with WithDorisContainer { @@ -28,5 +29,6 @@ trait WithDorisEngine extends WithJdbcEngine with WithDorisContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> "", ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "doris", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> "com.mysql.cj.jdbc.Driver") } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala index 9b31dd245bb..7dc12d56ba0 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.impala import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithImpalaEngine extends WithJdbcEngine with WithImpalaContainer { @@ -26,5 +27,6 @@ trait WithImpalaEngine extends WithJdbcEngine with WithImpalaContainer { ENGINE_JDBC_CONNECTION_URL.key -> hiveServerJdbcUrl, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "impala", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> ImpalaConnectionProvider.driverClass) } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/mysql/WithMySQLEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/mysql/WithMySQLEngine.scala index 39d2e0a59ec..09c9066a630 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/mysql/WithMySQLEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/mysql/WithMySQLEngine.scala @@ -21,6 +21,7 @@ import com.dimafeng.testcontainers.scalatest.TestContainerForAll import org.testcontainers.utility.DockerImageName import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithMySQLEngine extends WithJdbcEngine with TestContainerForAll { @@ -40,6 +41,7 @@ trait WithMySQLEngine extends WithJdbcEngine with TestContainerForAll { ENGINE_JDBC_CONNECTION_PASSWORD.key -> mysqlContainer.password, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "mysql", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> "com.mysql.cj.jdbc.Driver") } } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/phoenix/WithPhoenixEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/phoenix/WithPhoenixEngine.scala index adf328d7b0c..e1584341577 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/phoenix/WithPhoenixEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/phoenix/WithPhoenixEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.phoenix import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithPhoenixEngine extends WithJdbcEngine with WithPhoenixContainer { @@ -32,6 +33,7 @@ trait WithPhoenixEngine extends WithJdbcEngine with WithPhoenixContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> "", ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "phoenix", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> "org.apache.phoenix.queryserver.client.Driver") private def getConnectString: String = s"$jdbcUrlPrefix=http://$queryServerUrl;$serialization" diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLEngine.scala index 6d453934e6f..74c42f188f9 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.postgresql import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithPostgreSQLEngine extends WithJdbcEngine with WithPostgreSQLContainer { @@ -29,6 +30,7 @@ trait WithPostgreSQLEngine extends WithJdbcEngine with WithPostgreSQLContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> container.password, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "postgresql", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> container.driverClassName) } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/starrocks/WithStarRocksEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/starrocks/WithStarRocksEngine.scala index 6423186c050..3787bc2f176 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/starrocks/WithStarRocksEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/starrocks/WithStarRocksEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.starrocks import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine import org.apache.kyuubi.engine.jdbc.mysql.MySQL8ConnectionProvider @@ -32,5 +33,6 @@ trait WithStarRocksEngine extends WithJdbcEngine with WithStarRocksContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> password, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "starrocks", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> MySQL8ConnectionProvider.driverClass) } diff --git a/integration-tests/kyuubi-jdbc-it/pom.xml b/integration-tests/kyuubi-jdbc-it/pom.xml index f62d2d9af39..1279f3e8dd3 100644 --- a/integration-tests/kyuubi-jdbc-it/pom.xml +++ b/integration-tests/kyuubi-jdbc-it/pom.xml @@ -109,6 +109,37 @@ http test + + + + org.apache.hadoop + hadoop-client-minicluster + test + + + + org.bouncycastle + bcprov-jdk18on + test + + + + org.bouncycastle + bcpkix-jdk18on + test + + + + jakarta.activation + jakarta.activation-api + test + + + + jakarta.xml.bind + jakarta.xml.bind-api + test + diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/OperationWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..5561297ba68 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/OperationWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.mysql + +import org.apache.kyuubi.engine.jdbc.mysql.MySQLOperationSuite + +class OperationWithServerYarnModeSuite extends MySQLOperationSuite + with WithKyuubiServerAndMySQLContainer { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala new file mode 100644 index 00000000000..0567136e13f --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.it.jdbc.mysql + +import java.nio.file.{Files, Path, Paths} + +import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} +import org.apache.kyuubi.engine.deploy.DeployMode +import org.apache.kyuubi.engine.jdbc.mysql.MySQLOperationSuite + +class WithKyuubiServerAndMySQLContainerYarnMode extends MySQLOperationSuite + with WithKyuubiServerAndHadoopMiniCluster { + + private val mysqlJdbcConnectorPath: String = { + val keyword = "mysql-connector" + + val jarsDir = Paths.get(kyuubiHome) + .resolve("integration-tests") + .resolve("kyuubi-jdbc-it") + .resolve("target") + + Files.list(jarsDir) + .filter { p: Path => p.getFileName.toString contains keyword } + .findFirst + .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } + .toAbsolutePath + .toString + } + + override protected val conf: KyuubiConf = { + KyuubiConf() + .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) + .set(ENGINE_JDBC_EXTRA_CLASSPATH, mysqlJdbcConnectorPath) + .set(ENGINE_TYPE, "JDBC") + .set(KyuubiConf.ENGINE_JDBC_DEPLOY_MODE, DeployMode.YARN.toString) + .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) + } + + override def beforeAll(): Unit = { + val configs = withKyuubiConf + configs.foreach(config => conf.set(config._1, config._2)) + super.beforeAll() + conf + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) + } + + override protected def jdbcUrl: String = getJdbcUrl +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 170e6950c23..a0dc77f7fcf 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3131,6 +3131,19 @@ object KyuubiConf { .version("1.10.0") .fallbackConf(OPERATION_INCREMENTAL_COLLECT) + val ENGINE_JDBC_DEPLOY_MODE: ConfigEntry[String] = + buildConf("kyuubi.engine.jdbc.deploy.mode") + .doc("Configures the jdbc engine deploy mode, The value can be 'local', 'yarn'. " + + "In local mode, the engine operates on the same node as the KyuubiServer. " + + "In YARN mode, the engine runs within the Application Master (AM) container of YARN. ") + .version("1.10.0") + .stringConf + .transformToUpperCase + .checkValue( + mode => Set("LOCAL", "YARN").contains(mode), + "Invalid value for 'kyuubi.engine.jdbc.deploy.mode'. Valid values are 'local', 'yarn'.") + .createWithDefault(DeployMode.LOCAL.toString) + val ENGINE_OPERATION_CONVERT_CATALOG_DATABASE_ENABLED: ConfigEntry[Boolean] = buildConf("kyuubi.engine.operation.convert.catalog.database.enabled") .doc("When set to true, The engine converts the JDBC methods of set/get Catalog " + diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala index e2913539b46..e0fbe166748 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala @@ -313,15 +313,7 @@ abstract class EngineYarnModeSubmitter extends Logging { env) } } - // respect the following priority loading configuration, and distinct files - // hive configuration -> hadoop configuration -> yarn configuration - val hiveConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY) - listDistinctFiles(hiveConf.get).foreach(putEntry) - val hadoopConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY) - listDistinctFiles(hadoopConf.get).foreach(putEntry) - val yarnConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY) - listDistinctFiles(yarnConf.get).foreach(putEntry) - + listConfFiles().foreach(putEntry) val properties = confToProperties(kyuubiConf) amKeytabFileName.foreach(kt => properties.put(ENGINE_KEYTAB.key, kt)) writePropertiesToArchive(properties, KYUUBI_CONF_FILE, confStream) @@ -352,6 +344,14 @@ abstract class EngineYarnModeSubmitter extends Logging { }.toSeq } + def listConfFiles(): Seq[File] = { + // respect the following priority loading configuration, and distinct files + // hadoop configuration -> yarn configuration + val hadoopConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY) + val yarnConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY) + listDistinctFiles(hadoopConf.get) ++ listDistinctFiles(yarnConf.get) + } + private def distribute( srcPath: Path, resType: LocalResourceType, @@ -486,6 +486,21 @@ abstract class EngineYarnModeSubmitter extends Logging { writer.flush() out.closeEntry() } + + def parseClasspath(classpath: String, jars: ListBuffer[File]): Unit = { + classpath.split(":").filter(_.nonEmpty).foreach { cp => + if (cp.endsWith("/*")) { + val dir = cp.substring(0, cp.length - 2) + new File(dir) match { + case f if f.isDirectory => + f.listFiles().filter(_.getName.endsWith(".jar")).foreach(jars += _) + case _ => + } + } else { + jars += new File(cp) + } + } + } } object EngineYarnModeSubmitter { diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitterSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitterSuite.scala index f12d3f6fd7f..b9f419ea796 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitterSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitterSuite.scala @@ -18,12 +18,14 @@ package org.apache.kyuubi.engine.deploy.yarn import java.io.File +import scala.collection.mutable.ListBuffer + import org.apache.hadoop.fs.Path import org.apache.hadoop.yarn.api.ApplicationConstants.Environment import org.scalatest.matchers.must.Matchers import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper -import org.apache.kyuubi.{KyuubiFunSuite, Utils} +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION, Utils} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter.KYUUBI_ENGINE_DEPLOY_YARN_MODE_JARS_KEY import org.apache.kyuubi.util.JavaUtils @@ -66,6 +68,17 @@ class EngineYarnModeSubmitterSuite extends KyuubiFunSuite with Matchers { assert(targetFiles.length == files.length) } + test("hadoop class path") { + val jars = new ListBuffer[File] + val classpath = + s"$kyuubiHome/target/scala-$SCALA_COMPILE_VERSION/jars/*:" + + s"$kyuubiHome/target/kyuubi-common-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" + MockEngineYarnModeSubmitter.parseClasspath(classpath, jars) + assert(jars.nonEmpty) + assert(jars.exists( + _.getName == s"kyuubi-common-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar")) + } + } object MockEngineYarnModeSubmitter extends EngineYarnModeSubmitter { diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala index 0bb13b049f2..bb7f7ecbcf4 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala @@ -208,7 +208,14 @@ private[kyuubi] class EngineRef( extraEngineLog, defaultEngineName) case JDBC => - new JdbcProcessBuilder(appUser, doAsEnabled, conf, engineRefId, extraEngineLog) + conf.setIfMissing(JdbcProcessBuilder.JDBC_ENGINE_NAME, defaultEngineName) + JdbcProcessBuilder( + appUser, + doAsEnabled, + conf, + engineRefId, + extraEngineLog, + defaultEngineName) case CHAT => new ChatProcessBuilder(appUser, doAsEnabled, conf, engineRefId, extraEngineLog) } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala index 1afdcc3cf7c..0e914987c36 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala @@ -188,9 +188,11 @@ object KyuubiApplicationManager { case ("FLINK", Some("YARN")) => // running flink on other platforms is not yet supported setupFlinkYarnTag(applicationTag, conf) - // other engine types are running locally yet case ("HIVE", Some("YARN")) => setupEngineYarnModeTag(applicationTag, conf) + case ("JDBC", Some("YARN")) => + setupEngineYarnModeTag(applicationTag, conf) + // other engine types are running locally yet case _ => } } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala index de93fd44b55..e31376f365a 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala @@ -23,12 +23,15 @@ import java.nio.file.Paths import scala.collection.mutable import com.google.common.annotations.VisibleForTesting +import org.apache.hadoop.security.UserGroupInformation -import org.apache.kyuubi.{Logging, SCALA_COMPILE_VERSION, Utils} +import org.apache.kyuubi.{KyuubiException, Logging, SCALA_COMPILE_VERSION, Utils} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_JDBC_JAVA_OPTIONS, ENGINE_JDBC_MEMORY} +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_DEPLOY_YARN_MODE_APP_NAME, ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_DEPLOY_MODE, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_JDBC_JAVA_OPTIONS, ENGINE_JDBC_MEMORY, ENGINE_KEYTAB, ENGINE_PRINCIPAL} import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.ProcBuilder +import org.apache.kyuubi.engine.deploy.DeployMode +import org.apache.kyuubi.engine.deploy.DeployMode.{LOCAL, YARN} import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.util.command.CommandLineUtils._ @@ -113,3 +116,55 @@ class JdbcProcessBuilder( } } } + +object JdbcProcessBuilder extends Logging { + + final val JDBC_ENGINE_NAME = "jdbc.engine.name" + + def apply( + proxyUser: String, + doAsEnabled: Boolean, + conf: KyuubiConf, + engineRefId: String, + extraEngineLog: Option[OperationLog], + defaultEngineName: String): JdbcProcessBuilder = { + checkKeytab(proxyUser, conf) + DeployMode.withName(conf.get(ENGINE_JDBC_DEPLOY_MODE)) match { + case LOCAL => + new JdbcProcessBuilder(proxyUser, doAsEnabled, conf, engineRefId, extraEngineLog) + case YARN => + warn(s"JDBC on YARN model is experimental.") + conf.setIfMissing(ENGINE_DEPLOY_YARN_MODE_APP_NAME, Some(defaultEngineName)) + new JdbcYarnModeProcessBuilder(proxyUser, doAsEnabled, conf, engineRefId, extraEngineLog) + case other => throw new KyuubiException(s"Unsupported deploy mode: $other") + } + } + + private def checkKeytab(proxyUser: String, conf: KyuubiConf): Unit = { + val principal = conf.get(ENGINE_PRINCIPAL) + val keytab = conf.get(ENGINE_KEYTAB) + if (!UserGroupInformation.isSecurityEnabled) { + if (principal.isDefined || keytab.isDefined) { + warn("Principal and keytab takes no effect when hadoop security is not enabled.") + } + return + } + + require( + principal.isDefined == keytab.isDefined, + s"Both principal and keytab must be defined, or neither.") + if (principal.isDefined && keytab.isDefined) { + val ugi = UserGroupInformation + .loginUserFromKeytabAndReturnUGI(principal.get, keytab.get) + require( + ugi.getShortUserName == proxyUser, + s"Proxy user: $proxyUser is not same with " + + s"engine principal: ${ugi.getShortUserName}.") + } + + val deployMode = DeployMode.withName(conf.get(ENGINE_JDBC_DEPLOY_MODE)) + if (principal.isEmpty && keytab.isEmpty && deployMode == YARN) { + warn("JDBC on YARN can not work properly without principal and keytab.") + } + } +} diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala new file mode 100644 index 00000000000..fbc4416d73f --- /dev/null +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc + +import java.io.File +import java.nio.file.Paths +import java.util + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +import org.apache.kyuubi.{Logging, SCALA_COMPILE_VERSION, Utils} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_JDBC_MEMORY} +import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_ID, KYUUBI_SESSION_USER_KEY} +import org.apache.kyuubi.engine.{ApplicationManagerInfo, KyuubiApplicationManager} +import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter._ +import org.apache.kyuubi.operation.log.OperationLog +import org.apache.kyuubi.util.command.CommandLineUtils.{confKeyValue, confKeyValues} + +/** + * A process builder for JDBC on Yarn. + * + * It will new a process on kyuubi server side to submit jdbc engine to yarn. + */ +class JdbcYarnModeProcessBuilder( + override val proxyUser: String, + override val doAsEnabled: Boolean, + override val conf: KyuubiConf, + override val engineRefId: String, + override val extraEngineLog: Option[OperationLog] = None) + extends JdbcProcessBuilder(proxyUser, doAsEnabled, conf, engineRefId, extraEngineLog) + with Logging { + + override protected def mainClass: String = + "org.apache.kyuubi.engine.jdbc.deploy.JdbcYarnModeSubmitter" + + override def isClusterMode(): Boolean = true + + override def clusterManager(): Option[String] = Some("yarn") + + override def appMgrInfo(): ApplicationManagerInfo = ApplicationManagerInfo(clusterManager()) + + override protected val commands: Iterable[String] = { + KyuubiApplicationManager.tagApplication(engineRefId, shortName, clusterManager(), conf) + val buffer = new ArrayBuffer[String]() + buffer += executable + + val memory = conf.get(ENGINE_JDBC_MEMORY) + buffer += s"-Xmx$memory" + buffer += "-cp" + + val classpathEntries = new util.LinkedHashSet[String] + classpathEntries.addAll(hadoopConfFiles()) + classpathEntries.addAll(yarnConfFiles()) + classpathEntries.addAll(jarFiles(true)) + + buffer += classpathEntries.asScala.mkString(File.pathSeparator) + buffer += mainClass + + buffer ++= confKeyValue(KYUUBI_SESSION_USER_KEY, proxyUser) + buffer ++= confKeyValue(KYUUBI_ENGINE_ID, engineRefId) + + buffer ++= confKeyValue( + KYUUBI_ENGINE_DEPLOY_YARN_MODE_JARS_KEY, + jarFiles(false).asScala.mkString(KYUUBI_ENGINE_DEPLOY_YARN_MODE_ARCHIVE_SEPARATOR)) + buffer ++= confKeyValue( + KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY, + hadoopConfFiles().asScala.mkString(KYUUBI_ENGINE_DEPLOY_YARN_MODE_ARCHIVE_SEPARATOR)) + buffer ++= confKeyValue( + KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY, + yarnConfFiles().asScala.mkString(KYUUBI_ENGINE_DEPLOY_YARN_MODE_ARCHIVE_SEPARATOR)) + + buffer ++= confKeyValues(conf.getAll) + + buffer + } + + private def jarFiles(isClasspath: Boolean): util.LinkedHashSet[String] = { + val jarEntries = new util.LinkedHashSet[String] + mainResource.foreach(jarEntries.add) + val javaOptions = conf.get(ENGINE_JDBC_EXTRA_CLASSPATH) + if (isClasspath) { + javaOptions.foreach(jarEntries.add) + } + mainResource.foreach { path => + val parent = Paths.get(path).getParent + if (Utils.isTesting) { + // add dev classpath + val jdbcDeps = parent + .resolve(s"scala-$SCALA_COMPILE_VERSION") + .resolve("jars") + jarEntries.add(s"$jdbcDeps${appendClasspathSuffix(isClasspath)}") + } else { + // add prod classpath + jarEntries.add(s"$parent${appendClasspathSuffix(isClasspath)}") + } + } + jarEntries + } + + private def hadoopConfFiles(): util.LinkedHashSet[String] = { + val confEntries = new util.LinkedHashSet[String] + env.get("HADOOP_CONF_DIR").foreach(confEntries.add) + confEntries + } + + private def yarnConfFiles(): util.LinkedHashSet[String] = { + val confEntries = new util.LinkedHashSet[String] + env.get("YARN_CONF_DIR").foreach(confEntries.add) + confEntries + } + + private def appendClasspathSuffix(isClasspath: Boolean): String = { + if (isClasspath) { + s"${File.separator}*" + } else { + "" + } + } +} diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilderSuite.scala new file mode 100644 index 00000000000..6c415e88e38 --- /dev/null +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilderSuite.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL} +import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter.{KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY, KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY} +import org.apache.kyuubi.engine.hive.HiveProcessBuilder.HIVE_HADOOP_CLASSPATH_KEY + +class JdbcYarnModeProcessBuilderSuite extends KyuubiFunSuite { + + test("jdbc yarn mode process builder") { + val conf = KyuubiConf().set("kyuubi.on", "off") + .set(ENGINE_JDBC_CONNECTION_URL.key, "") + .set(ENGINE_JDBC_CONNECTION_PASSWORD.key, "123456") + val builder = new JdbcYarnModeProcessBuilder("kyuubi", true, conf, "") + val commands = builder.toString.split('\n') + assert(commands.head.contains("bin/java"), "wrong exec") + assert(builder.toString.contains("--conf kyuubi.session.user=kyuubi")) + assert(commands.exists(ss => ss.contains("kyuubi-jdbc-engine")), "wrong classpath") + assert(builder.toString.contains("--conf kyuubi.on=off")) + } + + test("hadoop conf dir") { + val conf = KyuubiConf().set("kyuubi.on", "off") + .set(ENGINE_JDBC_CONNECTION_URL.key, "") + .set(ENGINE_JDBC_CONNECTION_PASSWORD.key, "123456") + val builder = new JdbcYarnModeProcessBuilder("kyuubi", true, conf, "") { + override def env: Map[String, String] = + super.env + ("HADOOP_CONF_DIR" -> "/etc/hadoop/conf") + + (HIVE_HADOOP_CLASSPATH_KEY -> "/hadoop") + } + assert(builder.toString.contains( + s"--conf $KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY=/etc/hadoop/conf")) + } + + test("yarn conf dir") { + val conf = KyuubiConf().set("kyuubi.on", "off") + .set(ENGINE_JDBC_CONNECTION_URL.key, "") + .set(ENGINE_JDBC_CONNECTION_PASSWORD.key, "123456") + val builder = new JdbcYarnModeProcessBuilder("kyuubi", true, conf, "") { + override def env: Map[String, String] = + super.env + ("YARN_CONF_DIR" -> "/etc/hadoop/yarn/conf") + + (HIVE_HADOOP_CLASSPATH_KEY -> "/hadoop") + } + assert(builder.toString.contains( + s"--conf $KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY=/etc/hadoop/yarn/conf")) + } +}