From c384049184219cd700764a4db5f444390986289d Mon Sep 17 00:00:00 2001 From: senmiaoliu Date: Sun, 7 Apr 2024 15:20:42 +0800 Subject: [PATCH 1/5] jdbc engine on yarn --- docs/configuration/settings.md | 1 + .../kyuubi/engine/jdbc/JdbcSQLEngine.scala | 46 +++++- .../jdbc/deploy/JdbcYarnModeSubmitter.scala | 77 ++++++++++ .../jdbc/session/JdbcSessionManager.scala | 5 +- .../clickhouse/WithClickHouseEngine.scala | 2 + .../deploy/JdbcYarnModeSubmitterSuite.scala | 40 ++++++ .../engine/jdbc/doris/WithDorisEngine.scala | 2 + .../engine/jdbc/impala/WithImpalaEngine.scala | 2 + .../engine/jdbc/mysql/WithMySQLEngine.scala | 2 + .../jdbc/phoenix/WithPhoenixEngine.scala | 2 + .../postgresql/WithPostgreSQLEngine.scala | 2 + .../jdbc/starrocks/WithStarRocksEngine.scala | 2 + integration-tests/kyuubi-jdbc-it/pom.xml | 31 ++++ .../OperationWithServerYarnModeSuite.scala | 27 ++++ .../SessionWithServerYarnModeSuite.scala | 27 ++++ .../StatementWithServerYarnModeSuite.scala | 27 ++++ ...ServerAndClickHouseContainerYarnMode.scala | 63 ++++++++ .../OperationWithServerYarnModeSuite.scala | 27 ++++ ...yuubiServerAndDorisContainerYarnMode.scala | 64 +++++++++ .../OperationWithServerYarnModeSuite.scala | 27 ++++ .../SessionWithServerYarnModeSuite.scala | 27 ++++ .../StatementWithServerYarnModeSuite.scala | 27 ++++ ...uubiServerAndImpalaContainerYarnMode.scala | 64 +++++++++ .../OperationWithServerYarnModeSuite.scala | 27 ++++ ...yuubiServerAndMySQLContainerYarnMode.scala | 65 +++++++++ .../OperationWithServerYarnModeSuite.scala | 27 ++++ .../SessionWithServerYarnModeSuite.scala | 27 ++++ .../StatementWithServerYarnModeSuite.scala | 27 ++++ ...ubiServerAndPhoenixContainerYarnMode.scala | 63 ++++++++ .../OperationWithServerYarnModeSuite.scala | 27 ++++ .../SessionWithServerYarnModeSuite.scala | 27 ++++ .../StatementWithServerYarnModeSuite.scala | 27 ++++ ...ServerAndPostgreSQLContainerYarnMode.scala | 63 ++++++++ .../org/apache/kyuubi/config/KyuubiConf.scala | 13 ++ .../deploy/yarn/EngineYarnModeSubmitter.scala | 6 +- .../org/apache/kyuubi/engine/EngineRef.scala | 9 +- .../engine/KyuubiApplicationManager.scala | 4 +- .../engine/jdbc/JdbcProcessBuilder.scala | 59 +++++++- .../jdbc/JdbcYarnModeProcessBuilder.scala | 135 ++++++++++++++++++ .../JdbcYarnModeProcessBuilderSuite.scala | 64 +++++++++ 40 files changed, 1252 insertions(+), 12 deletions(-) create mode 100644 externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala create mode 100644 externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitterSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/OperationWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/SessionWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/StatementWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainerYarnMode.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainerYarnMode.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainerYarnMode.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/OperationWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/OperationWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/SessionWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/StatementWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainerYarnMode.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/OperationWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/SessionWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/StatementWithServerYarnModeSuite.scala create mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainerYarnMode.scala create mode 100644 kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala create mode 100644 kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilderSuite.scala diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index d939e9122cb..f4ce59d8248 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -161,6 +161,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.engine.jdbc.connection.provider | <undefined> | A JDBC connection provider plugin for the Kyuubi Server to establish a connection to the JDBC URL. The configuration value should be a subclass of `org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider`. Kyuubi provides the following built-in implementations:
  • doris: For establishing Doris connections.
  • mysql: For establishing MySQL connections.
  • phoenix: For establishing Phoenix connections.
  • postgresql: For establishing PostgreSQL connections.
  • starrocks: For establishing StarRocks connections.
  • impala: For establishing Impala connections.
  • clickhouse: For establishing clickhouse connections.
    • | string | 1.6.0 | | kyuubi.engine.jdbc.connection.url | <undefined> | The server url that engine will connect to | string | 1.6.0 | | kyuubi.engine.jdbc.connection.user | <undefined> | The user is used for connecting to server | string | 1.6.0 | +| kyuubi.engine.jdbc.deploy.mode | LOCAL | Configures the jdbc engine deploy mode, The value can be 'local', 'yarn'. In local mode, the engine operates on the same node as the KyuubiServer. In YARN mode, the engine runs within the Application Master (AM) container of YARN. | string | 1.10.0 | | kyuubi.engine.jdbc.driver.class | <undefined> | The driver class for JDBC engine connection | string | 1.6.0 | | kyuubi.engine.jdbc.extra.classpath | <undefined> | The extra classpath for the JDBC query engine, for configuring the location of the JDBC driver and etc. | string | 1.6.0 | | kyuubi.engine.jdbc.fetch.size | 1000 | The fetch size of JDBC engine | int | 1.9.0 | diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala index 6e0647f6c7a..a6291581c23 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala @@ -16,10 +16,15 @@ */ package org.apache.kyuubi.engine.jdbc +import java.security.PrivilegedExceptionAction + +import org.apache.hadoop.security.UserGroupInformation + import org.apache.kyuubi.{Logging, Utils} import org.apache.kyuubi.Utils.{addShutdownHook, JDBC_ENGINE_SHUTDOWN_PRIORITY} -import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.ENGINE_JDBC_INITIALIZE_SQL +import org.apache.kyuubi.config.{KyuubiConf, KyuubiReservedKeys} +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_DEPLOY_MODE, ENGINE_JDBC_INITIALIZE_SQL, ENGINE_KEYTAB, ENGINE_PRINCIPAL} +import org.apache.kyuubi.engine.deploy.DeployMode import org.apache.kyuubi.engine.jdbc.JdbcSQLEngine.currentEngine import org.apache.kyuubi.engine.jdbc.util.KyuubiJdbcUtils import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ZK_CONN_RETRY_POLICY @@ -38,6 +43,7 @@ class JdbcSQLEngine extends Serverable("JdbcSQLEngine") { // Start engine self-terminating checker after all services are ready and it can be reached by // all servers in engine spaces. backendService.sessionManager.startTerminatingChecker(() => { + selfExited = true currentEngine.foreach(_.stop()) }) } @@ -66,15 +72,45 @@ object JdbcSQLEngine extends Logging { def main(args: Array[String]): Unit = { SignalRegister.registerLogger(logger) - try { Utils.fromCommandLineArgs(args, kyuubiConf) kyuubiConf.setIfMissing(KyuubiConf.FRONTEND_THRIFT_BINARY_BIND_PORT, 0) kyuubiConf.setIfMissing(HA_ZK_CONN_RETRY_POLICY, RetryPolicies.N_TIME.toString) + val proxyUser = kyuubiConf.getOption(KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY) + require(proxyUser.isDefined, s"${KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY} is not set") + val realUser = UserGroupInformation.getLoginUser + val principal = kyuubiConf.get(ENGINE_PRINCIPAL) + val keytab = kyuubiConf.get(ENGINE_KEYTAB) - startEngine() + val ugi = DeployMode.withName(kyuubiConf.get(ENGINE_JDBC_DEPLOY_MODE)) match { + case DeployMode.LOCAL + if UserGroupInformation.isSecurityEnabled && principal.isDefined && keytab.isDefined => + UserGroupInformation.loginUserFromKeytab(principal.get, keytab.get) + UserGroupInformation.getCurrentUser + case DeployMode.LOCAL if proxyUser.get != realUser.getShortUserName => + val newUGI = UserGroupInformation.createProxyUser(proxyUser.get, realUser) + newUGI.doAs(new PrivilegedExceptionAction[Unit] { + override def run(): Unit = { + val engineCredentials = + kyuubiConf.getOption(KyuubiReservedKeys.KYUUBI_ENGINE_CREDENTIALS_KEY) + kyuubiConf.unset(KyuubiReservedKeys.KYUUBI_ENGINE_CREDENTIALS_KEY) + engineCredentials.filter(_.nonEmpty).foreach { credentials => + } + } + }) + newUGI + case _ => + UserGroupInformation.getCurrentUser + } - KyuubiJdbcUtils.initializeJdbcSession(kyuubiConf, kyuubiConf.get(ENGINE_JDBC_INITIALIZE_SQL)) + ugi.doAs(new PrivilegedExceptionAction[Unit] { + override def run(): Unit = { + startEngine() + KyuubiJdbcUtils.initializeJdbcSession( + kyuubiConf, + kyuubiConf.get(ENGINE_JDBC_INITIALIZE_SQL)) + } + }) } catch { case t: Throwable if currentEngine.isDefined => currentEngine.foreach { engine => diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala new file mode 100644 index 00000000000..fbb87587d63 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.deploy + +import java.io.File + +import scala.collection.mutable.ListBuffer + +import org.apache.hadoop.security.UserGroupInformation + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.ENGINE_JDBC_EXTRA_CLASSPATH +import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter +import org.apache.kyuubi.engine.jdbc.JdbcSQLEngine + +object JdbcYarnModeSubmitter extends EngineYarnModeSubmitter { + + def main(args: Array[String]): Unit = { + Utils.fromCommandLineArgs(args, kyuubiConf) + + if (UserGroupInformation.isSecurityEnabled) { + require( + kyuubiConf.get(KyuubiConf.ENGINE_PRINCIPAL).isDefined + && kyuubiConf.get(KyuubiConf.ENGINE_KEYTAB).isDefined, + s"${KyuubiConf.ENGINE_PRINCIPAL.key} and " + + s"${KyuubiConf.ENGINE_KEYTAB.key} must be set when submit " + + s"${JdbcSQLEngine.getClass.getSimpleName.stripSuffix("$")} to YARN") + } + run() + } + + override var engineType: String = "jdbc" + + override def engineMainClass(): String = JdbcSQLEngine.getClass.getName + + /** + * Jar list for the Hive engine. + */ + override def engineExtraJars(): Seq[File] = { + val hadoopCp = sys.env.get("HIVE_HADOOP_CLASSPATH") + val extraCp = kyuubiConf.get(ENGINE_JDBC_EXTRA_CLASSPATH) + val jars = new ListBuffer[File] + hadoopCp.foreach(cp => parseClasspath(cp, jars)) + extraCp.foreach(cp => parseClasspath(cp, jars)) + jars.toSeq + } + + private[jdbc] def parseClasspath(classpath: String, jars: ListBuffer[File]): Unit = { + classpath.split(":").filter(_.nonEmpty).foreach { cp => + if (cp.endsWith("/*")) { + val dir = cp.substring(0, cp.length - 2) + new File(dir) match { + case f if f.isDirectory => + f.listFiles().filter(_.getName.endsWith(".jar")).foreach(jars += _) + case _ => + } + } else { + jars += new File(cp) + } + } + } +} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala index 513e61303fd..101a06e7183 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala @@ -61,6 +61,9 @@ class JdbcSessionManager(name: String) } private def stopSession(): Unit = { - JdbcSQLEngine.currentEngine.foreach(_.stop()) + JdbcSQLEngine.currentEngine.foreach { engine => + engine.selfExited = true + engine.stop() + } } } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/clickhouse/WithClickHouseEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/clickhouse/WithClickHouseEngine.scala index 2cf695d28d3..bb834cd70af 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/clickhouse/WithClickHouseEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/clickhouse/WithClickHouseEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.clickhouse import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithClickHouseEngine extends WithJdbcEngine with WithClickHouseContainer { @@ -29,6 +30,7 @@ trait WithClickHouseEngine extends WithJdbcEngine with WithClickHouseContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> container.password, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "clickhouse", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> container.driverClassName) } } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitterSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitterSuite.scala new file mode 100644 index 00000000000..771de4d508c --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitterSuite.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.deploy + +import java.io.File + +import scala.collection.mutable.ListBuffer + +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION} +import org.apache.kyuubi.util.JavaUtils + +class JdbcYarnModeSubmitterSuite extends KyuubiFunSuite { + val jdbcEngineHome: String = JavaUtils.getCodeSourceLocation(getClass).split("/target")(0) + + test("hadoop class path") { + val jars = new ListBuffer[File] + val classpath = + s"$jdbcEngineHome/target/scala-$SCALA_COMPILE_VERSION/jars/*:" + + s"$jdbcEngineHome/target/kyuubi-jdbc-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" + JdbcYarnModeSubmitter.parseClasspath(classpath, jars) + assert(jars.nonEmpty) + assert(jars.exists( + _.getName == s"kyuubi-jdbc-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar")) + } + +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala index 692f37b9515..47e5a4ab96e 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.doris import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithDorisEngine extends WithJdbcEngine with WithDorisContainer { @@ -28,5 +29,6 @@ trait WithDorisEngine extends WithJdbcEngine with WithDorisContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> "", ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "doris", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> "com.mysql.cj.jdbc.Driver") } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala index 9b31dd245bb..7dc12d56ba0 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.impala import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithImpalaEngine extends WithJdbcEngine with WithImpalaContainer { @@ -26,5 +27,6 @@ trait WithImpalaEngine extends WithJdbcEngine with WithImpalaContainer { ENGINE_JDBC_CONNECTION_URL.key -> hiveServerJdbcUrl, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "impala", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> ImpalaConnectionProvider.driverClass) } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/mysql/WithMySQLEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/mysql/WithMySQLEngine.scala index 39d2e0a59ec..09c9066a630 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/mysql/WithMySQLEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/mysql/WithMySQLEngine.scala @@ -21,6 +21,7 @@ import com.dimafeng.testcontainers.scalatest.TestContainerForAll import org.testcontainers.utility.DockerImageName import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithMySQLEngine extends WithJdbcEngine with TestContainerForAll { @@ -40,6 +41,7 @@ trait WithMySQLEngine extends WithJdbcEngine with TestContainerForAll { ENGINE_JDBC_CONNECTION_PASSWORD.key -> mysqlContainer.password, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "mysql", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> "com.mysql.cj.jdbc.Driver") } } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/phoenix/WithPhoenixEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/phoenix/WithPhoenixEngine.scala index adf328d7b0c..e1584341577 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/phoenix/WithPhoenixEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/phoenix/WithPhoenixEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.phoenix import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithPhoenixEngine extends WithJdbcEngine with WithPhoenixContainer { @@ -32,6 +33,7 @@ trait WithPhoenixEngine extends WithJdbcEngine with WithPhoenixContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> "", ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "phoenix", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> "org.apache.phoenix.queryserver.client.Driver") private def getConnectString: String = s"$jdbcUrlPrefix=http://$queryServerUrl;$serialization" diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLEngine.scala index 6d453934e6f..74c42f188f9 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.postgresql import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine trait WithPostgreSQLEngine extends WithJdbcEngine with WithPostgreSQLContainer { @@ -29,6 +30,7 @@ trait WithPostgreSQLEngine extends WithJdbcEngine with WithPostgreSQLContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> container.password, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "postgresql", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> container.driverClassName) } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/starrocks/WithStarRocksEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/starrocks/WithStarRocksEngine.scala index 6423186c050..3787bc2f176 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/starrocks/WithStarRocksEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/starrocks/WithStarRocksEngine.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.jdbc.starrocks import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine import org.apache.kyuubi.engine.jdbc.mysql.MySQL8ConnectionProvider @@ -32,5 +33,6 @@ trait WithStarRocksEngine extends WithJdbcEngine with WithStarRocksContainer { ENGINE_JDBC_CONNECTION_PASSWORD.key -> password, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "starrocks", + KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> MySQL8ConnectionProvider.driverClass) } diff --git a/integration-tests/kyuubi-jdbc-it/pom.xml b/integration-tests/kyuubi-jdbc-it/pom.xml index f62d2d9af39..1279f3e8dd3 100644 --- a/integration-tests/kyuubi-jdbc-it/pom.xml +++ b/integration-tests/kyuubi-jdbc-it/pom.xml @@ -109,6 +109,37 @@ http test + + + + org.apache.hadoop + hadoop-client-minicluster + test + + + + org.bouncycastle + bcprov-jdk18on + test + + + + org.bouncycastle + bcpkix-jdk18on + test + + + + jakarta.activation + jakarta.activation-api + test + + + + jakarta.xml.bind + jakarta.xml.bind-api + test + diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/OperationWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..db8627d6646 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/OperationWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.clickhouse + +import org.apache.kyuubi.engine.jdbc.clickhouse.ClickHouseOperationSuite + +class OperationWithServerYarnModeSuite extends ClickHouseOperationSuite + with WithKyuubiServerAndClickHouseContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/SessionWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/SessionWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..3d6844ef78f --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/SessionWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.clickhouse + +import org.apache.kyuubi.engine.jdbc.clickhouse.ClickHouseSessionSuite + +class SessionWithServerYarnModeSuite extends ClickHouseSessionSuite + with WithKyuubiServerAndClickHouseContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/StatementWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/StatementWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..f1db4ab1a4e --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/StatementWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.clickhouse + +import org.apache.kyuubi.engine.jdbc.clickhouse.ClickHouseStatementSuite + +class StatementWithServerYarnModeSuite extends ClickHouseStatementSuite + with WithKyuubiServerAndClickHouseContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainerYarnMode.scala new file mode 100644 index 00000000000..1043fe5cd68 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainerYarnMode.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.it.jdbc.clickhouse + +import java.nio.file.{Files, Path, Paths} + +import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} +import org.apache.kyuubi.engine.deploy.DeployMode +import org.apache.kyuubi.engine.jdbc.clickhouse.WithClickHouseEngine + +trait WithKyuubiServerAndClickHouseContainerYarnMode + extends WithKyuubiServerAndHadoopMiniCluster with WithClickHouseEngine { + + private val clickHouseJdbcConnectorPath: String = { + val keyword = "clickhouse-jdbc" + + val jarsDir = Paths.get(kyuubiHome) + .resolve("integration-tests") + .resolve("kyuubi-jdbc-it") + .resolve("target") + + Files.list(jarsDir) + .filter { p: Path => p.getFileName.toString contains keyword } + .findFirst + .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } + .toAbsolutePath + .toString + } + + override protected val conf: KyuubiConf = { + KyuubiConf() + .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) + .set(ENGINE_JDBC_EXTRA_CLASSPATH, clickHouseJdbcConnectorPath) + .set(ENGINE_TYPE, "JDBC") + .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) + .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) + } + + override def beforeAll(): Unit = { + val configs = withKyuubiConf + configs.foreach(config => conf.set(config._1, config._2)) + super.beforeAll() + conf + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) + } +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..08737677491 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.doris + +import org.apache.kyuubi.engine.jdbc.doris.DorisOperationSuite + +class OperationWithServerYarnModeSuite extends DorisOperationSuite + with WithKyuubiServerAndDorisContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainerYarnMode.scala new file mode 100644 index 00000000000..56c5ca8e00f --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainerYarnMode.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.doris + +import java.nio.file.{Files, Path, Paths} + +import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} +import org.apache.kyuubi.engine.deploy.DeployMode +import org.apache.kyuubi.engine.jdbc.doris.WithDorisEngine + +trait WithKyuubiServerAndDorisContainerYarnMode extends WithKyuubiServerAndHadoopMiniCluster + with WithDorisEngine { + + private val mysqlJdbcConnectorPath: String = { + val keyword = "mysql-connector" + + val jarsDir = Paths.get(kyuubiHome) + .resolve("integration-tests") + .resolve("kyuubi-jdbc-it") + .resolve("target") + + Files.list(jarsDir) + .filter { p: Path => p.getFileName.toString contains keyword } + .findFirst + .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } + .toAbsolutePath + .toString + } + + override protected val conf: KyuubiConf = { + KyuubiConf() + .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) + .set(ENGINE_JDBC_EXTRA_CLASSPATH, mysqlJdbcConnectorPath) + .set(ENGINE_TYPE, "JDBC") + .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) + .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) + } + + override def beforeAll(): Unit = { + val configs = withKyuubiConf + configs.foreach(config => conf.set(config._1, config._2)) + super.beforeAll() + conf + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) + } +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..1c46c98df80 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.impala + +import org.apache.kyuubi.engine.jdbc.impala.ImpalaOperationSuite + +class OperationWithServerYarnModeSuite extends ImpalaOperationSuite + with WithKyuubiServerAndImpalaContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..c56df4753d9 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.impala + +import org.apache.kyuubi.engine.jdbc.impala.SessionSuite + +class SessionWithServerYarnModeSuite extends SessionSuite + with WithKyuubiServerAndImpalaContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..38f1eed6605 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.impala + +import org.apache.kyuubi.engine.jdbc.impala.StatementSuite + +class StatementWithServerYarnModeSuite extends StatementSuite + with WithKyuubiServerAndImpalaContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainerYarnMode.scala new file mode 100644 index 00000000000..6980aa8d521 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainerYarnMode.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.impala + +import java.nio.file.{Files, Path, Paths} + +import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} +import org.apache.kyuubi.engine.deploy.DeployMode +import org.apache.kyuubi.engine.jdbc.impala.WithImpalaEngine + +trait WithKyuubiServerAndImpalaContainerYarnMode extends WithKyuubiServerAndHadoopMiniCluster + with WithImpalaEngine { + + private val hiveJdbcConnectorPath: String = { + val keyword = "kyuubi-hive-jdbc-shaded" + + val jarsDir = Paths.get(kyuubiHome) + .resolve("integration-tests") + .resolve("kyuubi-jdbc-it") + .resolve("target") + + Files.list(jarsDir) + .filter { p: Path => p.getFileName.toString contains keyword } + .findFirst + .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } + .toAbsolutePath + .toString + } + + override protected val conf: KyuubiConf = { + KyuubiConf() + .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) + .set(ENGINE_JDBC_EXTRA_CLASSPATH, hiveJdbcConnectorPath) + .set(ENGINE_TYPE, "JDBC") + .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) + .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) + } + + override def beforeAll(): Unit = { + val configs = withKyuubiConf + configs.foreach(config => conf.set(config._1, config._2)) + super.beforeAll() + conf + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) + } +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/OperationWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..5561297ba68 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/OperationWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.mysql + +import org.apache.kyuubi.engine.jdbc.mysql.MySQLOperationSuite + +class OperationWithServerYarnModeSuite extends MySQLOperationSuite + with WithKyuubiServerAndMySQLContainer { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala new file mode 100644 index 00000000000..d016cba006b --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.it.jdbc.mysql + +import java.nio.file.{Files, Path, Paths} + +import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} +import org.apache.kyuubi.engine.deploy.DeployMode +import org.apache.kyuubi.engine.jdbc.mysql.MySQLOperationSuite + +class WithKyuubiServerAndMySQLContainerYarnMode extends MySQLOperationSuite + with WithKyuubiServerAndHadoopMiniCluster { + + private val mysqlJdbcConnectorPath: String = { + val keyword = "mysql-connector" + + val jarsDir = Paths.get(kyuubiHome) + .resolve("integration-tests") + .resolve("kyuubi-jdbc-it") + .resolve("target") + + Files.list(jarsDir) + .filter { p: Path => p.getFileName.toString contains keyword } + .findFirst + .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } + .toAbsolutePath + .toString + } + + override protected val conf: KyuubiConf = { + KyuubiConf() + .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) + .set(ENGINE_JDBC_EXTRA_CLASSPATH, mysqlJdbcConnectorPath) + .set(ENGINE_TYPE, "JDBC") + .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) + .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) + } + + override def beforeAll(): Unit = { + val configs = withKyuubiConf + configs.foreach(config => conf.set(config._1, config._2)) + super.beforeAll() + conf + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) + } + + override protected def jdbcUrl: String = getJdbcUrl +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/OperationWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..f55ddfa47c3 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/OperationWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.phoenix + +import org.apache.kyuubi.engine.jdbc.phoenix.PhoenixOperationSuite + +class OperationWithServerYarnModeSuite extends PhoenixOperationSuite + with WithKyuubiServerAndPhoenixContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/SessionWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/SessionWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..a614270e85f --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/SessionWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.phoenix + +import org.apache.kyuubi.engine.jdbc.phoenix.SessionSuite + +class SessionWithServerYarnModeSuite extends SessionSuite + with WithKyuubiServerAndPhoenixContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/StatementWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/StatementWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..e3270487b2a --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/StatementWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.phoenix + +import org.apache.kyuubi.engine.jdbc.phoenix.StatementSuite + +class StatementWithServerYarnModeSuite extends StatementSuite + with WithKyuubiServerAndPhoenixContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainerYarnMode.scala new file mode 100644 index 00000000000..b512f74d9dc --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainerYarnMode.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.it.jdbc.phoenix + +import java.nio.file.{Files, Path, Paths} + +import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} +import org.apache.kyuubi.engine.deploy.DeployMode +import org.apache.kyuubi.engine.jdbc.phoenix.WithPhoenixEngine + +trait WithKyuubiServerAndPhoenixContainerYarnMode + extends WithKyuubiServerAndHadoopMiniCluster with WithPhoenixEngine { + + private val phoenixJdbcConnectorPath: String = { + val keyword = "phoenix-queryserver" + + val jarsDir = Paths.get(kyuubiHome) + .resolve("integration-tests") + .resolve("kyuubi-jdbc-it") + .resolve("target") + + Files.list(jarsDir) + .filter { p: Path => p.getFileName.toString contains keyword } + .findFirst + .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } + .toAbsolutePath + .toString + } + + override protected val conf: KyuubiConf = { + KyuubiConf() + .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) + .set(ENGINE_JDBC_EXTRA_CLASSPATH, phoenixJdbcConnectorPath) + .set(ENGINE_TYPE, "JDBC") + .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) + .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) + } + + override def beforeAll(): Unit = { + val configs = withKyuubiConf + configs.foreach(config => conf.set(config._1, config._2)) + super.beforeAll() + conf + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) + } +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/OperationWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..6814445a087 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/OperationWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.postgresql + +import org.apache.kyuubi.engine.jdbc.postgresql.PostgreSQLOperationSuite + +class OperationWithServerYarnModeSuite extends PostgreSQLOperationSuite + with WithKyuubiServerAndPostgreSQLContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/SessionWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/SessionWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..4a759914fb3 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/SessionWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.postgresql + +import org.apache.kyuubi.engine.jdbc.postgresql.SessionSuite + +class SessionWithServerYarnModeSuite extends SessionSuite + with WithKyuubiServerAndPostgreSQLContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/StatementWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/StatementWithServerYarnModeSuite.scala new file mode 100644 index 00000000000..f32cdc35571 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/StatementWithServerYarnModeSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.postgresql + +import org.apache.kyuubi.engine.jdbc.postgresql.StatementSuite + +class StatementWithServerYarnModeSuite extends StatementSuite + with WithKyuubiServerAndPostgreSQLContainerYarnMode { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainerYarnMode.scala new file mode 100644 index 00000000000..5b4906d5b35 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainerYarnMode.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.it.jdbc.postgresql + +import java.nio.file.{Files, Path, Paths} + +import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} +import org.apache.kyuubi.engine.deploy.DeployMode +import org.apache.kyuubi.engine.jdbc.postgresql.WithPostgreSQLEngine + +trait WithKyuubiServerAndPostgreSQLContainerYarnMode extends WithKyuubiServerAndHadoopMiniCluster + with WithPostgreSQLEngine { + + private val postgresqlJdbcConnectorPath: String = { + val keyword = "postgresql" + + val jarsDir = Paths.get(kyuubiHome) + .resolve("integration-tests") + .resolve("kyuubi-jdbc-it") + .resolve("target") + + Files.list(jarsDir) + .filter { p: Path => p.getFileName.toString contains keyword } + .findFirst + .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } + .toAbsolutePath + .toString + } + + override protected val conf: KyuubiConf = { + KyuubiConf() + .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) + .set(ENGINE_JDBC_EXTRA_CLASSPATH, postgresqlJdbcConnectorPath) + .set(ENGINE_TYPE, "JDBC") + .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) + .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) + } + + override def beforeAll(): Unit = { + val configs = withKyuubiConf + configs.foreach(config => conf.set(config._1, config._2)) + super.beforeAll() + conf + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) + .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 170e6950c23..a0dc77f7fcf 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3131,6 +3131,19 @@ object KyuubiConf { .version("1.10.0") .fallbackConf(OPERATION_INCREMENTAL_COLLECT) + val ENGINE_JDBC_DEPLOY_MODE: ConfigEntry[String] = + buildConf("kyuubi.engine.jdbc.deploy.mode") + .doc("Configures the jdbc engine deploy mode, The value can be 'local', 'yarn'. " + + "In local mode, the engine operates on the same node as the KyuubiServer. " + + "In YARN mode, the engine runs within the Application Master (AM) container of YARN. ") + .version("1.10.0") + .stringConf + .transformToUpperCase + .checkValue( + mode => Set("LOCAL", "YARN").contains(mode), + "Invalid value for 'kyuubi.engine.jdbc.deploy.mode'. Valid values are 'local', 'yarn'.") + .createWithDefault(DeployMode.LOCAL.toString) + val ENGINE_OPERATION_CONVERT_CATALOG_DATABASE_ENABLED: ConfigEntry[Boolean] = buildConf("kyuubi.engine.operation.convert.catalog.database.enabled") .doc("When set to true, The engine converts the JDBC methods of set/get Catalog " + diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala index e2913539b46..492d635320f 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala @@ -315,8 +315,10 @@ abstract class EngineYarnModeSubmitter extends Logging { } // respect the following priority loading configuration, and distinct files // hive configuration -> hadoop configuration -> yarn configuration - val hiveConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY) - listDistinctFiles(hiveConf.get).foreach(putEntry) + if ("hive".equalsIgnoreCase(engineType)) { + val hiveConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY) + listDistinctFiles(hiveConf.get).foreach(putEntry) + } val hadoopConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY) listDistinctFiles(hadoopConf.get).foreach(putEntry) val yarnConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala index 0bb13b049f2..bb7f7ecbcf4 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala @@ -208,7 +208,14 @@ private[kyuubi] class EngineRef( extraEngineLog, defaultEngineName) case JDBC => - new JdbcProcessBuilder(appUser, doAsEnabled, conf, engineRefId, extraEngineLog) + conf.setIfMissing(JdbcProcessBuilder.JDBC_ENGINE_NAME, defaultEngineName) + JdbcProcessBuilder( + appUser, + doAsEnabled, + conf, + engineRefId, + extraEngineLog, + defaultEngineName) case CHAT => new ChatProcessBuilder(appUser, doAsEnabled, conf, engineRefId, extraEngineLog) } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala index 1afdcc3cf7c..0e914987c36 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala @@ -188,9 +188,11 @@ object KyuubiApplicationManager { case ("FLINK", Some("YARN")) => // running flink on other platforms is not yet supported setupFlinkYarnTag(applicationTag, conf) - // other engine types are running locally yet case ("HIVE", Some("YARN")) => setupEngineYarnModeTag(applicationTag, conf) + case ("JDBC", Some("YARN")) => + setupEngineYarnModeTag(applicationTag, conf) + // other engine types are running locally yet case _ => } } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala index de93fd44b55..0a922efbf78 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala @@ -23,12 +23,15 @@ import java.nio.file.Paths import scala.collection.mutable import com.google.common.annotations.VisibleForTesting +import org.apache.hadoop.security.UserGroupInformation -import org.apache.kyuubi.{Logging, SCALA_COMPILE_VERSION, Utils} +import org.apache.kyuubi.{KyuubiException, Logging, SCALA_COMPILE_VERSION, Utils} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_JDBC_JAVA_OPTIONS, ENGINE_JDBC_MEMORY} +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_DEPLOY_YARN_MODE_APP_NAME, ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_DEPLOY_MODE, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_JDBC_JAVA_OPTIONS, ENGINE_JDBC_MEMORY, ENGINE_KEYTAB, ENGINE_PRINCIPAL} import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.ProcBuilder +import org.apache.kyuubi.engine.deploy.DeployMode +import org.apache.kyuubi.engine.deploy.DeployMode.{LOCAL, YARN} import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.util.command.CommandLineUtils._ @@ -113,3 +116,55 @@ class JdbcProcessBuilder( } } } + +object JdbcProcessBuilder extends Logging { + + final val JDBC_ENGINE_NAME = "jdbc.engine.name" + + def apply( + proxyUser: String, + doAsEnabled: Boolean, + conf: KyuubiConf, + engineRefId: String, + extraEngineLog: Option[OperationLog], + defaultEngineName: String): JdbcProcessBuilder = { + checkKeytab(proxyUser, conf) + DeployMode.withName(conf.get(ENGINE_JDBC_DEPLOY_MODE)) match { + case LOCAL => + new JdbcProcessBuilder(proxyUser, doAsEnabled, conf, engineRefId, extraEngineLog) + case YARN => + warn(s"Hive on YARN model is experimental.") + conf.setIfMissing(ENGINE_DEPLOY_YARN_MODE_APP_NAME, Some(defaultEngineName)) + new JdbcYarnModeProcessBuilder(proxyUser, doAsEnabled, conf, engineRefId, extraEngineLog) + case other => throw new KyuubiException(s"Unsupported deploy mode: $other") + } + } + + private def checkKeytab(proxyUser: String, conf: KyuubiConf): Unit = { + val principal = conf.get(ENGINE_PRINCIPAL) + val keytab = conf.get(ENGINE_KEYTAB) + if (!UserGroupInformation.isSecurityEnabled) { + if (principal.isDefined || keytab.isDefined) { + warn("Principal and keytab takes no effect when hadoop security is not enabled.") + } + return + } + + require( + principal.isDefined == keytab.isDefined, + s"Both principal and keytab must be defined, or neither.") + if (principal.isDefined && keytab.isDefined) { + val ugi = UserGroupInformation + .loginUserFromKeytabAndReturnUGI(principal.get, keytab.get) + require( + ugi.getShortUserName == proxyUser, + s"Proxy user: $proxyUser is not same with " + + s"engine principal: ${ugi.getShortUserName}.") + } + + val deployMode = DeployMode.withName(conf.get(ENGINE_JDBC_DEPLOY_MODE)) + if (principal.isEmpty && keytab.isEmpty && deployMode == YARN) { + warn("Hive on YARN can not work properly without principal and keytab.") + } + } +} diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala new file mode 100644 index 00000000000..9a40e3eb74f --- /dev/null +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc + +import java.io.File +import java.nio.file.Paths +import java.util + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +import org.apache.kyuubi.{Logging, SCALA_COMPILE_VERSION, Utils} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_JDBC_MEMORY} +import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_ID, KYUUBI_SESSION_USER_KEY} +import org.apache.kyuubi.engine.{ApplicationManagerInfo, KyuubiApplicationManager} +import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter._ +import org.apache.kyuubi.operation.log.OperationLog +import org.apache.kyuubi.util.command.CommandLineUtils.{confKeyValue, confKeyValues} + +/** + * A process builder for Hive on Yarn. + * + * It will new a process on kyuubi server side to submit hive engine to yarn. + */ +class JdbcYarnModeProcessBuilder( + override val proxyUser: String, + override val doAsEnabled: Boolean, + override val conf: KyuubiConf, + override val engineRefId: String, + override val extraEngineLog: Option[OperationLog] = None) + extends JdbcProcessBuilder(proxyUser, doAsEnabled, conf, engineRefId, extraEngineLog) + with Logging { + + override protected def mainClass: String = + "org.apache.kyuubi.engine.jdbc.deploy.JdbcYarnModeSubmitter" + + override def isClusterMode(): Boolean = true + + override def clusterManager(): Option[String] = Some("yarn") + + override def appMgrInfo(): ApplicationManagerInfo = ApplicationManagerInfo(clusterManager()) + + override protected val commands: Iterable[String] = { + KyuubiApplicationManager.tagApplication(engineRefId, shortName, clusterManager(), conf) + val buffer = new ArrayBuffer[String]() + buffer += executable + + val memory = conf.get(ENGINE_JDBC_MEMORY) + buffer += s"-Xmx$memory" + buffer += "-cp" + + val classpathEntries = new util.LinkedHashSet[String] + classpathEntries.addAll(hadoopConfFiles()) + classpathEntries.addAll(yarnConfFiles()) + classpathEntries.addAll(jarFiles(true)) + + buffer += classpathEntries.asScala.mkString(File.pathSeparator) + buffer += mainClass + + buffer ++= confKeyValue(KYUUBI_SESSION_USER_KEY, proxyUser) + buffer ++= confKeyValue(KYUUBI_ENGINE_ID, engineRefId) + + buffer ++= confKeyValue( + KYUUBI_ENGINE_DEPLOY_YARN_MODE_JARS_KEY, + jarFiles(false).asScala.mkString(KYUUBI_ENGINE_DEPLOY_YARN_MODE_ARCHIVE_SEPARATOR)) + buffer ++= confKeyValue( + KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY, + hadoopConfFiles().asScala.mkString(KYUUBI_ENGINE_DEPLOY_YARN_MODE_ARCHIVE_SEPARATOR)) + buffer ++= confKeyValue( + KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY, + yarnConfFiles().asScala.mkString(KYUUBI_ENGINE_DEPLOY_YARN_MODE_ARCHIVE_SEPARATOR)) + + buffer ++= confKeyValues(conf.getAll) + + buffer + } + + private def jarFiles(isClasspath: Boolean): util.LinkedHashSet[String] = { + val jarEntries = new util.LinkedHashSet[String] + mainResource.foreach(jarEntries.add) + val javaOptions = conf.get(ENGINE_JDBC_EXTRA_CLASSPATH) + if (isClasspath) { + javaOptions.foreach(jarEntries.add) + } + mainResource.foreach { path => + val parent = Paths.get(path).getParent + if (Utils.isTesting) { + // add dev classpath + val jdbcDeps = parent + .resolve(s"scala-$SCALA_COMPILE_VERSION") + .resolve("jars") + jarEntries.add(s"$jdbcDeps${appendClasspathSuffix(isClasspath)}") + } else { + // add prod classpath + jarEntries.add(s"$parent${appendClasspathSuffix(isClasspath)}") + } + } + jarEntries + } + + private def hadoopConfFiles(): util.LinkedHashSet[String] = { + val confEntries = new util.LinkedHashSet[String] + env.get("HADOOP_CONF_DIR").foreach(confEntries.add) + confEntries + } + + private def yarnConfFiles(): util.LinkedHashSet[String] = { + val confEntries = new util.LinkedHashSet[String] + env.get("YARN_CONF_DIR").foreach(confEntries.add) + confEntries + } + + private def appendClasspathSuffix(isClasspath: Boolean): String = { + if (isClasspath) { + s"${File.separator}*" + } else { + "" + } + } +} diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilderSuite.scala new file mode 100644 index 00000000000..6c415e88e38 --- /dev/null +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilderSuite.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL} +import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter.{KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY, KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY} +import org.apache.kyuubi.engine.hive.HiveProcessBuilder.HIVE_HADOOP_CLASSPATH_KEY + +class JdbcYarnModeProcessBuilderSuite extends KyuubiFunSuite { + + test("jdbc yarn mode process builder") { + val conf = KyuubiConf().set("kyuubi.on", "off") + .set(ENGINE_JDBC_CONNECTION_URL.key, "") + .set(ENGINE_JDBC_CONNECTION_PASSWORD.key, "123456") + val builder = new JdbcYarnModeProcessBuilder("kyuubi", true, conf, "") + val commands = builder.toString.split('\n') + assert(commands.head.contains("bin/java"), "wrong exec") + assert(builder.toString.contains("--conf kyuubi.session.user=kyuubi")) + assert(commands.exists(ss => ss.contains("kyuubi-jdbc-engine")), "wrong classpath") + assert(builder.toString.contains("--conf kyuubi.on=off")) + } + + test("hadoop conf dir") { + val conf = KyuubiConf().set("kyuubi.on", "off") + .set(ENGINE_JDBC_CONNECTION_URL.key, "") + .set(ENGINE_JDBC_CONNECTION_PASSWORD.key, "123456") + val builder = new JdbcYarnModeProcessBuilder("kyuubi", true, conf, "") { + override def env: Map[String, String] = + super.env + ("HADOOP_CONF_DIR" -> "/etc/hadoop/conf") + + (HIVE_HADOOP_CLASSPATH_KEY -> "/hadoop") + } + assert(builder.toString.contains( + s"--conf $KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY=/etc/hadoop/conf")) + } + + test("yarn conf dir") { + val conf = KyuubiConf().set("kyuubi.on", "off") + .set(ENGINE_JDBC_CONNECTION_URL.key, "") + .set(ENGINE_JDBC_CONNECTION_PASSWORD.key, "123456") + val builder = new JdbcYarnModeProcessBuilder("kyuubi", true, conf, "") { + override def env: Map[String, String] = + super.env + ("YARN_CONF_DIR" -> "/etc/hadoop/yarn/conf") + + (HIVE_HADOOP_CLASSPATH_KEY -> "/hadoop") + } + assert(builder.toString.contains( + s"--conf $KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY=/etc/hadoop/yarn/conf")) + } +} From bf33de300b4984bd5fefc12badfe9cb6e3e1e42d Mon Sep 17 00:00:00 2001 From: senmiaoliu Date: Tue, 9 Apr 2024 17:09:26 +0800 Subject: [PATCH 2/5] fix style --- .../hive/deploy/HiveYarnModeSubmitter.scala | 17 ++--- .../deploy/HiveYarnModeSubmitterSuite.scala | 40 ------------ .../jdbc/deploy/JdbcYarnModeSubmitter.scala | 19 +----- .../deploy/JdbcYarnModeSubmitterSuite.scala | 40 ------------ .../OperationWithServerYarnModeSuite.scala | 27 -------- .../SessionWithServerYarnModeSuite.scala | 27 -------- .../StatementWithServerYarnModeSuite.scala | 27 -------- ...ServerAndClickHouseContainerYarnMode.scala | 63 ------------------ .../OperationWithServerYarnModeSuite.scala | 27 -------- ...yuubiServerAndDorisContainerYarnMode.scala | 64 ------------------- .../OperationWithServerYarnModeSuite.scala | 27 -------- .../SessionWithServerYarnModeSuite.scala | 27 -------- .../StatementWithServerYarnModeSuite.scala | 27 -------- ...uubiServerAndImpalaContainerYarnMode.scala | 64 ------------------- .../OperationWithServerYarnModeSuite.scala | 27 -------- .../SessionWithServerYarnModeSuite.scala | 27 -------- .../StatementWithServerYarnModeSuite.scala | 27 -------- ...ubiServerAndPhoenixContainerYarnMode.scala | 63 ------------------ .../OperationWithServerYarnModeSuite.scala | 27 -------- .../SessionWithServerYarnModeSuite.scala | 27 -------- .../StatementWithServerYarnModeSuite.scala | 27 -------- ...ServerAndPostgreSQLContainerYarnMode.scala | 63 ------------------ .../deploy/yarn/EngineYarnModeSubmitter.scala | 31 ++++++--- .../yarn/EngineYarnModeSubmitterSuite.scala | 15 ++++- .../engine/jdbc/JdbcProcessBuilder.scala | 2 +- .../jdbc/JdbcYarnModeProcessBuilder.scala | 4 +- 26 files changed, 44 insertions(+), 792 deletions(-) delete mode 100644 externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitterSuite.scala delete mode 100644 externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitterSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/OperationWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/SessionWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/StatementWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainerYarnMode.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainerYarnMode.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainerYarnMode.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/OperationWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/SessionWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/StatementWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainerYarnMode.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/OperationWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/SessionWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/StatementWithServerYarnModeSuite.scala delete mode 100644 integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainerYarnMode.scala diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala index a717f9dfd01..e223e2a10f5 100644 --- a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala @@ -26,6 +26,7 @@ import org.apache.kyuubi.Utils import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.ENGINE_HIVE_EXTRA_CLASSPATH import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter +import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter.KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY import org.apache.kyuubi.engine.hive.HiveSQLEngine object HiveYarnModeSubmitter extends EngineYarnModeSubmitter { @@ -60,18 +61,8 @@ object HiveYarnModeSubmitter extends EngineYarnModeSubmitter { jars.toSeq } - private[hive] def parseClasspath(classpath: String, jars: ListBuffer[File]): Unit = { - classpath.split(":").filter(_.nonEmpty).foreach { cp => - if (cp.endsWith("/*")) { - val dir = cp.substring(0, cp.length - 2) - new File(dir) match { - case f if f.isDirectory => - f.listFiles().filter(_.getName.endsWith(".jar")).foreach(jars += _) - case _ => - } - } else { - jars += new File(cp) - } - } + override def listConfFiles(): Seq[File] = { + val hiveConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY) + listDistinctFiles(hiveConf.get) ++ super.listConfFiles() } } diff --git a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitterSuite.scala b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitterSuite.scala deleted file mode 100644 index b04fefa0afd..00000000000 --- a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitterSuite.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kyuubi.engine.hive.deploy - -import java.io.File - -import scala.collection.mutable.ListBuffer - -import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION} -import org.apache.kyuubi.util.JavaUtils - -class HiveYarnModeSubmitterSuite extends KyuubiFunSuite { - val hiveEngineHome: String = JavaUtils.getCodeSourceLocation(getClass).split("/target")(0) - - test("hadoop class path") { - val jars = new ListBuffer[File] - val classpath = - s"$hiveEngineHome/target/scala-$SCALA_COMPILE_VERSION/jars/*:" + - s"$hiveEngineHome/target/kyuubi-hive-sql-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" - HiveYarnModeSubmitter.parseClasspath(classpath, jars) - assert(jars.nonEmpty) - assert(jars.exists( - _.getName == s"kyuubi-hive-sql-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar")) - } - -} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala index fbb87587d63..a585a9a6f46 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitter.scala @@ -49,29 +49,12 @@ object JdbcYarnModeSubmitter extends EngineYarnModeSubmitter { override def engineMainClass(): String = JdbcSQLEngine.getClass.getName /** - * Jar list for the Hive engine. + * Jar list for the JDBC engine. */ override def engineExtraJars(): Seq[File] = { - val hadoopCp = sys.env.get("HIVE_HADOOP_CLASSPATH") val extraCp = kyuubiConf.get(ENGINE_JDBC_EXTRA_CLASSPATH) val jars = new ListBuffer[File] - hadoopCp.foreach(cp => parseClasspath(cp, jars)) extraCp.foreach(cp => parseClasspath(cp, jars)) jars.toSeq } - - private[jdbc] def parseClasspath(classpath: String, jars: ListBuffer[File]): Unit = { - classpath.split(":").filter(_.nonEmpty).foreach { cp => - if (cp.endsWith("/*")) { - val dir = cp.substring(0, cp.length - 2) - new File(dir) match { - case f if f.isDirectory => - f.listFiles().filter(_.getName.endsWith(".jar")).foreach(jars += _) - case _ => - } - } else { - jars += new File(cp) - } - } - } } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitterSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitterSuite.scala deleted file mode 100644 index 771de4d508c..00000000000 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/deploy/JdbcYarnModeSubmitterSuite.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kyuubi.engine.jdbc.deploy - -import java.io.File - -import scala.collection.mutable.ListBuffer - -import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION} -import org.apache.kyuubi.util.JavaUtils - -class JdbcYarnModeSubmitterSuite extends KyuubiFunSuite { - val jdbcEngineHome: String = JavaUtils.getCodeSourceLocation(getClass).split("/target")(0) - - test("hadoop class path") { - val jars = new ListBuffer[File] - val classpath = - s"$jdbcEngineHome/target/scala-$SCALA_COMPILE_VERSION/jars/*:" + - s"$jdbcEngineHome/target/kyuubi-jdbc-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" - JdbcYarnModeSubmitter.parseClasspath(classpath, jars) - assert(jars.nonEmpty) - assert(jars.exists( - _.getName == s"kyuubi-jdbc-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar")) - } - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/OperationWithServerYarnModeSuite.scala deleted file mode 100644 index db8627d6646..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/OperationWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.clickhouse - -import org.apache.kyuubi.engine.jdbc.clickhouse.ClickHouseOperationSuite - -class OperationWithServerYarnModeSuite extends ClickHouseOperationSuite - with WithKyuubiServerAndClickHouseContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/SessionWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/SessionWithServerYarnModeSuite.scala deleted file mode 100644 index 3d6844ef78f..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/SessionWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.clickhouse - -import org.apache.kyuubi.engine.jdbc.clickhouse.ClickHouseSessionSuite - -class SessionWithServerYarnModeSuite extends ClickHouseSessionSuite - with WithKyuubiServerAndClickHouseContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/StatementWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/StatementWithServerYarnModeSuite.scala deleted file mode 100644 index f1db4ab1a4e..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/StatementWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.clickhouse - -import org.apache.kyuubi.engine.jdbc.clickhouse.ClickHouseStatementSuite - -class StatementWithServerYarnModeSuite extends ClickHouseStatementSuite - with WithKyuubiServerAndClickHouseContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainerYarnMode.scala deleted file mode 100644 index 1043fe5cd68..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainerYarnMode.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kyuubi.it.jdbc.clickhouse - -import java.nio.file.{Files, Path, Paths} - -import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster -import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} -import org.apache.kyuubi.engine.deploy.DeployMode -import org.apache.kyuubi.engine.jdbc.clickhouse.WithClickHouseEngine - -trait WithKyuubiServerAndClickHouseContainerYarnMode - extends WithKyuubiServerAndHadoopMiniCluster with WithClickHouseEngine { - - private val clickHouseJdbcConnectorPath: String = { - val keyword = "clickhouse-jdbc" - - val jarsDir = Paths.get(kyuubiHome) - .resolve("integration-tests") - .resolve("kyuubi-jdbc-it") - .resolve("target") - - Files.list(jarsDir) - .filter { p: Path => p.getFileName.toString contains keyword } - .findFirst - .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } - .toAbsolutePath - .toString - } - - override protected val conf: KyuubiConf = { - KyuubiConf() - .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) - .set(ENGINE_JDBC_EXTRA_CLASSPATH, clickHouseJdbcConnectorPath) - .set(ENGINE_TYPE, "JDBC") - .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) - .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) - } - - override def beforeAll(): Unit = { - val configs = withKyuubiConf - configs.foreach(config => conf.set(config._1, config._2)) - super.beforeAll() - conf - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) - } -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerYarnModeSuite.scala deleted file mode 100644 index 08737677491..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.doris - -import org.apache.kyuubi.engine.jdbc.doris.DorisOperationSuite - -class OperationWithServerYarnModeSuite extends DorisOperationSuite - with WithKyuubiServerAndDorisContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainerYarnMode.scala deleted file mode 100644 index 56c5ca8e00f..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainerYarnMode.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.doris - -import java.nio.file.{Files, Path, Paths} - -import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster -import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} -import org.apache.kyuubi.engine.deploy.DeployMode -import org.apache.kyuubi.engine.jdbc.doris.WithDorisEngine - -trait WithKyuubiServerAndDorisContainerYarnMode extends WithKyuubiServerAndHadoopMiniCluster - with WithDorisEngine { - - private val mysqlJdbcConnectorPath: String = { - val keyword = "mysql-connector" - - val jarsDir = Paths.get(kyuubiHome) - .resolve("integration-tests") - .resolve("kyuubi-jdbc-it") - .resolve("target") - - Files.list(jarsDir) - .filter { p: Path => p.getFileName.toString contains keyword } - .findFirst - .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } - .toAbsolutePath - .toString - } - - override protected val conf: KyuubiConf = { - KyuubiConf() - .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) - .set(ENGINE_JDBC_EXTRA_CLASSPATH, mysqlJdbcConnectorPath) - .set(ENGINE_TYPE, "JDBC") - .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) - .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) - } - - override def beforeAll(): Unit = { - val configs = withKyuubiConf - configs.foreach(config => conf.set(config._1, config._2)) - super.beforeAll() - conf - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) - } -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerYarnModeSuite.scala deleted file mode 100644 index 1c46c98df80..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.impala - -import org.apache.kyuubi.engine.jdbc.impala.ImpalaOperationSuite - -class OperationWithServerYarnModeSuite extends ImpalaOperationSuite - with WithKyuubiServerAndImpalaContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerYarnModeSuite.scala deleted file mode 100644 index c56df4753d9..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.impala - -import org.apache.kyuubi.engine.jdbc.impala.SessionSuite - -class SessionWithServerYarnModeSuite extends SessionSuite - with WithKyuubiServerAndImpalaContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerYarnModeSuite.scala deleted file mode 100644 index 38f1eed6605..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.impala - -import org.apache.kyuubi.engine.jdbc.impala.StatementSuite - -class StatementWithServerYarnModeSuite extends StatementSuite - with WithKyuubiServerAndImpalaContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainerYarnMode.scala deleted file mode 100644 index 6980aa8d521..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainerYarnMode.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.impala - -import java.nio.file.{Files, Path, Paths} - -import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster -import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} -import org.apache.kyuubi.engine.deploy.DeployMode -import org.apache.kyuubi.engine.jdbc.impala.WithImpalaEngine - -trait WithKyuubiServerAndImpalaContainerYarnMode extends WithKyuubiServerAndHadoopMiniCluster - with WithImpalaEngine { - - private val hiveJdbcConnectorPath: String = { - val keyword = "kyuubi-hive-jdbc-shaded" - - val jarsDir = Paths.get(kyuubiHome) - .resolve("integration-tests") - .resolve("kyuubi-jdbc-it") - .resolve("target") - - Files.list(jarsDir) - .filter { p: Path => p.getFileName.toString contains keyword } - .findFirst - .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } - .toAbsolutePath - .toString - } - - override protected val conf: KyuubiConf = { - KyuubiConf() - .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) - .set(ENGINE_JDBC_EXTRA_CLASSPATH, hiveJdbcConnectorPath) - .set(ENGINE_TYPE, "JDBC") - .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) - .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) - } - - override def beforeAll(): Unit = { - val configs = withKyuubiConf - configs.foreach(config => conf.set(config._1, config._2)) - super.beforeAll() - conf - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) - } -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/OperationWithServerYarnModeSuite.scala deleted file mode 100644 index f55ddfa47c3..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/OperationWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.phoenix - -import org.apache.kyuubi.engine.jdbc.phoenix.PhoenixOperationSuite - -class OperationWithServerYarnModeSuite extends PhoenixOperationSuite - with WithKyuubiServerAndPhoenixContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/SessionWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/SessionWithServerYarnModeSuite.scala deleted file mode 100644 index a614270e85f..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/SessionWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.phoenix - -import org.apache.kyuubi.engine.jdbc.phoenix.SessionSuite - -class SessionWithServerYarnModeSuite extends SessionSuite - with WithKyuubiServerAndPhoenixContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/StatementWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/StatementWithServerYarnModeSuite.scala deleted file mode 100644 index e3270487b2a..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/StatementWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.phoenix - -import org.apache.kyuubi.engine.jdbc.phoenix.StatementSuite - -class StatementWithServerYarnModeSuite extends StatementSuite - with WithKyuubiServerAndPhoenixContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainerYarnMode.scala deleted file mode 100644 index b512f74d9dc..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainerYarnMode.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kyuubi.it.jdbc.phoenix - -import java.nio.file.{Files, Path, Paths} - -import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster -import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} -import org.apache.kyuubi.engine.deploy.DeployMode -import org.apache.kyuubi.engine.jdbc.phoenix.WithPhoenixEngine - -trait WithKyuubiServerAndPhoenixContainerYarnMode - extends WithKyuubiServerAndHadoopMiniCluster with WithPhoenixEngine { - - private val phoenixJdbcConnectorPath: String = { - val keyword = "phoenix-queryserver" - - val jarsDir = Paths.get(kyuubiHome) - .resolve("integration-tests") - .resolve("kyuubi-jdbc-it") - .resolve("target") - - Files.list(jarsDir) - .filter { p: Path => p.getFileName.toString contains keyword } - .findFirst - .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } - .toAbsolutePath - .toString - } - - override protected val conf: KyuubiConf = { - KyuubiConf() - .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) - .set(ENGINE_JDBC_EXTRA_CLASSPATH, phoenixJdbcConnectorPath) - .set(ENGINE_TYPE, "JDBC") - .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) - .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) - } - - override def beforeAll(): Unit = { - val configs = withKyuubiConf - configs.foreach(config => conf.set(config._1, config._2)) - super.beforeAll() - conf - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) - } -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/OperationWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/OperationWithServerYarnModeSuite.scala deleted file mode 100644 index 6814445a087..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/OperationWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.postgresql - -import org.apache.kyuubi.engine.jdbc.postgresql.PostgreSQLOperationSuite - -class OperationWithServerYarnModeSuite extends PostgreSQLOperationSuite - with WithKyuubiServerAndPostgreSQLContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/SessionWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/SessionWithServerYarnModeSuite.scala deleted file mode 100644 index 4a759914fb3..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/SessionWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.postgresql - -import org.apache.kyuubi.engine.jdbc.postgresql.SessionSuite - -class SessionWithServerYarnModeSuite extends SessionSuite - with WithKyuubiServerAndPostgreSQLContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/StatementWithServerYarnModeSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/StatementWithServerYarnModeSuite.scala deleted file mode 100644 index f32cdc35571..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/StatementWithServerYarnModeSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.it.jdbc.postgresql - -import org.apache.kyuubi.engine.jdbc.postgresql.StatementSuite - -class StatementWithServerYarnModeSuite extends StatementSuite - with WithKyuubiServerAndPostgreSQLContainerYarnMode { - - override protected def jdbcUrl: String = getJdbcUrl - -} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainerYarnMode.scala deleted file mode 100644 index 5b4906d5b35..00000000000 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainerYarnMode.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kyuubi.it.jdbc.postgresql - -import java.nio.file.{Files, Path, Paths} - -import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster -import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} -import org.apache.kyuubi.engine.deploy.DeployMode -import org.apache.kyuubi.engine.jdbc.postgresql.WithPostgreSQLEngine - -trait WithKyuubiServerAndPostgreSQLContainerYarnMode extends WithKyuubiServerAndHadoopMiniCluster - with WithPostgreSQLEngine { - - private val postgresqlJdbcConnectorPath: String = { - val keyword = "postgresql" - - val jarsDir = Paths.get(kyuubiHome) - .resolve("integration-tests") - .resolve("kyuubi-jdbc-it") - .resolve("target") - - Files.list(jarsDir) - .filter { p: Path => p.getFileName.toString contains keyword } - .findFirst - .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } - .toAbsolutePath - .toString - } - - override protected val conf: KyuubiConf = { - KyuubiConf() - .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) - .set(ENGINE_JDBC_EXTRA_CLASSPATH, postgresqlJdbcConnectorPath) - .set(ENGINE_TYPE, "JDBC") - .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) - .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) - } - - override def beforeAll(): Unit = { - val configs = withKyuubiConf - configs.foreach(config => conf.set(config._1, config._2)) - super.beforeAll() - conf - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) - .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) - } -} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala index 492d635320f..3924ab8eb26 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala @@ -315,15 +315,7 @@ abstract class EngineYarnModeSubmitter extends Logging { } // respect the following priority loading configuration, and distinct files // hive configuration -> hadoop configuration -> yarn configuration - if ("hive".equalsIgnoreCase(engineType)) { - val hiveConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY) - listDistinctFiles(hiveConf.get).foreach(putEntry) - } - val hadoopConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY) - listDistinctFiles(hadoopConf.get).foreach(putEntry) - val yarnConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY) - listDistinctFiles(yarnConf.get).foreach(putEntry) - + listConfFiles().foreach(putEntry) val properties = confToProperties(kyuubiConf) amKeytabFileName.foreach(kt => properties.put(ENGINE_KEYTAB.key, kt)) writePropertiesToArchive(properties, KYUUBI_CONF_FILE, confStream) @@ -354,6 +346,12 @@ abstract class EngineYarnModeSubmitter extends Logging { }.toSeq } + def listConfFiles(): Seq[File] = { + val hadoopConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY) + val yarnConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY) + listDistinctFiles(hadoopConf.get) ++ listDistinctFiles(yarnConf.get) + } + private def distribute( srcPath: Path, resType: LocalResourceType, @@ -488,6 +486,21 @@ abstract class EngineYarnModeSubmitter extends Logging { writer.flush() out.closeEntry() } + + def parseClasspath(classpath: String, jars: ListBuffer[File]): Unit = { + classpath.split(":").filter(_.nonEmpty).foreach { cp => + if (cp.endsWith("/*")) { + val dir = cp.substring(0, cp.length - 2) + new File(dir) match { + case f if f.isDirectory => + f.listFiles().filter(_.getName.endsWith(".jar")).foreach(jars += _) + case _ => + } + } else { + jars += new File(cp) + } + } + } } object EngineYarnModeSubmitter { diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitterSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitterSuite.scala index f12d3f6fd7f..b9f419ea796 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitterSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitterSuite.scala @@ -18,12 +18,14 @@ package org.apache.kyuubi.engine.deploy.yarn import java.io.File +import scala.collection.mutable.ListBuffer + import org.apache.hadoop.fs.Path import org.apache.hadoop.yarn.api.ApplicationConstants.Environment import org.scalatest.matchers.must.Matchers import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper -import org.apache.kyuubi.{KyuubiFunSuite, Utils} +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION, Utils} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter.KYUUBI_ENGINE_DEPLOY_YARN_MODE_JARS_KEY import org.apache.kyuubi.util.JavaUtils @@ -66,6 +68,17 @@ class EngineYarnModeSubmitterSuite extends KyuubiFunSuite with Matchers { assert(targetFiles.length == files.length) } + test("hadoop class path") { + val jars = new ListBuffer[File] + val classpath = + s"$kyuubiHome/target/scala-$SCALA_COMPILE_VERSION/jars/*:" + + s"$kyuubiHome/target/kyuubi-common-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" + MockEngineYarnModeSubmitter.parseClasspath(classpath, jars) + assert(jars.nonEmpty) + assert(jars.exists( + _.getName == s"kyuubi-common-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar")) + } + } object MockEngineYarnModeSubmitter extends EngineYarnModeSubmitter { diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala index 0a922efbf78..c70c386aae5 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala @@ -133,7 +133,7 @@ object JdbcProcessBuilder extends Logging { case LOCAL => new JdbcProcessBuilder(proxyUser, doAsEnabled, conf, engineRefId, extraEngineLog) case YARN => - warn(s"Hive on YARN model is experimental.") + warn(s"JDBC on YARN model is experimental.") conf.setIfMissing(ENGINE_DEPLOY_YARN_MODE_APP_NAME, Some(defaultEngineName)) new JdbcYarnModeProcessBuilder(proxyUser, doAsEnabled, conf, engineRefId, extraEngineLog) case other => throw new KyuubiException(s"Unsupported deploy mode: $other") diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala index 9a40e3eb74f..fbc4416d73f 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcYarnModeProcessBuilder.scala @@ -33,9 +33,9 @@ import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.util.command.CommandLineUtils.{confKeyValue, confKeyValues} /** - * A process builder for Hive on Yarn. + * A process builder for JDBC on Yarn. * - * It will new a process on kyuubi server side to submit hive engine to yarn. + * It will new a process on kyuubi server side to submit jdbc engine to yarn. */ class JdbcYarnModeProcessBuilder( override val proxyUser: String, From b3e11444507be0182ad19ea9c933458aefd7f73f Mon Sep 17 00:00:00 2001 From: senmiaoliu Date: Wed, 10 Apr 2024 16:23:54 +0800 Subject: [PATCH 3/5] fix style --- .../kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala | 2 ++ .../kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala index e223e2a10f5..793fcdc7a4a 100644 --- a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala @@ -62,6 +62,8 @@ object HiveYarnModeSubmitter extends EngineYarnModeSubmitter { } override def listConfFiles(): Seq[File] = { + // respect the following priority loading configuration, and distinct files + // hive configuration -> hadoop configuration -> yarn configuration val hiveConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY) listDistinctFiles(hiveConf.get) ++ super.listConfFiles() } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala index 3924ab8eb26..e0fbe166748 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala @@ -313,8 +313,6 @@ abstract class EngineYarnModeSubmitter extends Logging { env) } } - // respect the following priority loading configuration, and distinct files - // hive configuration -> hadoop configuration -> yarn configuration listConfFiles().foreach(putEntry) val properties = confToProperties(kyuubiConf) amKeytabFileName.foreach(kt => properties.put(ENGINE_KEYTAB.key, kt)) @@ -347,6 +345,8 @@ abstract class EngineYarnModeSubmitter extends Logging { } def listConfFiles(): Seq[File] = { + // respect the following priority loading configuration, and distinct files + // hadoop configuration -> yarn configuration val hadoopConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY) val yarnConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY) listDistinctFiles(hadoopConf.get) ++ listDistinctFiles(yarnConf.get) From 86e032688089e00667231706d2eed03dd0e6dc6e Mon Sep 17 00:00:00 2001 From: senmiaoliu Date: Thu, 11 Apr 2024 19:36:00 +0800 Subject: [PATCH 4/5] fix style --- .../apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala | 13 ++----------- .../deploy/yarn/EngineYarnModeSubmitter.scala | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala index a6291581c23..2d1aeee9559 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala @@ -72,6 +72,7 @@ object JdbcSQLEngine extends Logging { def main(args: Array[String]): Unit = { SignalRegister.registerLogger(logger) + try { Utils.fromCommandLineArgs(args, kyuubiConf) kyuubiConf.setIfMissing(KyuubiConf.FRONTEND_THRIFT_BINARY_BIND_PORT, 0) @@ -88,17 +89,7 @@ object JdbcSQLEngine extends Logging { UserGroupInformation.loginUserFromKeytab(principal.get, keytab.get) UserGroupInformation.getCurrentUser case DeployMode.LOCAL if proxyUser.get != realUser.getShortUserName => - val newUGI = UserGroupInformation.createProxyUser(proxyUser.get, realUser) - newUGI.doAs(new PrivilegedExceptionAction[Unit] { - override def run(): Unit = { - val engineCredentials = - kyuubiConf.getOption(KyuubiReservedKeys.KYUUBI_ENGINE_CREDENTIALS_KEY) - kyuubiConf.unset(KyuubiReservedKeys.KYUUBI_ENGINE_CREDENTIALS_KEY) - engineCredentials.filter(_.nonEmpty).foreach { credentials => - } - } - }) - newUGI + UserGroupInformation.createProxyUser(proxyUser.get, realUser) case _ => UserGroupInformation.getCurrentUser } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala index e0fbe166748..7126cd25ae7 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala @@ -346,7 +346,7 @@ abstract class EngineYarnModeSubmitter extends Logging { def listConfFiles(): Seq[File] = { // respect the following priority loading configuration, and distinct files - // hadoop configuration -> yarn configuration + // hive configuration -> hadoop configuration -> yarn configuration val hadoopConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY) val yarnConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY) listDistinctFiles(hadoopConf.get) ++ listDistinctFiles(yarnConf.get) From 5ed4af041412893de49edea4dc8fa6881598bad3 Mon Sep 17 00:00:00 2001 From: senmiaoliu Date: Fri, 12 Apr 2024 09:33:45 +0800 Subject: [PATCH 5/5] fix style --- .../scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala | 1 + .../jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala | 2 +- .../kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala | 2 +- .../org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala index 2d1aeee9559..ca620207c11 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala @@ -89,6 +89,7 @@ object JdbcSQLEngine extends Logging { UserGroupInformation.loginUserFromKeytab(principal.get, keytab.get) UserGroupInformation.getCurrentUser case DeployMode.LOCAL if proxyUser.get != realUser.getShortUserName => + kyuubiConf.unset(KyuubiReservedKeys.KYUUBI_ENGINE_CREDENTIALS_KEY) UserGroupInformation.createProxyUser(proxyUser.get, realUser) case _ => UserGroupInformation.getCurrentUser diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala index d016cba006b..0567136e13f 100644 --- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala @@ -48,7 +48,7 @@ class WithKyuubiServerAndMySQLContainerYarnMode extends MySQLOperationSuite .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) .set(ENGINE_JDBC_EXTRA_CLASSPATH, mysqlJdbcConnectorPath) .set(ENGINE_TYPE, "JDBC") - .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) + .set(KyuubiConf.ENGINE_JDBC_DEPLOY_MODE, DeployMode.YARN.toString) .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala index 7126cd25ae7..e0fbe166748 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala @@ -346,7 +346,7 @@ abstract class EngineYarnModeSubmitter extends Logging { def listConfFiles(): Seq[File] = { // respect the following priority loading configuration, and distinct files - // hive configuration -> hadoop configuration -> yarn configuration + // hadoop configuration -> yarn configuration val hadoopConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY) val yarnConf = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY) listDistinctFiles(hadoopConf.get) ++ listDistinctFiles(yarnConf.get) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala index c70c386aae5..e31376f365a 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala @@ -164,7 +164,7 @@ object JdbcProcessBuilder extends Logging { val deployMode = DeployMode.withName(conf.get(ENGINE_JDBC_DEPLOY_MODE)) if (principal.isEmpty && keytab.isEmpty && deployMode == YARN) { - warn("Hive on YARN can not work properly without principal and keytab.") + warn("JDBC on YARN can not work properly without principal and keytab.") } } }