From eb49537ea0c3d11234d5f8e93f59d65176d7f02c Mon Sep 17 00:00:00 2001 From: fwang12 Date: Thu, 30 Mar 2023 18:28:28 +0800 Subject: [PATCH 1/5] conf overlay --- .../client/api/v1/dto/StatementRequest.java | 17 ++++++++++++++++- .../kyuubi/server/api/v1/SessionsResource.scala | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java index 436017f3c1e..27e84de5237 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java @@ -17,6 +17,8 @@ package org.apache.kyuubi.client.api.v1.dto; +import java.util.Collections; +import java.util.Map; import java.util.Objects; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -25,13 +27,15 @@ public class StatementRequest { private String statement; private boolean runAsync; private Long queryTimeout; + private Map confOverlay; public StatementRequest() {} - public StatementRequest(String statement, boolean runAsync, Long queryTimeout) { + public StatementRequest(String statement, boolean runAsync, Long queryTimeout, Map confOverlay) { this.statement = statement; this.runAsync = runAsync; this.queryTimeout = queryTimeout; + this.confOverlay = confOverlay; } public String getStatement() { @@ -58,6 +62,17 @@ public void setQueryTimeout(Long queryTimeout) { this.queryTimeout = queryTimeout; } + public Map getConfOverlay() { + if (confOverlay == null) { + return Collections.emptyMap(); + } + return confOverlay; + } + + public void setConfOverlay(Map confOverlay) { + this.confOverlay = confOverlay; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala index 253c738c485..5bc9d196894 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala @@ -175,7 +175,7 @@ private[v1] class SessionsResource extends ApiRequestContext with Logging { fe.be.executeStatement( sessionHandleStr, request.getStatement, - Map.empty, + request.getConfOverlay.asScala, request.isRunAsync, request.getQueryTimeout) } catch { From f840cb4d9d635352b7a09ee03302a0a50d902d94 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Thu, 30 Mar 2023 18:28:54 +0800 Subject: [PATCH 2/5] conf overlay --- .../org/apache/kyuubi/client/api/v1/dto/StatementRequest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java index 27e84de5237..821a4a8abd0 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java @@ -31,7 +31,8 @@ public class StatementRequest { public StatementRequest() {} - public StatementRequest(String statement, boolean runAsync, Long queryTimeout, Map confOverlay) { + public StatementRequest( + String statement, boolean runAsync, Long queryTimeout, Map confOverlay) { this.statement = statement; this.runAsync = runAsync; this.queryTimeout = queryTimeout; From 2fe8a1659a54b95da929ffe45fe04d9f218e8648 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Thu, 30 Mar 2023 18:33:42 +0800 Subject: [PATCH 3/5] save --- docs/deployment/migration-guide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/deployment/migration-guide.md b/docs/deployment/migration-guide.md index 2ebf16ac40e..52cc5aa3ace 100644 --- a/docs/deployment/migration-guide.md +++ b/docs/deployment/migration-guide.md @@ -20,6 +20,7 @@ ## Upgrading from Kyuubi 1.7.0 to 1.7.1 * Since Kyuubi 1.7.1, `protocolVersion` is removed from the request parameters of the REST API `Open(create) a session`. All removed or unknown parameters will be silently ignored and affects nothing. +* Since Kyuubi 1.7.1, `confOverlay` is supported from the request parameters of the REST API `Create an operation with EXECUTE_STATEMENT type`. ## Upgrading from Kyuubi 1.6 to 1.7 From f4f2bc8834ad88b84a1497e08b17edcacf61e3fb Mon Sep 17 00:00:00 2001 From: fwang12 Date: Thu, 30 Mar 2023 18:37:14 +0800 Subject: [PATCH 4/5] doc --- docs/client/rest/rest_api.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/client/rest/rest_api.md b/docs/client/rest/rest_api.md index c622c401e23..6dbb31f960a 100644 --- a/docs/client/rest/rest_api.md +++ b/docs/client/rest/rest_api.md @@ -110,11 +110,12 @@ Create an operation with EXECUTE_STATEMENT type #### Request Body -| Name | Description | Type | -|:-------------|:---------------------------------------------------------------|:--------| -| statement | The SQL statement that you execute | String | -| runAsync | The flag indicates whether the query runs synchronously or not | Boolean | -| queryTimeout | The interval of query time out | Long | +| Name | Description | Type | +|:-------------|:---------------------------------------------------------------|:---------------| +| statement | The SQL statement that you execute | String | +| runAsync | The flag indicates whether the query runs synchronously or not | Boolean | +| queryTimeout | The interval of query time out | Long | +| confOverlay | The conf to overlay only for current operation | Map of key=val | #### Response Body From cdf828f9a65b411e33bbef18921102327b879201 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Thu, 30 Mar 2023 18:41:44 +0800 Subject: [PATCH 5/5] add ut --- docs/deployment/migration-guide.md | 2 +- .../client/api/v1/dto/StatementRequest.java | 4 ++++ .../kyuubi/server/api/v1/SessionsResource.scala | 2 +- .../server/api/v1/SessionsResourceSuite.scala | 16 ++++++++++++++-- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/deployment/migration-guide.md b/docs/deployment/migration-guide.md index 52cc5aa3ace..fc916048c43 100644 --- a/docs/deployment/migration-guide.md +++ b/docs/deployment/migration-guide.md @@ -20,7 +20,7 @@ ## Upgrading from Kyuubi 1.7.0 to 1.7.1 * Since Kyuubi 1.7.1, `protocolVersion` is removed from the request parameters of the REST API `Open(create) a session`. All removed or unknown parameters will be silently ignored and affects nothing. -* Since Kyuubi 1.7.1, `confOverlay` is supported from the request parameters of the REST API `Create an operation with EXECUTE_STATEMENT type`. +* Since Kyuubi 1.7.1, `confOverlay` is supported in the request parameters of the REST API `Create an operation with EXECUTE_STATEMENT type`. ## Upgrading from Kyuubi 1.6 to 1.7 diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java index 821a4a8abd0..f2dc060d5ec 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java @@ -31,6 +31,10 @@ public class StatementRequest { public StatementRequest() {} + public StatementRequest(String statement, boolean runAsync, Long queryTimeout) { + this(statement, runAsync, queryTimeout, Collections.emptyMap()); + } + public StatementRequest( String statement, boolean runAsync, Long queryTimeout, Map confOverlay) { this.statement = statement; diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala index 5bc9d196894..03b606bdbf7 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala @@ -175,7 +175,7 @@ private[v1] class SessionsResource extends ApiRequestContext with Logging { fe.be.executeStatement( sessionHandleStr, request.getStatement, - request.getConfOverlay.asScala, + request.getConfOverlay.asScala.toMap, request.isRunAsync, request.getQueryTimeout) } catch { diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/SessionsResourceSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/SessionsResourceSuite.scala index 7e5eb5247a6..25950fc3824 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/SessionsResourceSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/SessionsResourceSuite.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.server.api.v1 import java.nio.charset.StandardCharsets import java.util -import java.util.Base64 +import java.util.{Base64, Collections} import javax.ws.rs.client.Entity import javax.ws.rs.core.{GenericType, MediaType, Response} @@ -192,7 +192,7 @@ class SessionsResourceSuite extends KyuubiFunSuite with RestFrontendTestHelper { val pathPrefix = s"api/v1/sessions/$sessionHandle" - val statementReq = new StatementRequest("show tables", true, 3000) + var statementReq = new StatementRequest("show tables", true, 3000) response = webTarget .path(s"$pathPrefix/operations/statement").request(MediaType.APPLICATION_JSON_TYPE) .post(Entity.entity(statementReq, MediaType.APPLICATION_JSON_TYPE)) @@ -200,6 +200,18 @@ class SessionsResourceSuite extends KyuubiFunSuite with RestFrontendTestHelper { var operationHandle = response.readEntity(classOf[OperationHandle]) assert(operationHandle !== null) + statementReq = new StatementRequest( + "spark.sql(\"show tables\")", + true, + 3000, + Collections.singletonMap(KyuubiConf.OPERATION_LANGUAGE.key, "SCALA")) + response = webTarget + .path(s"$pathPrefix/operations/statement").request(MediaType.APPLICATION_JSON_TYPE) + .post(Entity.entity(statementReq, MediaType.APPLICATION_JSON_TYPE)) + assert(200 == response.getStatus) + operationHandle = response.readEntity(classOf[OperationHandle]) + assert(operationHandle !== null) + response = webTarget.path(s"$pathPrefix/operations/typeInfo").request() .post(Entity.entity(null, MediaType.APPLICATION_JSON_TYPE)) assert(200 == response.getStatus)