Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[KYUUBI #1509] Make KYUUBI_WORK_DIR_ROOT as the default root path. #1519

Closed
wants to merge 1 commit into from
Closed

[KYUUBI #1509] Make KYUUBI_WORK_DIR_ROOT as the default root path. #1519

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions ...engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package org.apache.kyuubi.engine.spark.session
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.spark.sql.SparkSession

import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.{KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.engine.ShareLevel
Expand All @@ -42,7 +42,8 @@ class SparkSQLSessionManager private (name: String, spark: SparkSession)
def this(spark: SparkSession) = this(classOf[SparkSQLSessionManager].getSimpleName, spark)

override def initialize(conf: KyuubiConf): Unit = {
_operationLogRoot = Some(conf.get(ENGINE_OPERATION_LOG_DIR_ROOT))
val absPath = Utils.getAbsolutePathFromWork(conf.get(ENGINE_OPERATION_LOG_DIR_ROOT))
_operationLogRoot = Some(absPath.toAbsolutePath.toString)
super.initialize(conf)
}

Expand Down
13 changes: 13 additions & 0 deletions kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,19 @@ object Utils extends Logging {
error)
}

def getAbsolutePathFromWork(pathStr: String, env: Map[String, String] = sys.env): Path = {
val path = Paths.get(pathStr)
if (path.isAbsolute) {
path
} else {
val workDir = env.get("KYUUBI_WORK_DIR_ROOT") match {
case Some(dir) => dir
case _ => System.getProperty("user.dir")
}
Paths.get(workDir, pathStr)
}
}

/**
* Delete a directory recursively.
*/
Expand Down
12 changes: 11 additions & 1 deletion kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package org.apache.kyuubi

import java.io.{File, IOException}
import java.net.InetAddress
import java.nio.file.Files
import java.nio.file.{Files, Paths}
import java.security.PrivilegedExceptionAction
import java.util.Properties

Expand Down Expand Up @@ -130,4 +130,14 @@ class UtilsSuite extends KyuubiFunSuite {
assert(Utils.findLocalInetAddress !== InetAddress.getLocalHost)
}
}

test("getAbsolutePathFromWork") {
val workDir = System.getenv("KYUUBI_WORK_DIR_ROOT")
val path1 = "path1"
assert(Utils.getAbsolutePathFromWork(path1).toAbsolutePath.toString ===
Paths.get(workDir, path1).toAbsolutePath.toString)

val path2 = "/tmp/path2"
assert(Utils.getAbsolutePathFromWork(path2).toString === path2)
}
}
3 changes: 2 additions & 1 deletion kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper

import org.apache.kyuubi.Utils
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.metrics.MetricsConf._
import org.apache.kyuubi.service.AbstractService
Expand All @@ -43,7 +44,7 @@ class JsonReporterService(registry: MetricRegistry)
private var reportPath: Path = _

override def initialize(conf: KyuubiConf): Unit = synchronized {
reportDir = Paths.get(conf.get(METRICS_JSON_LOCATION)).toAbsolutePath
reportDir = Utils.getAbsolutePathFromWork(conf.get(METRICS_JSON_LOCATION))
Files.createDirectories(reportDir)
reportPath = Paths.get(reportDir.toString, "report.json").toAbsolutePath
super.initialize(conf)
Expand Down
5 changes: 3 additions & 2 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package org.apache.kyuubi.session
import com.codahale.metrics.MetricRegistry
import org.apache.hive.service.rpc.thrift.TProtocolVersion

import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.{KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.credentials.HadoopCredentialsManager
Expand All @@ -37,7 +37,8 @@ class KyuubiSessionManager private (name: String) extends SessionManager(name) {

override def initialize(conf: KyuubiConf): Unit = {
addService(credentialsManager)
_operationLogRoot = Some(conf.get(SERVER_OPERATION_LOG_DIR_ROOT))
val absPath = Utils.getAbsolutePathFromWork(conf.get(SERVER_OPERATION_LOG_DIR_ROOT))
_operationLogRoot = Some(absPath.toAbsolutePath.toString)
super.initialize(conf)
}

Expand Down