diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala index f8613052773..7989f1754f8 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala @@ -327,16 +327,13 @@ object Utils extends Logging { } def redactCommandLineArgs(conf: KyuubiConf, commands: Iterable[String]): Iterable[String] = { - conf.get(SERVER_SECRET_REDACTION_PATTERN) match { - case Some(redactionPattern) => - commands.map { - case PATTERN_FOR_KEY_VALUE_ARG(key, value) => - val (_, newValue) = redact(redactionPattern, Seq((key, value))).head - genKeyValuePair(key, newValue) - case cmd => - cmd - } - case _ => commands + val redactionPattern = conf.get(SERVER_SECRET_REDACTION_PATTERN) + commands.map { + case PATTERN_FOR_KEY_VALUE_ARG(key, value) => + val (_, newValue) = redact(redactionPattern, Seq((key, value))).head + genKeyValuePair(key, newValue) + case cmd => + cmd } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 4e97e71659d..bd4bf35d804 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3572,15 +3572,17 @@ object KyuubiConf { .booleanConf .createWithDefault(false) - val SERVER_SECRET_REDACTION_PATTERN: OptionalConfigEntry[Regex] = + val SERVER_SECRET_REDACTION_PATTERN: ConfigEntry[Regex] = buildConf("kyuubi.server.redaction.regex") .audience(SERVER) .immutable .doc("Regex to decide which Kyuubi contain sensitive information. When this regex matches " + - "a property key or value, the value is redacted from the various logs.") + "a property key or value, the value is redacted from the various logs. Defaults to " + + "the same pattern Spark uses for spark.redaction.regex, so common secret-bearing keys " + + "are redacted out of the box.") .version("1.6.0") .regexConf - .createOptional + .createWithDefault("(?i)secret|password|token|access[.]key".r) val SERVER_CONF_RETRIEVE_MODE: ConfigEntry[String] = buildConf("kyuubi.server.conf.retrieveMode") @@ -3589,7 +3591,8 @@ object KyuubiConf { .doc("Controls how session configurations are returned in REST API responses. " + "Supported values: " + "") diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala index 3cfb20ae04b..84b9235464c 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala @@ -42,7 +42,7 @@ object ApiUtils extends Logging { case ConfRetrieveMode.ORIGINAL => rawConf.asJava case ConfRetrieveMode.REDACTED => val pattern = session.sessionManager.getConf.get(SERVER_SECRET_REDACTION_PATTERN) - Utils.redact(pattern, rawConf.toSeq).toMap.asJava + Utils.redact(Some(pattern), rawConf.toSeq).toMap.asJava } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/SessionsResourceSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/SessionsResourceSuite.scala index add41df5b8f..b1b234cccdb 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/SessionsResourceSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/SessionsResourceSuite.scala @@ -439,6 +439,29 @@ class SessionsResourceSuite extends KyuubiFunSuite with RestFrontendTestHelper { } } + test("get /sessions redacts spark confs by default with no explicit redaction regex") { + withNoExplicitRedactionPattern { + val sensitiveKey = "spark.password" + val sensitiveValue = "shouldNeverLeak" + val requestObj = new SessionOpenRequest(Map(sensitiveKey -> sensitiveValue).asJava) + + val r = webTarget.path("api/v1/sessions") + .request(MediaType.APPLICATION_JSON_TYPE) + .post(Entity.entity(requestObj, MediaType.APPLICATION_JSON_TYPE)) + assert(200 == r.getStatus) + val sessionHandle = r.readEntity(classOf[SessionHandle]).getIdentifier + + val r2 = webTarget.path("api/v1/sessions").request().get() + assert(200 == r2.getStatus) + val sessions = r2.readEntity(new GenericType[Seq[SessionData]]() {}) + val sessionConf = sessions.find(_.getIdentifier == sessionHandle.toString).get.getConf + assert(sessionConf.get(sensitiveKey) != sensitiveValue) + assert(sessionConf.get(sensitiveKey) == "*********(redacted)") + + webTarget.path(s"api/v1/sessions/$sessionHandle").request().delete() + } + } + test("get /sessions returns raw conf when mode is ORIGINAL") { withSessionConfDisplayMode("ORIGINAL") { val sensitiveKey = "spark.password" @@ -465,4 +488,10 @@ class SessionsResourceSuite extends KyuubiFunSuite with RestFrontendTestHelper { try f finally conf.set(KyuubiConf.SERVER_CONF_RETRIEVE_MODE, "REDACTED") } + + private def withNoExplicitRedactionPattern(f: => Unit): Unit = { + conf.unset(KyuubiConf.SERVER_SECRET_REDACTION_PATTERN) + try f + finally conf.set(KyuubiConf.SERVER_SECRET_REDACTION_PATTERN, "(?i)password".r) + } }