forked from apache/spark
-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SPARK-44214][CORE] Support Spark Driver Live Log UI
### What changes were proposed in this pull request? This PR aims to add `Driver Log` UI in order to provide a better UX like `Standalone Cluster`. ### Why are the changes needed? - Since Apache Spark 3.0.0, Spark History Server supports driver logs via `spark.driver.log.persistToDfs.enabled` and `spark.driver.log.dfsDir`. - For Apache Spark 4.0.0, this PR aims to support `Live Driver Log UI` page in Spark Driver UI (port 4040) via `spark.driver.log.localDir`. It means we can see the live driver log like the following via UI. **SPARK-SHELL** ``` $ bin/spark-shell -c spark.driver.log.localDir=/tmp/logs ```  **K8S** ``` bin/spark-submit \ --master k8s://$K8S_MASTER \ --deploy-mode cluster \ --name log \ -c spark.executor.instances=10 \ -c spark.driver.log.localDir=file://tmp/ \ -c spark.kubernetes.driver.master=$K8S_MASTER \ -c spark.kubernetes.namespace=$NAMESPACE \ ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs with the newly added test case. Closes apache#42542 from dongjoon-hyun/SPARK-44214. Authored-by: Dongjoon Hyun <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
- Loading branch information
1 parent
4646991
commit b137b28
Showing
6 changed files
with
251 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
153 changes: 153 additions & 0 deletions
153
core/src/main/scala/org/apache/spark/ui/DriverLogPage.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.spark.ui | ||
|
|
||
| import javax.servlet.http.HttpServletRequest | ||
|
|
||
| import scala.xml.{Node, Unparsed} | ||
|
|
||
| import org.apache.spark.SparkConf | ||
| import org.apache.spark.internal.Logging | ||
| import org.apache.spark.internal.config.DRIVER_LOG_LOCAL_DIR | ||
| import org.apache.spark.util.Utils | ||
| import org.apache.spark.util.logging.DriverLogger.DRIVER_LOG_FILE | ||
| import org.apache.spark.util.logging.RollingFileAppender | ||
|
|
||
| /** | ||
| * Live Spark Driver Log UI Page. | ||
| * | ||
| * This is similar with Spark worker's LogPage class. | ||
| */ | ||
| private[ui] class DriverLogPage( | ||
| parent: DriverLogTab, | ||
| conf: SparkConf) | ||
| extends WebUIPage("") with Logging { | ||
| require(conf.get(DRIVER_LOG_LOCAL_DIR).nonEmpty, s"Please specify ${DRIVER_LOG_LOCAL_DIR.key}.") | ||
|
|
||
| private val supportedLogTypes = Set(DRIVER_LOG_FILE, "stderr", "stdout") | ||
| private val defaultBytes = 100 * 1024 | ||
| private val logDir = conf.get(DRIVER_LOG_LOCAL_DIR).get | ||
|
|
||
| def render(request: HttpServletRequest): Seq[Node] = { | ||
| val logType = Option(request.getParameter("logType")).getOrElse(DRIVER_LOG_FILE) | ||
| val offset = Option(request.getParameter("offset")).map(_.toLong) | ||
| val byteLength = Option(request.getParameter("byteLength")).map(_.toInt) | ||
| .getOrElse(defaultBytes) | ||
| val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength) | ||
| val curLogLength = endByte - startByte | ||
| val range = | ||
| <span id="log-data"> | ||
| Showing {curLogLength} Bytes: {startByte.toString} - {endByte.toString} of {logLength} | ||
| </span> | ||
|
|
||
| val moreButton = | ||
| <button type="button" onclick={"loadMore()"} class="log-more-btn btn btn-secondary"> | ||
| Load More | ||
| </button> | ||
|
|
||
| val newButton = | ||
| <button type="button" onclick={"loadNew()"} class="log-new-btn btn btn-secondary"> | ||
| Load New | ||
| </button> | ||
|
|
||
| val alert = | ||
| <div class="no-new-alert alert alert-info" style="display: none;"> | ||
| End of Log | ||
| </div> | ||
|
|
||
| val logParams = "/?logType=%s".format(logType) | ||
| val jsOnload = "window.onload = " + | ||
| s"initLogPage('$logParams', $curLogLength, $startByte, $endByte, $logLength, $byteLength);" | ||
|
|
||
| val content = | ||
| <script src={UIUtils.prependBaseUri(request, "/static/utils.js")}></script> ++ | ||
| <div> | ||
| Logs at {logDir} | ||
| {range} | ||
| <div class="log-content" style="height:80vh; overflow:auto; padding:5px;"> | ||
| <div>{moreButton}</div> | ||
| <pre>{logText}</pre> | ||
| {alert} | ||
| <div>{newButton}</div> | ||
| </div> | ||
| <script>{Unparsed(jsOnload)}</script> | ||
| </div> | ||
|
|
||
| UIUtils.headerSparkPage(request, "Logs", content, parent) | ||
| } | ||
|
|
||
| def renderLog(request: HttpServletRequest): String = { | ||
| val logType = Option(request.getParameter("logType")).getOrElse(DRIVER_LOG_FILE) | ||
| val offset = Option(request.getParameter("offset")).map(_.toLong) | ||
| val byteLength = Option(request.getParameter("byteLength")).map(_.toInt) | ||
| .getOrElse(defaultBytes) | ||
|
|
||
| val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength) | ||
| val pre = s"==== Bytes $startByte-$endByte of $logLength of $logDir$logType ====\n" | ||
| pre + logText | ||
| } | ||
|
|
||
| /** Get the part of the log files given the offset and desired length of bytes */ | ||
| private def getLog( | ||
| logDirectory: String, | ||
| logType: String, | ||
| offsetOption: Option[Long], | ||
| byteLength: Int | ||
| ): (String, Long, Long, Long) = { | ||
|
|
||
| if (!supportedLogTypes.contains(logType)) { | ||
| return ("Error: Log type must be one of " + supportedLogTypes.mkString(", "), 0, 0, 0) | ||
| } | ||
|
|
||
| try { | ||
| val files = RollingFileAppender.getSortedRolledOverFiles(logDirectory, logType) | ||
| logDebug(s"Sorted log files of type $logType in $logDirectory:\n${files.mkString("\n")}") | ||
|
|
||
| val fileLengths: Seq[Long] = files.map(Utils.getFileLength(_, conf)) | ||
| val totalLength = fileLengths.sum | ||
| val offset = offsetOption.getOrElse(totalLength - byteLength) | ||
| val startIndex = { | ||
| if (offset < 0) { | ||
| 0L | ||
| } else if (offset > totalLength) { | ||
| totalLength | ||
| } else { | ||
| offset | ||
| } | ||
| } | ||
| val endIndex = math.min(startIndex + byteLength, totalLength) | ||
| logDebug(s"Getting log from $startIndex to $endIndex") | ||
| val logText = Utils.offsetBytes(files, fileLengths, startIndex, endIndex) | ||
| logDebug(s"Got log of length ${logText.length} bytes") | ||
| (logText, startIndex, endIndex, totalLength) | ||
| } catch { | ||
| case e: Exception => | ||
| logError(s"Error getting $logType logs from directory $logDirectory", e) | ||
| ("Error getting logs due to exception: " + e.getMessage, 0, 0, 0) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Live Spark Driver Log UI Tab. | ||
| */ | ||
| private[ui] class DriverLogTab(parent: SparkUI) extends SparkUITab(parent, "logs") { | ||
| private val page = new DriverLogPage(this, parent.conf) | ||
| attachPage(page) | ||
|
|
||
| def getPage: DriverLogPage = page | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
65 changes: 65 additions & 0 deletions
65
core/src/test/scala/org/apache/spark/ui/DriverLogPageSuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.spark.ui | ||
|
|
||
| import java.io.File | ||
| import java.nio.charset.StandardCharsets | ||
| import javax.servlet.http.HttpServletRequest | ||
|
|
||
| import org.apache.commons.io.FileUtils | ||
| import org.mockito.Mockito.{mock, when} | ||
|
|
||
| import org.apache.spark.{SparkConf, SparkFunSuite} | ||
| import org.apache.spark.internal.config.DRIVER_LOG_LOCAL_DIR | ||
|
|
||
|
|
||
| class DriverLogPageSuite extends SparkFunSuite { | ||
|
|
||
| test("DriverLogTab requires driver log location") { | ||
| val sparkUI = mock(classOf[SparkUI]) | ||
| when(sparkUI.conf).thenReturn(new SparkConf(false)) | ||
|
|
||
| val m = intercept[IllegalArgumentException] { | ||
| new DriverLogTab(sparkUI) | ||
| }.getMessage | ||
| assert(m.contains(s"Please specify ${DRIVER_LOG_LOCAL_DIR.key}")) | ||
|
|
||
| } | ||
|
|
||
| test("DriverLogPage requires driver log location") { | ||
| val conf = new SparkConf(false) | ||
| val m = intercept[IllegalArgumentException] { | ||
| new DriverLogPage(null, conf) | ||
| }.getMessage | ||
| assert(m.contains(s"Please specify ${DRIVER_LOG_LOCAL_DIR.key}")) | ||
| } | ||
|
|
||
| test("renderLog reads driver.log file") { | ||
| val conf = new SparkConf(false) | ||
| withTempDir { dir => | ||
| val page = new DriverLogPage(null, conf.set(DRIVER_LOG_LOCAL_DIR, dir.getCanonicalPath)) | ||
| val file = new File(dir, "driver.log") | ||
| FileUtils.writeStringToFile(file, "driver log content", StandardCharsets.UTF_8) | ||
| val request = mock(classOf[HttpServletRequest]) | ||
| val log = page.renderLog(request) | ||
| assert(log.startsWith("==== Bytes 0-18 of 18 of")) | ||
| assert(log.contains("driver.log")) | ||
| assert(log.contains("driver log content")) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters