Skip to content

Commit

Permalink
[KYUUBI #6032] Expose KyuubiOperation exec time metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
wangjunbo committed Feb 2, 2024
1 parent 4bd259a commit 8fb7e10
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 17 deletions.
2 changes: 1 addition & 1 deletion docs/monitor/metrics.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ These metrics include:
| `kyuubi.operation.opened` | `${operationType}` | counter | 1.5.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> current opened count for the operation `${operationType}`</div> |
| `kyuubi.operation.failed` | `${operationType}`<br/>`.${errorType}` | counter | 1.5.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> cumulative failed count for the operation `${operationType}` with a particular `${errorType}`, e.g. `execute_statement.AnalysisException`</div> |
| `kyuubi.operation.state` | `${operationState}` | meter | 1.5.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> kyuubi operation state rate </div> |
| `kyuubi.operation.exec_time` | `${operationType}` | histogram | 1.7.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> execution time histogram for the operation `${operationType}`, now only `ExecuteStatement` is enabled. </div> |
| `kyuubi.operation.exec_time` | `${operationType}` | histogram | 1.7.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> execution time histogram for the operation `${operationType}`, now `KyuubiOperation` is enabled. </div> |
| `kyuubi.engine.total` | | counter | 1.2.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> cumulative created engines</div> |
| `kyuubi.engine.timeout` | | counter | 1.2.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> cumulative timeout engines</div> |
| `kyuubi.engine.failed` | `${user}` | counter | 1.2.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> cumulative explicitly failed engine count for a `${user}`</div> |
Expand Down
9 changes: 0 additions & 9 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/operation/ExecuteStatement.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,9 @@ package org.apache.kyuubi.operation

import scala.collection.JavaConverters._

import com.codahale.metrics.MetricRegistry

import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.OPERATION_QUERY_TIMEOUT_MONITOR_ENABLED
import org.apache.kyuubi.metrics.{MetricsConstants, MetricsSystem}
import org.apache.kyuubi.operation.FetchOrientation.FETCH_NEXT
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.session.Session
Expand Down Expand Up @@ -139,12 +136,6 @@ class ExecuteStatement(
}
sendCredentialsIfNeeded()
}
MetricsSystem.tracing { ms =>
val execTime = System.currentTimeMillis() - startTime
ms.updateHistogram(
MetricRegistry.name(MetricsConstants.OPERATION_EXEC_TIME, opType),
execTime)
}
// see if anymore log could be fetched
fetchQueryLog()
} catch onError()
Expand Down
7 changes: 5 additions & 2 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import org.apache.commons.lang3.StringUtils
import org.apache.kyuubi.{KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_OPERATION_HANDLE_KEY
import org.apache.kyuubi.events.{EventBus, KyuubiOperationEvent}
import org.apache.kyuubi.metrics.MetricsConstants.{OPERATION_FAIL, OPERATION_OPEN, OPERATION_STATE, OPERATION_TOTAL}
import org.apache.kyuubi.metrics.MetricsConstants.{OPERATION_EXEC_TIME, OPERATION_FAIL, OPERATION_OPEN, OPERATION_STATE, OPERATION_TOTAL}
import org.apache.kyuubi.metrics.MetricsSystem
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.operation.OperationState.OperationState
Expand Down Expand Up @@ -210,8 +210,11 @@ abstract class KyuubiOperation(session: Session) extends AbstractOperation(sessi

override def setState(newState: OperationState): Unit = {
MetricsSystem.tracing { ms =>
if (!OperationState.isTerminal(state)) {
if (!isTerminalState(state)) {
ms.markMeter(MetricRegistry.name(OPERATION_STATE, opType, state.toString.toLowerCase), -1)
} else {
val execTime = System.currentTimeMillis() - startTime
ms.updateHistogram(MetricRegistry.name(OPERATION_EXEC_TIME, opType), execTime)
}
ms.markMeter(MetricRegistry.name(OPERATION_STATE, opType, newState.toString.toLowerCase))
ms.markMeter(MetricRegistry.name(OPERATION_STATE, newState.toString.toLowerCase))
Expand Down
16 changes: 11 additions & 5 deletions kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,14 +325,20 @@ class KyuubiOperationPerUserSuite
}
}

test("trace ExecuteStatement exec time histogram") {
test("trace KyuubiOperation exec time histogram") {
withJdbcStatement() { statement =>
statement.executeQuery("select engine_name()")
statement.executeQuery("KYUUBI DESC ENGINE")
}
val metric =
s"${MetricsConstants.OPERATION_EXEC_TIME}.${classOf[ExecuteStatement].getSimpleName}"
val snapshot = MetricsSystem.histogramSnapshot(metric).get
assert(snapshot.getMax > 0 && snapshot.getMedian > 0)
Seq(
classOf[LaunchEngine].getSimpleName,
classOf[ExecutedCommandExec].getSimpleName,
classOf[ExecuteStatement].getSimpleName)
.map(className => s"${MetricsConstants.OPERATION_EXEC_TIME}.$className")
.foreach { metric =>
val snapshot = MetricsSystem.histogramSnapshot(metric).get
assert(snapshot.getMax > 0 && snapshot.getMedian > 0)
}
}

test("align the server/engine session/executeStatement handle for Spark engine") {
Expand Down

0 comments on commit 8fb7e10

Please sign in to comment.