[KYUUBI #5894] Separate closed and online sessions/statements in the …

…SparkUI's engine tab

Separate closed and online sessions[statements] in the SparkUI's engine tab

# 🔍 Description
## Issue References 🔗

This pull request fixes #5894

## Describe Your Solution 🔧
Separate closed and online sessions[statements] in the SparkUI's engine tab
![image](https://github.com/apache/kyuubi/assets/25627922/b39f9215-f629-4c5d-ac3d-9d7c4a8ebff0)

## Types of changes 🔖

- [ ] Bugfix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)

## Test Plan 🧪

#### Behavior Without This Pull Request ⚰️

#### Behavior With This Pull Request 🎉

#### Related Unit Tests

unit test will add soon.

---

# Checklist 📝

- [ ] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html)

**Be nice. Be informative.**

Closes #5946 from Kwafoor/kyuubi_5894.

Closes #5894

ade3fc1 [wangjunbo] fix
d71a61a [wangjunbo] rename span id
0b600f5 [wangjunbo] rename span id
f56ca7e [wangjunbo] fix code
79b8e33 [wangjunbo] delete test code
cd3ecac [wangjunbo] fix CI test, fix html document id duplicate
3a0e370 [wangjunbo] [KYUUBI #5894] fix scalastyle check
83c7b15 [wangjunbo] [KYUUBI #5894] Separate closed and online sessions[statements] in the SparkUI's engine tab

Authored-by: wangjunbo <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala

@@ -23,6 +23,7 @@ import java.util.Date
 import javax.servlet.http.HttpServletRequest
 
 import scala.collection.JavaConverters.mapAsScalaMapConverter
+import scala.collection.mutable
 import scala.xml.{Node, Unparsed}
 
 import org.apache.commons.text.StringEscapeUtils
@@ -36,18 +37,46 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") {
   private val store = parent.store
 
   override def render(request: HttpServletRequest): Seq[Node] = {
+    val onlineSession = new mutable.ArrayBuffer[SessionEvent]()
+    val closedSession = new mutable.ArrayBuffer[SessionEvent]()
+
+    val runningSqlStat = new mutable.ArrayBuffer[SparkOperationEvent]()
+    val completedSqlStat = new mutable.ArrayBuffer[SparkOperationEvent]()
+    val failedSqlStat = new mutable.ArrayBuffer[SparkOperationEvent]()
+
+    store.getSessionList.foreach { s =>
+      if (s.endTime <= 0L) {
+        onlineSession += s
+      } else {
+        closedSession += s
+      }
+    }
+
+    store.getStatementList.foreach { op =>
+      if (op.completeTime <= 0L) {
+        runningSqlStat += op
+      } else if (op.exception.isDefined) {
+        failedSqlStat += op
+      } else {
+        completedSqlStat += op
+      }
+    }
+
     val content =
       generateBasicStats() ++
         <br/> ++
         stop(request) ++
         <br/> ++
         <h4>
-        {store.getSessionCount} session(s) are online,
-        running {store.getStatementCount}
-        operations
+        {onlineSession.size} session(s) are online,
+        running {runningSqlStat.size} operation(s)
       </h4> ++
-        generateSessionStatsTable(request) ++
-        generateStatementStatsTable(request)
+        generateSessionStatsTable(request, onlineSession.toSeq, closedSession.toSeq) ++
+        generateStatementStatsTable(
+          request,
+          runningSqlStat.toSeq,
+          completedSqlStat.toSeq,
+          failedSqlStat.toSeq)
     UIUtils.headerSparkPage(request, parent.name, content, parent)
   }
 
@@ -129,102 +158,199 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") {
     }
   }
 
-  /** Generate stats of statements for the engine */
-  private def generateStatementStatsTable(request: HttpServletRequest): Seq[Node] = {
-
-    val numStatement = store.getStatementList.size
-
-    val table =
-      if (numStatement > 0) {
+  /** Generate stats of running statements for the engine */
+  private def generateStatementStatsTable(
+      request: HttpServletRequest,
+      running: Seq[SparkOperationEvent],
+      completed: Seq[SparkOperationEvent],
+      failed: Seq[SparkOperationEvent]): Seq[Node] = {
+
+    val content = mutable.ListBuffer[Node]()
+    if (running.nonEmpty) {
+      val sqlTableTag = "running-sqlstat"
+      val table =
+        statementStatsTable(request, sqlTableTag, parent, running)
+      content ++=
+        <span id="running-sqlstat" class="collapse-aggregated-runningSqlstat collapse-table"
+              onClick="collapseTable('collapse-aggregated-runningSqlstat',
+              'aggregated-runningSqlstat')">
+          <h4>
+            <span class="collapse-table-arrow arrow-open"></span>
+            <a>Running Statement Statistics (
+              {running.size}
+              )</a>
+          </h4>
+        </span> ++
+          <div class="aggregated-runningSqlstat collapsible-table">
+            {table}
+          </div>
+    }
 
-        val sqlTableTag = "sqlstat"
+    if (completed.nonEmpty) {
+      val table = {
+        val sqlTableTag = "completed-sqlstat"
+        statementStatsTable(
+          request,
+          sqlTableTag,
+          parent,
+          completed)
+      }
 
-        val sqlTablePage =
-          Option(request.getParameter(s"$sqlTableTag.page")).map(_.toInt).getOrElse(1)
+      content ++=
+        <span id="completed-sqlstat" class="collapse-aggregated-completedSqlstat collapse-table"
+              onClick="collapseTable('collapse-aggregated-completedSqlstat',
+              'aggregated-completedSqlstat')">
+          <h4>
+            <span class="collapse-table-arrow arrow-open"></span>
+            <a>Completed Statement Statistics (
+              {completed.size}
+              )</a>
+          </h4>
+        </span> ++
+          <div class="aggregated-completedSqlstat collapsible-table">
+          {table}
+        </div>
+    }
 
-        try {
-          Some(new StatementStatsPagedTable(
-            request,
-            parent,
-            store.getStatementList,
-            "kyuubi",
-            UIUtils.prependBaseUri(request, parent.basePath),
-            sqlTableTag).table(sqlTablePage))
-        } catch {
-          case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
-            Some(<div class="alert alert-error">
-            <p>Error while rendering job table:</p>
-            <pre>
-              {Utils.stringifyException(e)}
-            </pre>
-          </div>)
-        }
-      } else {
-        None
+    if (failed.nonEmpty) {
+      val table = {
+        val sqlTableTag = "failed-sqlstat"
+        statementStatsTable(
+          request,
+          sqlTableTag,
+          parent,
+          failed)
       }
-    val content =
-      <span id="sqlstat" class="collapse-aggregated-sqlstat collapse-table"
-            onClick="collapseTable('collapse-aggregated-sqlstat',
-                'aggregated-sqlstat')">
-        <h4>
-          <span class="collapse-table-arrow arrow-open"></span>
-          <a>Statement Statistics ({numStatement})</a>
-        </h4>
-      </span> ++
-        <div class="aggregated-sqlstat collapsible-table">
-          {table.getOrElse("No statistics have been generated yet.")}
+
+      content ++=
+        <span id="failed-sqlstat" class="collapse-aggregated-failedSqlstat collapse-table"
+              onClick="collapseTable('collapse-aggregated-failedSqlstat',
+              'aggregated-failedSqlstat')">
+          <h4>
+            <span class="collapse-table-arrow arrow-open"></span>
+            <a>Failed Statement Statistics (
+              {failed.size}
+              )</a>
+          </h4>
+        </span> ++
+          <div class="aggregated-failedSqlstat collapsible-table">
+          {table}
         </div>
+    }
     content
   }
 
-  /** Generate stats of sessions for the engine */
-  private def generateSessionStatsTable(request: HttpServletRequest): Seq[Node] = {
-    val numSessions = store.getSessionList.size
-    val table =
-      if (numSessions > 0) {
+  private def statementStatsTable(
+      request: HttpServletRequest,
+      sqlTableTag: String,
+      parent: EngineTab,
+      data: Seq[SparkOperationEvent]): Seq[Node] = {
+
+    val sqlTablePage =
+      Option(request.getParameter(s"$sqlTableTag.page")).map(_.toInt).getOrElse(1)
 
-        val sessionTableTag = "sessionstat"
+    try {
+      new StatementStatsPagedTable(
+        request,
+        parent,
+        data,
+        "kyuubi",
+        UIUtils.prependBaseUri(request, parent.basePath),
+        s"${sqlTableTag}-table").table(sqlTablePage)
+    } catch {
+      case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
+        <div class="alert alert-error">
+              <p>Error while rendering job table:</p>
+              <pre>
+                {Utils.stringifyException(e)}
+              </pre>
+            </div>
+    }
+  }
 
-        val sessionTablePage =
-          Option(request.getParameter(s"$sessionTableTag.page")).map(_.toInt).getOrElse(1)
+  /** Generate stats of online sessions for the engine */
+  private def generateSessionStatsTable(
+      request: HttpServletRequest,
+      online: Seq[SessionEvent],
+      closed: Seq[SessionEvent]): Seq[Node] = {
+    val content = mutable.ListBuffer[Node]()
+    if (online.nonEmpty) {
+      val sessionTableTag = "online-sessionstat"
+      val table = sessionTable(
+        request,
+        sessionTableTag,
+        parent,
+        online)
+      content ++=
+        <span id="online-sessionstat" class="collapse-aggregated-onlineSessionstat collapse-table"
+              onClick="collapseTable('collapse-aggregated-onlineSessionstat',
+              'aggregated-onlineSessionstat')">
+          <h4>
+            <span class="collapse-table-arrow arrow-open"></span>
+            <a>Online Session Statistics (
+              {online.size}
+              )</a>
+          </h4>
+        </span> ++
+          <div class="aggregated-onlineSessionstat collapsible-table">
+            {table}
+          </div>
+    }
 
-        try {
-          Some(new SessionStatsPagedTable(
-            request,
-            parent,
-            store.getSessionList,
-            "kyuubi",
-            UIUtils.prependBaseUri(request, parent.basePath),
-            sessionTableTag).table(sessionTablePage))
-        } catch {
-          case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
-            Some(<div class="alert alert-error">
-            <p>Error while rendering job table:</p>
-            <pre>
-              {Utils.stringifyException(e)}
-            </pre>
-          </div>)
-        }
-      } else {
-        None
+    if (closed.nonEmpty) {
+      val table = {
+        val sessionTableTag = "closed-sessionstat"
+        sessionTable(
+          request,
+          sessionTableTag,
+          parent,
+          closed)
       }
 
-    val content =
-      <span id="sessionstat" class="collapse-aggregated-sessionstat collapse-table"
-            onClick="collapseTable('collapse-aggregated-sessionstat',
-                'aggregated-sessionstat')">
-        <h4>
-          <span class="collapse-table-arrow arrow-open"></span>
-          <a>Session Statistics ({numSessions})</a>
-        </h4>
-      </span> ++
-        <div class="aggregated-sessionstat collapsible-table">
-          {table.getOrElse("No statistics have been generated yet.")}
+      content ++=
+        <span id="closed-sessionstat" class="collapse-aggregated-closedSessionstat collapse-table"
+              onClick="collapseTable('collapse-aggregated-closedSessionstat',
+              'aggregated-closedSessionstat')">
+          <h4>
+            <span class="collapse-table-arrow arrow-open"></span>
+            <a>Closed Session Statistics (
+              {closed.size}
+              )</a>
+          </h4>
+        </span> ++
+          <div class="aggregated-closedSessionstat collapsible-table">
+          {table}
         </div>
-
+    }
     content
   }
 
+  private def sessionTable(
+      request: HttpServletRequest,
+      sessionTage: String,
+      parent: EngineTab,
+      data: Seq[SessionEvent]): Seq[Node] = {
+    val sessionPage =
+      Option(request.getParameter(s"$sessionTage.page")).map(_.toInt).getOrElse(1)
+    try {
+      new SessionStatsPagedTable(
+        request,
+        parent,
+        data,
+        "kyuubi",
+        UIUtils.prependBaseUri(request, parent.basePath),
+        s"${sessionTage}-table").table(sessionPage)
+    } catch {
+      case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
+        <div class="alert alert-error">
+          <p>Error while rendering job table:</p>
+          <pre>
+            {Utils.stringifyException(e)}
+          </pre>
+        </div>
+    }
+  }
+
   private class SessionStatsPagedTable(
       request: HttpServletRequest,
       parent: EngineTab,