From 992aefd7fe7c572bd3c248019e4cde48b28c8df9 Mon Sep 17 00:00:00 2001 From: Fei Wang Date: Sun, 4 Feb 2024 10:03:48 -0800 Subject: [PATCH] async stop --- .../org/apache/kyuubi/engine/spark/SparkSQLEngine.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala index 1339c7193a1..062bf0567bf 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala @@ -175,9 +175,12 @@ case class SparkSQLEngine(spark: SparkSession) extends Serverable("SparkSQLEngin val elapsedTime = System.currentTimeMillis() - getStartTime if (!shutdown.get && elapsedTime > maxLifetime) { if (deregistered.compareAndSet(false, true)) { - info(s"Spark engine has been running for more than $maxLifetime ms," + - s" deregistering from engine discovery space.") - frontendServices.flatMap(_.discoveryService).foreach(_.stop()) + ThreadUtils.runInNewThread("engine-de-register") { + // for ETCD, the de-registering process might be blocked, so deregister it async + info(s"Spark engine has been running for more than $maxLifetime ms," + + s" deregistering from engine discovery space.") + frontendServices.flatMap(_.discoveryService).foreach(_.stop()) + } } if (backendService.sessionManager.getOpenSessionCount <= 0) {