From b6e37eaeea49b6fed8d4eb738d395f8193895789 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 19 Nov 2024 16:52:37 +0800 Subject: [PATCH] fixup --- .../sql/execution/ColumnarCachedBatchSerializer.scala | 4 ++-- .../columnar/heuristic/ExpandFallbackPolicy.scala | 11 +++++------ .../spark/sql/execution/GlutenExplainUtils.scala | 6 ++---- .../apache/spark/sql/execution/GlutenImplicits.scala | 5 +++-- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index d6e7aa5b2244..6fd346f64cbd 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -46,7 +46,7 @@ case class CachedColumnarBatch( bytes: Array[Byte]) extends CachedBatch {} -// spotless:off +// format: off /** * Feature: * 1. This serializer supports column pruning @@ -75,7 +75,7 @@ case class CachedColumnarBatch( * - Deserializer DefaultCachedBatch -> InternalRow (unsupport ColumnarToRow) * -> Convert DefaultCachedBatch to InternalRow using vanilla Spark serializer */ -// spotless:on +// format: on class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHelper with Logging { private lazy val rowBasedCachedBatchSerializer = new DefaultCachedBatchSerializer diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/heuristic/ExpandFallbackPolicy.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/heuristic/ExpandFallbackPolicy.scala index d6e674bbf291..e1c8b6f41f44 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/heuristic/ExpandFallbackPolicy.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/heuristic/ExpandFallbackPolicy.scala @@ -31,7 +31,8 @@ import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.execution.command.ExecutedCommandExec import org.apache.spark.sql.execution.exchange.Exchange -// spotless:off + +// format: off /** * Note, this rule should only fallback to row-based plan if there is no harm. * The follow case should be handled carefully @@ -64,7 +65,7 @@ import org.apache.spark.sql.execution.exchange.Exchange * @param isAdaptiveContext If is inside AQE * @param originalPlan The vanilla SparkPlan without apply gluten transform rules */ -// spotless:on +// format: on case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkPlan) extends Rule[SparkPlan] { import ExpandFallbackPolicy._ @@ -106,14 +107,13 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP transitionCost } + // format: off /** * When making a stage fall back, it's possible that we need a ColumnarToRow to adapt to last * stage's columnar output. So we need to evaluate the cost, i.e., the number of required * ColumnarToRow between entirely fallback stage and last stage(s). Thus, we can avoid possible * performance degradation caused by fallback policy. * - * spotless:off - * * Spark plan before applying fallback policy: * * ColumnarExchange @@ -136,9 +136,8 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP * Project * * So by considering the cost, the fallback policy will not be applied. - * - * spotless:on */ + // format: on private def countStageFallbackTransitionCost(plan: SparkPlan): Int = { var stageFallbackTransitionCost = 0 diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala index fa697789c8cf..d6167c931cd6 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala @@ -147,15 +147,13 @@ object GlutenExplainUtils extends AdaptiveSparkPlanHelper { } } - // spotless:off - // scalastyle:off + // format: off /** * Given a input physical plan, performs the following tasks. * 1. Generates the explain output for the input plan excluding the subquery plans. 2. Generates * the explain output for each subquery referenced in the plan. */ - // scalastyle:on - // spotless:on + // format: on def processPlan[T <: QueryPlan[T]]( plan: T, append: String => Unit, diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala index 4ecc674d4b13..435bf9239e8a 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala @@ -36,7 +36,8 @@ import org.apache.spark.sql.internal.SQLConf import scala.collection.mutable import scala.collection.mutable.ArrayBuffer -// spotless:off + +// format: off /** * A helper class to get the Gluten fallback summary from a Spark [[Dataset]]. * @@ -52,7 +53,7 @@ import scala.collection.mutable.ArrayBuffer * df.fallbackSummary * }}} */ -// spotless:on +// format: on object GlutenImplicits { case class FallbackSummary(