From 1870095d5510f4e1b782b07654d7492187ce3660 Mon Sep 17 00:00:00 2001 From: liangbowen Date: Mon, 4 Dec 2023 00:10:38 +0800 Subject: [PATCH] sum time cost of all types and comparing with all types --- .../engine/spark/schema/RowSetSuite.scala | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala index cc716ffd25a..eb0228e8577 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala @@ -274,14 +274,14 @@ class RowSetSuite extends KyuubiFunSuite { } test("to row set benchmark") { - val rowCount = 10000 + val rowCount = 1000 val allRows = (0 until rowCount).map(genRow) def benchmarkToTRowSet( clue: String, rows: Seq[Row], schema: StructType, - protocolVersion: TProtocolVersion): Unit = { + protocolVersion: TProtocolVersion): BigDecimal = { val sw = StopWatch.createStarted() RowSet.toTRowSet(rows, schema, protocolVersion) sw.stop() @@ -296,9 +296,13 @@ class RowSetSuite extends KyuubiFunSuite { s"$msTimeCost ms", s"$rowsPerMilliSecond rows/ms") // scalastyle:on + msTimeCost } - def singleColumn(field: StructField, index: Int, protocolVersion: TProtocolVersion): Unit = { + def singleColumn( + field: StructField, + index: Int, + protocolVersion: TProtocolVersion): BigDecimal = { benchmarkToTRowSet( field.getComment().getOrElse(field.dataType.typeName), allRows.map(row => Row(row.get(index))).asInstanceOf[Seq[Row]], @@ -325,13 +329,22 @@ class RowSetSuite extends KyuubiFunSuite { "Rows/ms") // scalastyle:on - benchmarkToTRowSet("with all types", allRows, schema, protocolVersion) - schemaStructFields.zipWithIndex.foreach { case (field, index) => + val totalMsCost = schemaStructFields.zipWithIndex.map { case (field, index) => singleColumn(field, index, protocolVersion) - } - + }.sum + val totalRowsPerMilliSecond: BigDecimal = (BigDecimal(rowCount) / totalMsCost) + .setScale(3, RoundingMode.HALF_UP) // scalastyle:off println() + printf( + "%20s %20s %20s\n", + "sum(all types)", + s"$totalMsCost ms", + s"$totalRowsPerMilliSecond rows/ms") + benchmarkToTRowSet("with all types", allRows, schema, protocolVersion) + + println() + println() // scalastyle:on } }