Skip to content

Commit

Permalink
sum time cost of all types and comparing with all types
Browse files Browse the repository at this point in the history
  • Loading branch information
bowenliang123 committed Dec 3, 2023
1 parent 689e773 commit 1870095
Showing 1 changed file with 20 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -274,14 +274,14 @@ class RowSetSuite extends KyuubiFunSuite {
}

test("to row set benchmark") {
val rowCount = 10000
val rowCount = 1000
val allRows = (0 until rowCount).map(genRow)

def benchmarkToTRowSet(
clue: String,
rows: Seq[Row],
schema: StructType,
protocolVersion: TProtocolVersion): Unit = {
protocolVersion: TProtocolVersion): BigDecimal = {
val sw = StopWatch.createStarted()
RowSet.toTRowSet(rows, schema, protocolVersion)
sw.stop()
Expand All @@ -296,9 +296,13 @@ class RowSetSuite extends KyuubiFunSuite {
s"$msTimeCost ms",
s"$rowsPerMilliSecond rows/ms")
// scalastyle:on
msTimeCost
}

def singleColumn(field: StructField, index: Int, protocolVersion: TProtocolVersion): Unit = {
def singleColumn(
field: StructField,
index: Int,
protocolVersion: TProtocolVersion): BigDecimal = {
benchmarkToTRowSet(
field.getComment().getOrElse(field.dataType.typeName),
allRows.map(row => Row(row.get(index))).asInstanceOf[Seq[Row]],
Expand All @@ -325,13 +329,22 @@ class RowSetSuite extends KyuubiFunSuite {
"Rows/ms")
// scalastyle:on

benchmarkToTRowSet("with all types", allRows, schema, protocolVersion)
schemaStructFields.zipWithIndex.foreach { case (field, index) =>
val totalMsCost = schemaStructFields.zipWithIndex.map { case (field, index) =>
singleColumn(field, index, protocolVersion)
}

}.sum
val totalRowsPerMilliSecond: BigDecimal = (BigDecimal(rowCount) / totalMsCost)
.setScale(3, RoundingMode.HALF_UP)
// scalastyle:off
println()
printf(
"%20s %20s %20s\n",
"sum(all types)",
s"$totalMsCost ms",
s"$totalRowsPerMilliSecond rows/ms")
benchmarkToTRowSet("with all types", allRows, schema, protocolVersion)

println()
println()
// scalastyle:on
}
}
Expand Down

0 comments on commit 1870095

Please sign in to comment.