Skip to content

Commit

Permalink
[GLUTEN-7751][VL] Merge two consecutive aggregates to one in complete…
Browse files Browse the repository at this point in the history
… mode (#7752)

What changes were proposed in this pull request?
Fix #7751.

In issue #4668, support for the CH backend's merge agg capability was added. We can also support this feature in the velox backend.

How was this patch tested?
Through existing unit tests and GA verification, and additional new unit tests have been added.
  • Loading branch information
yikf authored Nov 20, 2024
1 parent 9315835 commit 5d7b963
Show file tree
Hide file tree
Showing 37 changed files with 2,559 additions and 2,561 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ object VeloxRuleApi {
injector.injectPreTransform(_ => PushDownInputFileExpression.PreOffload)
injector.injectPreTransform(c => FallbackOnANSIMode.apply(c.session))
injector.injectPreTransform(c => FallbackMultiCodegens.apply(c.session))
injector.injectPreTransform(c => MergeTwoPhasesHashBaseAggregate(c.session))
injector.injectPreTransform(_ => RewriteSubqueryBroadcast())
injector.injectPreTransform(c => BloomFilterMightContainJointRewriteRule.apply(c.session))
injector.injectPreTransform(c => ArrowScanReplaceRule.apply(c.session))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ abstract class HashAggregateExecTransformer(
VeloxIntermediateData.getIntermediateTypeNode(aggregateFunction)
)
aggregateNodeList.add(aggFunctionNode)
case Final =>
case Final | Complete =>
val aggFunctionNode = ExpressionBuilder.makeAggregateFunction(
VeloxAggregateFunctionsBuilder.create(args, aggregateFunction, aggregateMode),
childrenNodeList,
Expand Down Expand Up @@ -242,7 +242,7 @@ abstract class HashAggregateExecTransformer(
aggregateFunction.inputAggBufferAttributes.head.nullable)
)
aggregateNodeList.add(partialNode)
case Final =>
case Final | Complete =>
val aggFunctionNode = ExpressionBuilder.makeAggregateFunction(
VeloxAggregateFunctionsBuilder.create(args, aggregateFunction, aggregateMode),
childrenNodeList,
Expand Down Expand Up @@ -275,7 +275,7 @@ abstract class HashAggregateExecTransformer(
expression.mode match {
case Partial | PartialMerge =>
typeNodeList.add(VeloxIntermediateData.getIntermediateTypeNode(aggregateFunction))
case Final =>
case Final | Complete =>
typeNodeList.add(
ConverterUtils
.getTypeNode(aggregateFunction.dataType, aggregateFunction.nullable))
Expand Down Expand Up @@ -356,7 +356,7 @@ abstract class HashAggregateExecTransformer(
// The process of handling the inconsistency in column types and order between
// Spark and Velox is exactly the opposite of applyExtractStruct.
aggregateExpression.mode match {
case PartialMerge | Final =>
case PartialMerge | Final | Complete =>
val newInputAttributes = new ArrayBuffer[Attribute]()
val childNodes = new JArrayList[ExpressionNode]()
val (sparkOrders, sparkTypes) =
Expand Down Expand Up @@ -467,7 +467,7 @@ abstract class HashAggregateExecTransformer(
// by previous projection.
childrenNodes.add(ExpressionBuilder.makeSelection(colIdx))
colIdx += 1
case Partial =>
case Partial | Complete =>
aggFunc.children.foreach {
_ =>
childrenNodes.add(ExpressionBuilder.makeSelection(colIdx))
Expand Down Expand Up @@ -600,7 +600,7 @@ abstract class HashAggregateExecTransformer(
}
val aggregateFunc = aggExpr.aggregateFunction
val childrenNodes = aggExpr.mode match {
case Partial =>
case Partial | Complete =>
aggregateFunc.children.toList.map(
expr => {
ExpressionConverter
Expand Down Expand Up @@ -784,7 +784,7 @@ case class HashAggregateExecPullOutHelper(
expr.mode match {
case Partial | PartialMerge =>
expr.aggregateFunction.aggBufferAttributes
case Final =>
case Final | Complete =>
Seq(aggregateAttributes(index))
case other =>
throw new GlutenNotSupportException(s"Unsupported aggregate mode: $other.")
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 5d7b963

Please sign in to comment.