Skip to content

Commit

Permalink
[feature][dingo-calcite,dingo-exec] Support IFNULL and SUBSTRING_INDE…
Browse files Browse the repository at this point in the history 
…X functions, and ON DUPLICATE KEY UPDATE syntax
  • Loading branch information
@wt0530 @githubgxll
wt0530 authored and githubgxll committed Dec 25, 2024
1 parent dca8ed5 commit 656be33
Show file tree
Hide file tree
Showing 19 changed files with 680 additions and 68 deletions.
35 changes: 31 additions & 4 deletions dingo-calcite/src/main/codegen/templates/Parser.jj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1902,14 +1902,21 @@ SqlNode SqlInsert() :
final SqlNodeList columnList;
final Span s;
final Pair<SqlNodeList, SqlNodeList> p;
SqlNodeList sourceExpressionList;
SqlNodeList targetColumnList;
SqlIdentifier id;
}
{
(
<INSERT>
|
<UPSERT> { keywords.add(SqlInsertKeyword.UPSERT.symbol(getPos())); }
)
{ s = span(); }
{
s = span();
targetColumnList = new SqlNodeList(s.pos());
sourceExpressionList = new SqlNodeList(s.pos());
}
SqlInsertKeywords(keywords) {
keywordList = new SqlNodeList(keywords, s.addAll(keywords).pos());
}
Expand All @@ -1930,9 +1937,22 @@ SqlNode SqlInsert() :
}
| { columnList = null; }
)
source = OrderedQueryOrExpr(ExprContext.ACCEPT_QUERY) {
return new SqlInsert(s.end(source), keywordList, tableRef, source,
columnList);
source = OrderedQueryOrExpr(ExprContext.ACCEPT_QUERY)
[
<ON> <DUPLICATE> <KEY> <UPDATE>
id = SimpleIdentifier() {
targetColumnList.add(id);
}
<EQ> AddExpression(sourceExpressionList, ExprContext.ACCEPT_SUB_QUERY)
(
<COMMA>
id = SimpleIdentifier() { targetColumnList.add(id); }
<EQ> AddExpression(sourceExpressionList, ExprContext.ACCEPT_SUB_QUERY)
)*
]
{
return new io.dingodb.calcite.grammar.dml.SqlInsert(s.end(source), keywordList, tableRef, source,
columnList, sourceExpressionList, targetColumnList);
}
}

Expand Down Expand Up @@ -7523,13 +7543,15 @@ SqlIdentifier ReservedFunctionName() :
| <CURRENT_TIME>
| <CURRENT_TIMESTAMP>
| <DENSE_RANK>
| <DUPLICATE>
| <ELEMENT>
| <EVERY>
| <EXP>
| <FIRST_VALUE>
| <FLOOR>
| <FUSION>
| <INTERSECTION>
| <IFNULL>
| <GROUPING>
| <HOUR>
| <LAG>
Expand All @@ -7549,6 +7571,7 @@ SqlIdentifier ReservedFunctionName() :
| <NTILE>
| <NULLIF>
| <OCTET_LENGTH>
| <ON>
| <PERCENT_RANK>
| <PERCENTILE_CONT>
| <PERCENTILE_DISC>
Expand All @@ -7564,6 +7587,7 @@ SqlIdentifier ReservedFunctionName() :
| <SQRT>
| <STDDEV_POP>
| <STDDEV_SAMP>
| <SUBSTRING_INDEX>
| <SUM>
| <UPPER>
| <USER>
Expand Down Expand Up @@ -8067,6 +8091,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < DOUBLE: "DOUBLE" >
| < DOW: "DOW" >
| < DOY: "DOY" >
| < DUPLICATE: "DUPLICATE">
| < DROP: "DROP" >
| < DYNAMIC: "DYNAMIC" >
| < DYNAMIC_FUNCTION: "DYNAMIC_FUNCTION" >
Expand Down Expand Up @@ -8140,6 +8165,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < HOUR: "HOUR" >
| < HOURS: "HOURS" >
| < IDENTITY: "IDENTITY" >
| < IFNULL: "IFNULL">
| < IGNORE: "IGNORE" >
| < ILIKE: "ILIKE" >
| < IMMEDIATE: "IMMEDIATE" >
Expand Down Expand Up @@ -8494,6 +8520,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < SUBSTITUTE: "SUBSTITUTE" >
| < SUBSTRING: "SUBSTRING" >
| < SUBSTRING_REGEX: "SUBSTRING_REGEX" >
| < SUBSTRING_INDEX: "SUBSTRING_INDEX" >
| < SUCCEEDS: "SUCCEEDS" >
| < SUM: "SUM" >
| < SUNDAY: "SUNDAY" >
Expand Down
105 changes: 104 additions & 1 deletion dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlToRelConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,32 +16,45 @@

package io.dingodb.calcite;

import com.google.common.collect.ImmutableList;
import io.dingodb.calcite.rel.DingoFunctionScan;
import io.dingodb.calcite.rel.LogicalDingoDiskAnn;
import io.dingodb.calcite.rel.LogicalDingoDiskAnnBuild;
import io.dingodb.calcite.rel.LogicalDingoDiskAnnCountMemory;
import io.dingodb.calcite.rel.LogicalDingoDiskAnnLoad;
import io.dingodb.calcite.rel.LogicalDingoDiskAnnReset;
import io.dingodb.calcite.rel.LogicalDingoDiskAnnStatus;
import io.dingodb.calcite.rel.LogicalDingoDocument;
import io.dingodb.calcite.rel.LogicalDingoVector;
import io.dingodb.calcite.rel.logical.LogicalTableModify;
import io.dingodb.calcite.traits.DingoConvention;
import io.dingodb.common.table.DiskAnnTable;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.core.TableModify;
import org.apache.calcite.rel.hint.HintStrategyTable;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexRangeRef;
import org.apache.calcite.schema.ColumnStrategy;
import org.apache.calcite.schema.ModifiableTable;
import org.apache.calcite.schema.Table;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlInsert;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.sql.validate.TableDiskAnnFunctionNamespace;
import org.apache.calcite.sql.validate.TableFunctionNamespace;
import org.apache.calcite.sql.validate.TableHybridFunctionNamespace;
Expand All @@ -56,7 +69,10 @@
import org.checkerframework.checker.nullness.qual.Nullable;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;

class DingoSqlToRelConverter extends SqlToRelConverter {
Expand Down Expand Up @@ -113,6 +129,93 @@ protected void convertFrom(Blackboard bb, @Nullable SqlNode from) {
super.convertFrom(bb, from);
}

@Override
protected RelNode convertInsert(SqlInsert call) {
RelOptTable targetTable = getTargetTable(call);

final RelDataType targetRowType =
Objects.requireNonNull(validator, "validator").getValidatedNodeType(call);
assert targetRowType != null;
RelNode sourceRel = convertQueryRecursive(call.getSource(), true, targetRowType).project();
RelNode messageRel = convertColumnList(call, sourceRel);
final List<String> targetColumnNames = new ArrayList<>();
ImmutableList.Builder<RexNode> rexNodeSourceExpressionListBuilder = null;
if (call instanceof io.dingodb.calcite.grammar.dml.SqlInsert) {
io.dingodb.calcite.grammar.dml.SqlInsert sqlInsert = (io.dingodb.calcite.grammar.dml.SqlInsert) call;
SqlNodeList targetColumnList = sqlInsert.getTargetColumnList2();
if (targetColumnList != null && !targetColumnList.isEmpty()) {
for (SqlNode sqlNode : targetColumnList) {
SqlIdentifier id = (SqlIdentifier) sqlNode;
RelDataTypeField field = SqlValidatorUtil.getTargetField(
targetTable.getRowType(), typeFactory, id, catalogReader, targetTable);
assert field != null : "column " + id.toString() + " not found";
targetColumnNames.add(field.getName());
}
} else {
return super.convertInsert(call);
}
RelDataType sourceRowType = sourceRel.getRowType();
final RexRangeRef sourceRef = rexBuilder.makeRangeReference(sourceRowType, 0, false);
final Blackboard bb = createInsertBlackboard(targetTable, sourceRef, targetColumnNames);
rexNodeSourceExpressionListBuilder = ImmutableList.builder();
for (SqlNode n : sqlInsert.getSourceExpressionList()) {
RexNode rn = bb.convertExpression(n);
rexNodeSourceExpressionListBuilder.add(rn);
}

}

return createModify(targetTable, messageRel, targetColumnNames, rexNodeSourceExpressionListBuilder.build());
}

private RelNode createModify(RelOptTable targetTable,
RelNode source,
List<String> targetColumnNames,
List<RexNode> sourceExpressionList) {
final ModifiableTable modifiableTable =
targetTable.unwrap(ModifiableTable.class);
if (modifiableTable != null
&& modifiableTable == targetTable.unwrap(Table.class)) {
return modifiableTable.toModificationRel(cluster, targetTable,
catalogReader, source, LogicalTableModify.Operation.INSERT, null,
null, false);
}
return LogicalTableModify.create(
targetTable,
catalogReader,
source,
TableModify.Operation.INSERT,
null,
null,
false,
targetColumnNames,
sourceExpressionList);
}

private Blackboard createInsertBlackboard(
RelOptTable targetTable,
RexNode sourceRef,
List<String> targetColumnNames
) {
final Map<String, RexNode> nameToNameMap = new HashMap<>();
int j = 0;

final List<ColumnStrategy> strategies = targetTable.getColumnStrategies();
final List<String> targetFields = targetTable.getRowType().getFieldNames();
for (String targetColumnName : targetColumnNames) {
final int i = targetFields.indexOf(targetColumnName);
switch (strategies.get(i)) {
case STORED:
case VIRTUAL:
break;
default:
nameToNameMap.put(targetColumnName,
rexBuilder.makeFieldAccess(sourceRef, j++));
}
}
return createBlackboard(null, nameToNameMap, false);
}

@Override
protected void convertCollectionTable(Blackboard bb, SqlCall call) {
final SqlOperator operator = call.getOperator();
Expand Down
17 changes: 17 additions & 0 deletions dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoOperatorTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import io.dingodb.exec.fun.vector.VectorImageFun;
import io.dingodb.exec.fun.vector.VectorL2DistanceFun;
import io.dingodb.exec.fun.vector.VectorTextFun;
import io.dingodb.expr.runtime.op.special.IfNullFunFactory;
import io.dingodb.expr.runtime.op.string.ConcatFunFactory;
import io.dingodb.expr.runtime.op.string.LTrim1FunFactory;
import io.dingodb.expr.runtime.op.string.LeftFunFactory;
Expand All @@ -47,6 +48,7 @@
import io.dingodb.expr.runtime.op.string.RepeatFunFactory;
import io.dingodb.expr.runtime.op.string.ReverseFunFactory;
import io.dingodb.expr.runtime.op.string.RightFunFactory;
import io.dingodb.expr.runtime.op.string.SubStringIndexFunFactory;
import io.dingodb.expr.runtime.op.time.DateDiffFunFactory;
import io.dingodb.expr.runtime.op.time.DateFormat1FunFactory;
import io.dingodb.expr.runtime.op.time.FromUnixTimeFunFactory;
Expand Down Expand Up @@ -132,6 +134,13 @@ private void init() {
OperandTypes.NUMERIC_NUMERIC,
SqlFunctionCategory.STRING
);
registerFunction(
IfNullFunFactory.NAME,
ReturnTypes.explicit(SqlTypeName.ANY),
InferTypes.ANY_NULLABLE,
OperandTypes.ANY_ANY,
SqlFunctionCategory.USER_DEFINED_FUNCTION
);

// string
registerFunction(
Expand Down Expand Up @@ -201,6 +210,14 @@ private void init() {
SqlFunctionCategory.STRING
);

registerFunction(
SubStringIndexFunFactory.NAME,
ReturnTypes.VARCHAR_2000_NULLABLE,
DingoInferTypes.VARCHAR,
family(SqlTypeFamily.STRING, SqlTypeFamily.STRING, SqlTypeFamily.NUMERIC),
SqlFunctionCategory.STRING
);

// time
registerFunction(
FromUnixTimeFunFactory.NAME,
Expand Down
18 changes: 16 additions & 2 deletions dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dml/SqlInsert.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
package io.dingodb.calcite.grammar.dml;

import lombok.Getter;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNumericLiteral;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.checkerframework.checker.nullness.qual.Nullable;
Expand All @@ -31,6 +29,10 @@ public class SqlInsert extends org.apache.calcite.sql.SqlInsert {

@Getter
private boolean trace;
@Getter
private @Nullable SqlNodeList sourceExpressionList;
@Getter
private @Nullable SqlNodeList targetColumnList2;

public SqlInsert(SqlParserPos pos,
SqlNodeList keywords,
Expand All @@ -42,4 +44,16 @@ public SqlInsert(SqlParserPos pos,
this.trace = trace;
}

public SqlInsert(SqlParserPos pos,
SqlNodeList keywords,
SqlNode targetTable,
SqlNode source,
@Nullable SqlNodeList columnList,
@Nullable SqlNodeList sourceExpressionList,
@Nullable SqlNodeList targetColumnList) {
super(pos, keywords, targetTable, source, columnList);
this.sourceExpressionList = sourceExpressionList;
this.targetColumnList2 = targetColumnList;
}

}
38 changes: 37 additions & 1 deletion dingo-calcite/src/main/java/io/dingodb/calcite/rel/DingoTableModify.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ public final class DingoTableModify extends TableModify implements DingoRel {
@Getter
@Setter
private int autoIncrementColIndex;
@Getter
@Setter
private List<String> targetColumnNames;
@Getter
@Setter
private List<RexNode> sourceExpressionList2;

public DingoTableModify(
RelOptCluster cluster,
Expand All @@ -67,6 +73,34 @@ public DingoTableModify(
);
}

public DingoTableModify(
RelOptCluster cluster,
RelTraitSet traitSet,
RelOptTable table,
Prepare.CatalogReader catalogReader,
RelNode input,
Operation operation,
@Nullable List<String> updateColumnList,
@Nullable List<RexNode> sourceExpressionList,
boolean flattened,
List<String> targetColumnNames,
@Nullable List<RexNode> sourceExpressionList2
) {
super(
cluster,
traitSet,
table,
catalogReader,
input,
operation,
updateColumnList,
sourceExpressionList,
flattened
);
this.targetColumnNames = targetColumnNames;
this.sourceExpressionList2 = sourceExpressionList2;
}

@Override
public <T> T accept(@NonNull DingoRelVisitor<T> visitor) {
return visitor.visit(this);
Expand All @@ -83,7 +117,9 @@ public <T> T accept(@NonNull DingoRelVisitor<T> visitor) {
getOperation(),
getUpdateColumnList(),
getSourceExpressionList(),
isFlattened()
isFlattened(),
getTargetColumnNames(),
getSourceExpressionList2()
);
}

Expand Down
Loading

0 comments on commit 656be33

Please sign in to comment.