-
Notifications
You must be signed in to change notification settings - Fork 103
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Support UDF in plan generator #3040
base: main
Are you sure you want to change the base?
Changes from 71 commits
3a5a44b
53416bf
3f88bb4
773e0dd
fa472d7
4c32432
ccd8e8d
403a4a2
cada59a
3b02019
138a7fc
f8b3658
b4edc0a
c42c0ce
a570479
f4e5cac
c10ec5b
0d9cc50
057fee0
37c8641
a52d15a
74c24fb
2f33668
01c4d01
8c70b2a
5d59fa2
7e84176
0d54f1e
543a79c
8fd8208
f5b3314
9956a9f
5476bb4
f9fd6a9
651816c
2c886c8
38a8004
4f7a23f
25eb6cf
6ebd19b
f8088f5
157ab0c
0daa4ce
9a325bf
bffaca4
b3589d2
46eaaf1
523859e
18312ed
0a70d6e
d058c09
67fad06
3adcce3
74602e7
fa28a85
b0e5b37
91322a2
eee22fc
41112ed
df4235e
c343ed3
6aaae1d
4f2b546
e11b315
29b9cb2
52b3ba4
1cc89a6
91e6117
262f03b
4015220
e8c4dfc
8076737
cea7e31
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -28,6 +28,7 @@ | |
import com.apple.foundationdb.record.metadata.MetaDataException; | ||
import com.apple.foundationdb.record.metadata.RecordType; | ||
import com.apple.foundationdb.record.metadata.SyntheticRecordType; | ||
import com.apple.foundationdb.record.metadata.ScalarValuedFunction; | ||
import com.apple.foundationdb.record.metadata.UnnestedRecordType; | ||
import com.apple.foundationdb.record.metadata.expressions.KeyExpression; | ||
import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression; | ||
|
@@ -48,6 +49,7 @@ | |
import java.util.LinkedHashMap; | ||
import java.util.List; | ||
import java.util.Map; | ||
import java.util.Set; | ||
import java.util.TreeMap; | ||
import java.util.concurrent.ConcurrentHashMap; | ||
import java.util.stream.Collectors; | ||
|
@@ -83,6 +85,8 @@ public class RecordMetaData implements RecordMetaDataProvider { | |
@Nonnull | ||
private final Map<Object, SyntheticRecordType<?>> recordTypeKeyToSyntheticTypeMap; | ||
@Nonnull | ||
private final Set<ScalarValuedFunction> scalarValuedFunctions; | ||
@Nonnull | ||
private final Map<String, Index> indexes; | ||
@Nonnull | ||
private final Map<String, Index> universalIndexes; | ||
|
@@ -112,6 +116,7 @@ protected RecordMetaData(@Nonnull RecordMetaData orig) { | |
Collections.unmodifiableMap(orig.indexes), | ||
Collections.unmodifiableMap(orig.universalIndexes), | ||
Collections.unmodifiableList(orig.formerIndexes), | ||
Collections.unmodifiableSet(orig.scalarValuedFunctions), | ||
orig.splitLongRecords, | ||
orig.storeRecordVersions, | ||
orig.version, | ||
|
@@ -131,6 +136,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor, | |
@Nonnull Map<String, Index> indexes, | ||
@Nonnull Map<String, Index> universalIndexes, | ||
@Nonnull List<FormerIndex> formerIndexes, | ||
@Nonnull Set<ScalarValuedFunction> scalarValuedFunctions, | ||
boolean splitLongRecords, | ||
boolean storeRecordVersions, | ||
int version, | ||
|
@@ -147,6 +153,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor, | |
this.indexes = indexes; | ||
this.universalIndexes = universalIndexes; | ||
this.formerIndexes = formerIndexes; | ||
this.scalarValuedFunctions = scalarValuedFunctions; | ||
this.splitLongRecords = splitLongRecords; | ||
this.storeRecordVersions = storeRecordVersions; | ||
this.version = version; | ||
|
@@ -342,6 +349,11 @@ public List<FormerIndex> getFormerIndexes() { | |
return formerIndexes; | ||
} | ||
|
||
@Nonnull | ||
public Collection<ScalarValuedFunction> getAllScalarValuedFunctions() { | ||
return scalarValuedFunctions; | ||
} | ||
|
||
public boolean isSplitLongRecords() { | ||
return splitLongRecords; | ||
} | ||
|
@@ -693,6 +705,7 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor | |
} | ||
|
||
// Add in the final options. | ||
builder.addAllScalarValuedFunction(scalarValuedFunctions.stream().map(ScalarValuedFunction::toProto).collect(Collectors.toList())); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is not really a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. removed the confusing comment. |
||
builder.setSplitLongRecords(splitLongRecords); | ||
builder.setStoreRecordVersions(storeRecordVersions); | ||
builder.setVersion(version); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -22,7 +22,7 @@ | |
|
||
import com.apple.foundationdb.annotation.API; | ||
import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; | ||
import com.apple.foundationdb.record.RecordMetaDataProto; | ||
import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; | ||
import com.apple.foundationdb.record.logging.LogMessageKeys; | ||
import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression; | ||
import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; | ||
|
@@ -392,21 +392,21 @@ public static class Evaluated { | |
* Values used in index keys in place of missing fields. | ||
*/ | ||
public enum NullStandin { | ||
NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks. | ||
NULL_UNIQUE(RecordMetaDataProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness. | ||
NOT_NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value. | ||
NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. static imports may make this more readable. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. unfortunately this file also has metadata.expressions.KeyExpression. |
||
NULL_UNIQUE(RecordKeyExpressionProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness. | ||
NOT_NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value. | ||
|
||
private RecordMetaDataProto.Field.NullInterpretation proto; | ||
private RecordKeyExpressionProto.Field.NullInterpretation proto; | ||
|
||
NullStandin(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) { | ||
NullStandin(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) { | ||
proto = nullInterpretation; | ||
} | ||
|
||
public RecordMetaDataProto.Field.NullInterpretation toProto() { | ||
public RecordKeyExpressionProto.Field.NullInterpretation toProto() { | ||
return proto; | ||
} | ||
|
||
public static NullStandin valueOf(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException { | ||
public static NullStandin valueOf(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException { | ||
switch (nullInterpretation) { | ||
case NOT_UNIQUE: | ||
return NULL; | ||
|
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -0,0 +1,67 @@ | ||||||
/* | ||||||
* UDF.java | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
* | ||||||
* This source file is part of the FoundationDB open source project | ||||||
* | ||||||
* Copyright 2015-2025 Apple Inc. and the FoundationDB project authors | ||||||
* | ||||||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||||||
* you may not use this file except in compliance with the License. | ||||||
* You may obtain a copy of the License at | ||||||
* | ||||||
* http://www.apache.org/licenses/LICENSE-2.0 | ||||||
* | ||||||
* Unless required by applicable law or agreed to in writing, software | ||||||
* distributed under the License is distributed on an "AS IS" BASIS, | ||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||||||
* See the License for the specific language governing permissions and | ||||||
* limitations under the License. | ||||||
*/ | ||||||
|
||||||
package com.apple.foundationdb.record.metadata; | ||||||
|
||||||
import com.apple.foundationdb.record.PlanHashable; | ||||||
import com.apple.foundationdb.record.PlanSerializationContext; | ||||||
import com.apple.foundationdb.record.RecordMetaDataProto; | ||||||
import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; | ||||||
import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; | ||||||
|
||||||
import javax.annotation.Nonnull; | ||||||
|
||||||
/** | ||||||
* Defines a scalar User-defined-function. | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please expand on the documentation. |
||||||
*/ | ||||||
public class ScalarValuedFunction { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why don't we just make a trait-like interface called
|
||||||
@Nonnull | ||||||
private final MacroFunction macroFunction; | ||||||
|
||||||
public ScalarValuedFunction(@Nonnull MacroFunction functionValue) { | ||||||
this.macroFunction = functionValue; | ||||||
} | ||||||
|
||||||
@Nonnull | ||||||
public MacroFunction getMacroFunction() { | ||||||
return macroFunction; | ||||||
} | ||||||
|
||||||
@Nonnull | ||||||
public String getFunctionName() { | ||||||
return macroFunction.getFunctionName(); | ||||||
} | ||||||
|
||||||
@Nonnull | ||||||
public RecordMetaDataProto.ScalarValuedFunction toProto() { | ||||||
PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, | ||||||
PlanHashable.CURRENT_FOR_CONTINUATION); | ||||||
return RecordMetaDataProto.ScalarValuedFunction.newBuilder() | ||||||
.setFunctionValue(macroFunction.toProto(serializationContext)) | ||||||
.build(); | ||||||
} | ||||||
|
||||||
@Nonnull | ||||||
public static ScalarValuedFunction fromProto(RecordMetaDataProto.ScalarValuedFunction proto) { | ||||||
PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, | ||||||
PlanHashable.CURRENT_FOR_CONTINUATION); | ||||||
return new ScalarValuedFunction(MacroFunction.fromProto(serializationContext, proto.getFunctionValue())); | ||||||
} | ||||||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There are a few different comments further down in this PR around this. Let me just point out here that this structure is persisted and long-lived. In fact, modifying its contents and layout may be impossible or very hard once this is used. One thing I would like to avoid is to get ourselves into a dead end here.
We most likely want to support other functions-like objects than macros. For instance, we would like to use full SQL-bodied functions or table functions. This attribute here should potentially be able to contain all of these. Using my naming from a files down, this should be something named
SerializableFunction
(because that's all you need here, i.e. a name, a bunch of parameter declarations with their types and a return type). Under the hood it can be any of the particular flavors I mentioned (of which we only have a simpleMacroFunction
as of yet).