From c6d605c7dac92bd29c64e53fa5a87bdb1f68dcbc Mon Sep 17 00:00:00 2001 From: Ivan Chesnov Date: Mon, 23 Oct 2023 11:57:18 +0300 Subject: [PATCH] GH-38246: [JAVA] added new getTransferPair() function that takes in a Field type for Complex Type Vectors (#38261) (#57) ### Rationale for this change Additionally, a new function **getTransferPair(Field, Allocator)** is introduced so that a new Field method is not constructed each time getTransferPair is called on the Vector. Added Field.setChildren() method and made **children** not final - for updating **field** object inside complex vector(it possible that Field will be without children on creation) - optimisation for keeping using already existing Field object. ### What changes are included in this PR? - `getTransferPair` method for ComplexType Vectors and for BaseVariableWidthVector's - added `Field.setChildren()` method and made **children** not final - for updating **field** object inside complex vector(it possible that Field will be without children on creation) - optimisation for keeping using already existing Field object. ### Are these changes tested? Yes, some tests have been added to verify these changes. ### Are there any user-facing changes? Yes. **This PR includes breaking changes to public APIs.** * Closes: #38246 Lead-authored-by: Ivan Chesnov Co-authored-by: Ivan Chesnov Signed-off-by: David Li (cherry picked from commit a376e3c21f60540ed8952b066f9367be99ba145f) --- .../codegen/templates/DenseUnionVector.java | 22 +++++- .../main/codegen/templates/UnionVector.java | 19 ++++- .../arrow/vector/BaseFixedWidthVector.java | 12 +++ .../vector/BaseLargeVariableWidthVector.java | 21 ++++++ .../arrow/vector/BaseVariableWidthVector.java | 20 +++++ .../arrow/vector/ExtensionTypeVector.java | 10 +++ .../arrow/vector/LargeVarBinaryVector.java | 9 +++ .../arrow/vector/LargeVarCharVector.java | 9 +++ .../org/apache/arrow/vector/NullVector.java | 12 ++- .../org/apache/arrow/vector/ValueVector.java | 38 ++++++++++ .../apache/arrow/vector/VarBinaryVector.java | 9 +++ .../apache/arrow/vector/VarCharVector.java | 9 +++ .../vector/complex/FixedSizeListVector.java | 51 ++++++++++--- .../arrow/vector/complex/LargeListVector.java | 43 +++++++++-- .../arrow/vector/complex/ListVector.java | 41 +++++++++-- .../arrow/vector/complex/MapVector.java | 17 ++++- .../complex/NonNullableStructVector.java | 73 +++++++++++++++---- .../arrow/vector/complex/StructVector.java | 54 +++++++++++++- .../arrow/vector/TestLargeListVector.java | 23 ++++++ .../vector/TestLargeVarBinaryVector.java | 18 +++++ .../arrow/vector/TestLargeVarCharVector.java | 17 +++++ .../apache/arrow/vector/TestListVector.java | 21 ++++++ .../apache/arrow/vector/TestMapVector.java | 22 ++++++ .../apache/arrow/vector/TestStructVector.java | 43 +++++++++++ .../arrow/vector/TestVectorSchemaRoot.java | 1 + 25 files changed, 563 insertions(+), 51 deletions(-) diff --git a/java/vector/src/main/codegen/templates/DenseUnionVector.java b/java/vector/src/main/codegen/templates/DenseUnionVector.java index fba9302f34221..12fc52af3c18e 100644 --- a/java/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/java/vector/src/main/codegen/templates/DenseUnionVector.java @@ -234,8 +234,8 @@ public synchronized byte registerNewTypeId(Field field) { typeFields.length + " relative types. Please use union of union instead"); } byte typeId = nextTypeId; - if (fieldType != null) { - int[] typeIds = ((ArrowType.Union) fieldType.getType()).getTypeIds(); + if (this.fieldType != null) { + int[] typeIds = ((ArrowType.Union) this.fieldType.getType()).getTypeIds(); if (typeIds != null) { int thisTypeId = typeIds[nextTypeId]; if (thisTypeId > Byte.MAX_VALUE) { @@ -533,7 +533,7 @@ public Field getField() { } else { final UnionMode mode = UnionMode.Dense; fieldType = new FieldType(this.fieldType.isNullable(), new ArrowType.Union(mode, typeIds), - this.fieldType.getDictionary(), this.fieldType.getMetadata()); + this.fieldType.getDictionary(), this.fieldType.getMetadata()); } return new Field(name, fieldType, childFields); @@ -554,6 +554,16 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB return new org.apache.arrow.vector.complex.DenseUnionVector.TransferImpl(ref, allocator, callBack); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return getTransferPair(field, allocator, null); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return new org.apache.arrow.vector.complex.DenseUnionVector.TransferImpl(field, allocator, callBack); + } + @Override public TransferPair makeTransferPair(ValueVector target) { return new TransferImpl((DenseUnionVector) target); @@ -598,6 +608,12 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { createTransferPairs(); } + public TransferImpl(Field field, BufferAllocator allocator, CallBack callBack) { + to = new DenseUnionVector(field.getName(), allocator, null, callBack); + internalStruct.makeTransferPair(to.internalStruct); + createTransferPairs(); + } + public TransferImpl(DenseUnionVector to) { this.to = to; internalStruct.makeTransferPair(to.internalStruct); diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 0446faab7a95c..ea79c5c2fba76 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -144,8 +144,8 @@ public void initializeChildrenFromFields(List children) { int count = 0; for (Field child: children) { int typeId = Types.getMinorTypeForArrowType(child.getType()).ordinal(); - if (fieldType != null) { - int[] typeIds = ((ArrowType.Union)fieldType.getType()).getTypeIds(); + if (this.fieldType != null) { + int[] typeIds = ((ArrowType.Union)this.fieldType.getType()).getTypeIds(); if (typeIds != null) { typeId = typeIds[count++]; } @@ -495,6 +495,16 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB return new org.apache.arrow.vector.complex.UnionVector.TransferImpl(ref, allocator, callBack); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return getTransferPair(field, allocator, null); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return new org.apache.arrow.vector.complex.UnionVector.TransferImpl(field, allocator, callBack); + } + @Override public TransferPair makeTransferPair(ValueVector target) { return new TransferImpl((UnionVector) target); @@ -547,6 +557,11 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { internalStructVectorTransferPair = internalStruct.makeTransferPair(to.internalStruct); } + public TransferImpl(Field field, BufferAllocator allocator, CallBack callBack) { + to = new UnionVector(field.getName(), allocator, null, callBack); + internalStructVectorTransferPair = internalStruct.makeTransferPair(to.internalStruct); + } + public TransferImpl(UnionVector to) { this.to = to; internalStructVectorTransferPair = internalStruct.makeTransferPair(to.internalStruct); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 223ae9aa8cb1c..02a2bbc57909a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -562,6 +562,18 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB return getTransferPair(ref, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param field The field materialized by this vector. + * @param allocator allocator for the target vector + * @param callBack not used + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(field, allocator); + } + /** * Construct a transfer pair of this vector and another vector of same type. * @param allocator allocator for the target vector diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 90694db830cd6..cf0ea69336b60 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -655,6 +655,18 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB return getTransferPair(ref, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param field The field materialized by this vector + * @param allocator allocator for the target vector + * @param callBack not used + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(field, allocator); + } + /** * Construct a transfer pair of this vector and another vector of same type. * @param allocator allocator for the target vector @@ -665,6 +677,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { return getTransferPair(getName(), allocator); } + /** * Construct a transfer pair of this vector and another vector of same type. * @param ref name of the target vector @@ -673,6 +686,14 @@ public TransferPair getTransferPair(BufferAllocator allocator) { */ public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param field The field materialized by this vector + * @param allocator allocator for the target vector + * @return TransferPair + */ + public abstract TransferPair getTransferPair(Field field, BufferAllocator allocator); + /** * Transfer this vector'data to another vector. The memory associated * with this vector is transferred to the allocator of target vector diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 2a89590bf8440..8554eb7f353fa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -685,6 +685,18 @@ public ArrowBuf[] getBuffers(boolean clear) { return buffers; } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param field The field materialized by this vector. + * @param allocator allocator for the target vector + * @param callBack not used + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(field, allocator); + } + /** * Construct a transfer pair of this vector and another vector of same type. * @param ref name of the target vector @@ -715,6 +727,14 @@ public TransferPair getTransferPair(BufferAllocator allocator) { */ public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param field The field materialized by this vector. + * @param allocator allocator for the target vector + * @return TransferPair + */ + public abstract TransferPair getTransferPair(Field field, BufferAllocator allocator); + /** * Transfer this vector'data to another vector. The memory associated * with this vector is transferred to the allocator of target vector diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java index 9433719c5b8b2..a70efe61bcdfe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java @@ -125,6 +125,16 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB return underlyingVector.getTransferPair(ref, allocator, callBack); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return underlyingVector.getTransferPair(field, allocator); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return underlyingVector.getTransferPair(field, allocator, callBack); + } + @Override public TransferPair makeTransferPair(ValueVector target) { return underlyingVector.makeTransferPair(target); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java index 0063a61da570a..6806b958dad93 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java @@ -253,6 +253,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new TransferImpl(ref, allocator); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new TransferImpl(field, allocator); + } + /** * Construct a TransferPair with a desired target vector of the same type. * @@ -271,6 +276,10 @@ public TransferImpl(String ref, BufferAllocator allocator) { to = new LargeVarBinaryVector(ref, field.getFieldType(), allocator); } + public TransferImpl(Field field, BufferAllocator allocator) { + to = new LargeVarBinaryVector(field, allocator); + } + public TransferImpl(LargeVarBinaryVector to) { this.to = to; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java index 1f8d9b7d3a85c..b1eff66e2f863 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java @@ -280,6 +280,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new LargeVarCharVector.TransferImpl(ref, allocator); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new LargeVarCharVector.TransferImpl(field, allocator); + } + /** * Construct a TransferPair with a desired target vector of the same type. * @@ -298,6 +303,10 @@ public TransferImpl(String ref, BufferAllocator allocator) { to = new LargeVarCharVector(ref, field.getFieldType(), allocator); } + public TransferImpl(Field field, BufferAllocator allocator) { + to = new LargeVarCharVector(field, allocator); + } + public TransferImpl(LargeVarCharVector to) { this.to = to; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java index 6e4c2764bdcc4..05a718e910df5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java @@ -106,7 +106,7 @@ public Types.MinorType getMinorType() { @Override public TransferPair getTransferPair(BufferAllocator allocator) { - return getTransferPair(null, allocator); + return getTransferPair((String) null, allocator); } @Override @@ -162,11 +162,21 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new TransferImpl(); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new TransferImpl(); + } + @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return getTransferPair(ref, allocator); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(field, allocator); + } + @Override public TransferPair makeTransferPair(ValueVector target) { return new TransferImpl((NullVector) target); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index aa29c29314e33..cd7e8903ff931 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -131,10 +131,48 @@ public interface ValueVector extends Closeable, Iterable { */ TransferPair getTransferPair(BufferAllocator allocator); + /** + * To transfer quota responsibility. + * + * @param ref the name of the vector + * @param allocator the target allocator + * @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of + * the same type. + */ TransferPair getTransferPair(String ref, BufferAllocator allocator); + /** + * To transfer quota responsibility. + * + * @param field the Field object used by the target vector + * @param allocator the target allocator + * @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of + * the same type. + */ + TransferPair getTransferPair(Field field, BufferAllocator allocator); + + /** + * To transfer quota responsibility. + * + * @param ref the name of the vector + * @param allocator the target allocator + * @param callBack A schema change callback. + * @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of + * the same type. + */ TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack); + /** + * To transfer quota responsibility. + * + * @param field the Field object used by the target vector + * @param allocator the target allocator + * @param callBack A schema change callback. + * @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of + * the same type. + */ + TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack); + /** * Makes a new transfer pair used to transfer underlying buffers. * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java index 34e072aaa8324..b43cd33d05391 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java @@ -254,6 +254,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new TransferImpl(ref, allocator); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new TransferImpl(field, allocator); + } + /** * Construct a TransferPair with a desired target vector of the same type. * @@ -272,6 +277,10 @@ public TransferImpl(String ref, BufferAllocator allocator) { to = new VarBinaryVector(ref, field.getFieldType(), allocator); } + public TransferImpl(Field field, BufferAllocator allocator) { + to = new VarBinaryVector(field, allocator); + } + public TransferImpl(VarBinaryVector to) { this.to = to; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java index bc5c68b29f310..b4145617dd82d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java @@ -280,6 +280,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new TransferImpl(ref, allocator); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new TransferImpl(field, allocator); + } + /** * Construct a TransferPair with a desired target vector of the same type. * @@ -298,6 +303,10 @@ public TransferImpl(String ref, BufferAllocator allocator) { to = new VarCharVector(ref, field.getFieldType(), allocator); } + public TransferImpl(Field field, BufferAllocator allocator) { + to = new VarCharVector(field, allocator); + } + public TransferImpl(VarCharVector to) { this.to = to; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 0f78829181142..367335436aecd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -71,8 +71,7 @@ public static FixedSizeListVector empty(String name, int size, BufferAllocator a private FieldVector vector; private ArrowBuf validityBuffer; private final int listSize; - private final FieldType fieldType; - private final String name; + private Field field; private UnionFixedSizeListReader reader; private int valueCount; @@ -90,13 +89,25 @@ public FixedSizeListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack unusedSchemaChangeCallback) { + this(new Field(name, fieldType, null), allocator, unusedSchemaChangeCallback); + } + + /** + * Creates a new instance. + * + * @param field The field materialized by this vector. + * @param allocator The allocator to use for creating/reallocating buffers for the vector. + * @param unusedSchemaChangeCallback Currently unused. + */ + public FixedSizeListVector(Field field, + BufferAllocator allocator, + CallBack unusedSchemaChangeCallback) { super(allocator); - this.name = name; + this.field = field; this.validityBuffer = allocator.getEmpty(); this.vector = ZeroVector.INSTANCE; - this.fieldType = fieldType; - this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize(); + this.listSize = ((ArrowType.FixedSizeList) field.getFieldType().getType()).getListSize(); Preconditions.checkArgument(listSize >= 0, "list size must be non-negative"); this.valueCount = 0; this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); @@ -104,8 +115,11 @@ public FixedSizeListVector(String name, @Override public Field getField() { - List children = Collections.singletonList(getDataVector().getField()); - return new Field(name, fieldType, children); + if (field.getChildren().contains(getDataVector().getField())) { + return field; + } + field = new Field(field.getName(), field.getFieldType(), Collections.singletonList(getDataVector().getField())); + return field; } @Override @@ -115,7 +129,7 @@ public MinorType getMinorType() { @Override public String getName() { - return name; + return field.getName(); } /** Get the fixed size for each list. */ @@ -133,6 +147,7 @@ public void initializeChildrenFromFields(List children) { checkArgument(addOrGetVector.isCreated(), "Child vector already existed: %s", addOrGetVector.getVector()); addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren()); + this.field = new Field(this.field.getName(), this.field.getFieldType(), children); } @Override @@ -172,8 +187,6 @@ private void setReaderAndWriterIndex() { /** * Get the inner vectors. * - * @deprecated This API will be removed as the current implementations no longer support inner vectors. - * * @return the inner vectors for this field as defined by the TypeLayout */ @Deprecated @@ -403,7 +416,7 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { @Override public UnionVector promoteToUnion() { - UnionVector vector = new UnionVector(name, allocator, /* field type */ null, /* call-back */ null); + UnionVector vector = new UnionVector(getName(), allocator, /* field type */ null, /* call-back */ null); this.vector.clear(); this.vector = vector; invalidateReader(); @@ -519,11 +532,21 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return getTransferPair(ref, allocator, null); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return getTransferPair(field, allocator, null); + } + @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return new TransferImpl(ref, allocator, callBack); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return new TransferImpl(field, allocator, callBack); + } + @Override public TransferPair makeTransferPair(ValueVector target) { return new TransferImpl((FixedSizeListVector) target); @@ -567,7 +590,11 @@ private class TransferImpl implements TransferPair { TransferPair dataPair; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { - this(new FixedSizeListVector(name, allocator, fieldType, callBack)); + this(new FixedSizeListVector(name, allocator, field.getFieldType(), callBack)); + } + + public TransferImpl(Field field, BufferAllocator allocator, CallBack callBack) { + this(new FixedSizeListVector(field, allocator, callBack)); } public TransferImpl(FixedSizeListVector to) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 6ef5f994fc6f4..4d20485204dae 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -21,7 +21,6 @@ import static org.apache.arrow.memory.util.LargeMemoryUtil.capAtMaxInt; import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; import static org.apache.arrow.util.Preconditions.checkArgument; -import static org.apache.arrow.util.Preconditions.checkNotNull; import java.util.ArrayList; import java.util.Arrays; @@ -98,12 +97,11 @@ public static LargeListVector empty(String name, BufferAllocator allocator) { protected final CallBack callBack; protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; - private final String name; protected String defaultDataVectorName = DATA_VECTOR_NAME; protected ArrowBuf validityBuffer; protected UnionLargeListReader reader; - private final FieldType fieldType; + private Field field; private int validityAllocationSizeInBytes; /** @@ -120,10 +118,20 @@ public static LargeListVector empty(String name, BufferAllocator allocator) { * @param callBack A schema change callback. */ public LargeListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + this(new Field(name, fieldType, null), allocator, callBack); + } + + /** + * Creates a new instance. + * + * @param field The field materialized by this vector. + * @param allocator The allocator to use for creating/reallocating buffers for the vector. + * @param callBack A schema change callback. + */ + public LargeListVector(Field field, BufferAllocator allocator, CallBack callBack) { super(allocator); - this.name = name; + this.field = field; this.validityBuffer = allocator.getEmpty(); - this.fieldType = checkNotNull(fieldType); this.callBack = callBack; this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); this.lastSet = -1; @@ -142,6 +150,7 @@ public void initializeChildrenFromFields(List children) { checkArgument(addOrGetVector.isCreated(), "Child vector already existed: %s", addOrGetVector.getVector()); addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren()); + this.field = new Field(this.field.getName(), this.field.getFieldType(), children); } @Override @@ -474,11 +483,21 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return getTransferPair(ref, allocator, null); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return getTransferPair(field, allocator, null); + } + @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return new TransferImpl(ref, allocator, callBack); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return new TransferImpl(field, allocator, callBack); + } + @Override public TransferPair makeTransferPair(ValueVector target) { return new TransferImpl((LargeListVector) target); @@ -569,7 +588,11 @@ private class TransferImpl implements TransferPair { TransferPair dataTransferPair; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { - this(new LargeListVector(name, allocator, fieldType, callBack)); + this(new LargeListVector(name, allocator, field.getFieldType(), callBack)); + } + + public TransferImpl(Field field, BufferAllocator allocator, CallBack callBack) { + this(new LargeListVector(field, allocator, callBack)); } public TransferImpl(LargeListVector to) { @@ -763,7 +786,11 @@ public int getBufferSizeFor(int valueCount) { @Override public Field getField() { - return new Field(getName(), fieldType, Collections.singletonList(getDataVector().getField())); + if (field.getChildren().contains(getDataVector().getField())) { + return field; + } + field = new Field(field.getName(), field.getFieldType(), Collections.singletonList(getDataVector().getField())); + return field; } @Override @@ -773,7 +800,7 @@ public MinorType getMinorType() { @Override public String getName() { - return name; + return field.getName(); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 730068034b77e..505df40dce189 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -21,7 +21,6 @@ import static org.apache.arrow.memory.util.LargeMemoryUtil.capAtMaxInt; import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; import static org.apache.arrow.util.Preconditions.checkArgument; -import static org.apache.arrow.util.Preconditions.checkNotNull; import java.util.ArrayList; import java.util.Arrays; @@ -76,7 +75,7 @@ public static ListVector empty(String name, BufferAllocator allocator) { protected ArrowBuf validityBuffer; protected UnionListReader reader; private CallBack callBack; - protected final FieldType fieldType; + protected Field field; protected int validityAllocationSizeInBytes; /** @@ -93,9 +92,20 @@ public static ListVector empty(String name, BufferAllocator allocator) { * @param callBack A schema change callback. */ public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, allocator, callBack); + this(new Field(name, fieldType, null), allocator, callBack); + } + + /** + * Constructs a new instance. + * + * @param field The field materialized by this vector. + * @param allocator The allocator to use for allocating/reallocating buffers. + * @param callBack A schema change callback. + */ + public ListVector(Field field, BufferAllocator allocator, CallBack callBack) { + super(field.getName(), allocator, callBack); this.validityBuffer = allocator.getEmpty(); - this.fieldType = checkNotNull(fieldType); + this.field = field; this.callBack = callBack; this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); this.lastSet = -1; @@ -111,6 +121,7 @@ public void initializeChildrenFromFields(List children) { checkArgument(addOrGetVector.isCreated(), "Child vector already existed: %s", addOrGetVector.getVector()); addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren()); + this.field = new Field(this.field.getName(), this.field.getFieldType(), children); } @Override @@ -370,11 +381,21 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return getTransferPair(ref, allocator, null); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return getTransferPair(field, allocator, null); + } + @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return new TransferImpl(ref, allocator, callBack); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return new TransferImpl(field, allocator, callBack); + } + @Override public TransferPair makeTransferPair(ValueVector target) { return new TransferImpl((ListVector) target); @@ -440,7 +461,11 @@ private class TransferImpl implements TransferPair { TransferPair dataTransferPair; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { - this(new ListVector(name, allocator, fieldType, callBack)); + this(new ListVector(name, allocator, field.getFieldType(), callBack)); + } + + public TransferImpl(Field field, BufferAllocator allocator, CallBack callBack) { + this(new ListVector(field, allocator, callBack)); } public TransferImpl(ListVector to) { @@ -611,7 +636,11 @@ public int getBufferSizeFor(int valueCount) { @Override public Field getField() { - return new Field(getName(), fieldType, Collections.singletonList(getDataVector().getField())); + if (field.getChildren().contains(getDataVector().getField())) { + return field; + } + field = new Field(field.getName(), field.getFieldType(), Collections.singletonList(getDataVector().getField())); + return field; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index b8f3f32a73a29..c1913574bab19 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -76,6 +76,11 @@ public MapVector(String name, BufferAllocator allocator, FieldType fieldType, Ca defaultDataVectorName = DATA_VECTOR_NAME; } + public MapVector(Field field, BufferAllocator allocator, CallBack callBack) { + super(field, allocator, callBack); + defaultDataVectorName = DATA_VECTOR_NAME; + } + /** * Initialize child vectors of the map from the given list of fields. * @@ -99,6 +104,7 @@ public void initializeChildrenFromFields(List children) { checkArgument(addOrGetVector.isCreated(), "Child vector already existed: %s", addOrGetVector.getVector()); addOrGetVector.getVector().initializeChildrenFromFields(structField.getChildren()); + this.field = new Field(this.field.getName(), this.field.getFieldType(), children); } /** @@ -130,6 +136,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return getTransferPair(ref, allocator, null); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new TransferImpl(field, allocator, null); + } + @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return new TransferImpl(ref, allocator, callBack); @@ -146,7 +157,11 @@ private class TransferImpl implements TransferPair { TransferPair dataTransferPair; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { - this(new MapVector(name, allocator, fieldType, callBack)); + this(new MapVector(name, allocator, field.getFieldType(), callBack)); + } + + public TransferImpl(Field field, BufferAllocator allocator, CallBack callBack) { + this(new MapVector(field, allocator, callBack)); } public TransferImpl(MapVector to) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java index 4da2668121af6..30a2cffa37572 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java @@ -17,8 +17,6 @@ package org.apache.arrow.vector.complex; -import static org.apache.arrow.util.Preconditions.checkNotNull; - import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -62,7 +60,7 @@ public static NonNullableStructVector emptyWithDuplicates(String name, BufferAll } private final SingleStructReaderImpl reader = new SingleStructReaderImpl(this); - protected final FieldType fieldType; + protected Field field; public int valueCount; /** @@ -76,13 +74,20 @@ public NonNullableStructVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, - allocator, - callBack, - null, - true); - this.fieldType = checkNotNull(fieldType); - this.valueCount = 0; + this(new Field(name, fieldType, null), allocator, callBack); + } + + /** + * Constructs a new instance. + * + * @param field The field materialized by this vector. + * @param allocator The allocator to use to allocating/reallocating buffers. + * @param callBack A schema change callback. + */ + public NonNullableStructVector(Field field, + BufferAllocator allocator, + CallBack callBack) { + this(field, allocator, callBack, null, true); } /** @@ -100,8 +105,24 @@ public NonNullableStructVector(String name, CallBack callBack, ConflictPolicy conflictPolicy, boolean allowConflictPolicyChanges) { - super(name, allocator, callBack, conflictPolicy, allowConflictPolicyChanges); - this.fieldType = checkNotNull(fieldType); + this(new Field(name, fieldType, null), allocator, callBack, conflictPolicy, allowConflictPolicyChanges); + } + + /** + * Constructs a new instance. + * + * @param field The field materialized by this vector. + * @param allocator The allocator to use to allocating/reallocating buffers. + * @param callBack A schema change callback. + * @param conflictPolicy How to handle duplicate field names in the struct. + */ + public NonNullableStructVector(Field field, + BufferAllocator allocator, + CallBack callBack, + ConflictPolicy conflictPolicy, + boolean allowConflictPolicyChanges) { + super(field.getName(), allocator, callBack, conflictPolicy, allowConflictPolicyChanges); + this.field = field; this.valueCount = 0; } @@ -208,7 +229,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return new StructTransferPair(this, new NonNullableStructVector(name, allocator, - fieldType, + field.getFieldType(), callBack, getConflictPolicy(), allowConflictPolicyChanges), false); @@ -223,7 +244,25 @@ public TransferPair makeTransferPair(ValueVector to) { public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new StructTransferPair(this, new NonNullableStructVector(ref, allocator, - fieldType, + field.getFieldType(), + callBack, + getConflictPolicy(), + allowConflictPolicyChanges), false); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new StructTransferPair(this, new NonNullableStructVector(field, + allocator, + callBack, + getConflictPolicy(), + allowConflictPolicyChanges), false); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return new StructTransferPair(this, new NonNullableStructVector(field, + allocator, callBack, getConflictPolicy(), allowConflictPolicyChanges), false); @@ -404,7 +443,11 @@ public Field getField() { for (ValueVector child : getChildren()) { children.add(child.getField()); } - return new Field(name, fieldType, children); + if (children.isEmpty() || field.getChildren().equals(children)) { + return field; + } + field = new Field(field.getName(), field.getFieldType(), children); + return field; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java index 2dabc6e014969..d947249fd3cdd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java @@ -42,6 +42,7 @@ import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.OversizedAllocationException; @@ -113,6 +114,44 @@ public StructVector(String name, BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); } + /** + * Constructs a new instance. + * + * @param field The field materialized by this vector. + * @param allocator The allocator to use to allocating/reallocating buffers. + * @param callBack A schema change callback. + */ + public StructVector(Field field, + BufferAllocator allocator, + CallBack callBack) { + super(field, + checkNotNull(allocator), + callBack); + this.validityBuffer = allocator.getEmpty(); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + } + + /** + * Constructs a new instance. + * + * @param field The field materialized by this vector. + * @param allocator The allocator to use to allocating/reallocating buffers. + * @param callBack A schema change callback. + * @param conflictPolicy policy to determine how duplicate names are handled. + * @param allowConflictPolicyChanges wether duplicate names are allowed at all. + */ + public StructVector(Field field, + BufferAllocator allocator, + CallBack callBack, + ConflictPolicy conflictPolicy, + boolean allowConflictPolicyChanges) { + super(field, checkNotNull(allocator), callBack, conflictPolicy, allowConflictPolicyChanges); + this.validityBuffer = allocator.getEmpty(); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + } + @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { if (ownBuffers.size() != 1) { @@ -167,7 +206,7 @@ public NullableStructWriter getWriter() { public TransferPair getTransferPair(BufferAllocator allocator) { return new NullableStructTransferPair(this, new StructVector(name, allocator, - fieldType, + field.getFieldType(), null, getConflictPolicy(), allowConflictPolicyChanges), false); @@ -182,7 +221,7 @@ public TransferPair makeTransferPair(ValueVector to) { public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new NullableStructTransferPair(this, new StructVector(ref, allocator, - fieldType, + field.getFieldType(), null, getConflictPolicy(), allowConflictPolicyChanges), false); @@ -192,12 +231,21 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return new NullableStructTransferPair(this, new StructVector(ref, allocator, - fieldType, + field.getFieldType(), callBack, getConflictPolicy(), allowConflictPolicyChanges), false); } + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new NullableStructTransferPair(this, new StructVector(field, + allocator, + null, + getConflictPolicy(), + allowConflictPolicyChanges), false); + } + /** * {@link TransferPair} for this (nullable) {@link StructVector}. */ diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index c1d60da4d5988..baffcd12a3c84 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import java.util.ArrayList; @@ -972,6 +973,28 @@ public void testIsEmpty() { } } + @Test + public void testGetTransferPairWithField() throws Exception { + try (final LargeListVector fromVector = LargeListVector.empty("list", allocator)) { + + UnionLargeListWriter writer = fromVector.getWriter(); + writer.allocate(); + + //set some values + writer.startList(); + writer.integer().writeInt(1); + writer.integer().writeInt(2); + writer.endList(); + fromVector.setValueCount(2); + + final TransferPair transferPair = fromVector.getTransferPair(fromVector.getField(), + allocator); + final LargeListVector toVector = (LargeListVector) transferPair.getTo(); + // Field inside a new vector created by reusing a field should be the same in memory as the original field. + assertSame(toVector.getField(), fromVector.getField()); + } + } + private void writeIntValues(UnionLargeListWriter writer, int[] values) { writer.startList(); for (int v: values) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java index 644827ce995e8..ce7bb15bb1482 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java @@ -18,12 +18,14 @@ package org.apache.arrow.vector; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.holders.NullableLargeVarBinaryHolder; +import org.apache.arrow.vector.util.TransferPair; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -101,4 +103,20 @@ public void testSetNullableLargeVarBinaryHolderSafe() { buf.close(); } } + + @Test + public void testGetTransferPairWithField() { + try (BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 1000000, 1000000); + LargeVarBinaryVector v1 = new LargeVarBinaryVector("v1", childAllocator1)) { + v1.allocateNew(); + v1.setSafe(4094, "hello world".getBytes(), 0, 11); + v1.setValueCount(4001); + + TransferPair tp = v1.getTransferPair(v1.getField(), allocator); + tp.transfer(); + LargeVarBinaryVector v2 = (LargeVarBinaryVector) tp.getTo(); + assertSame(v1.getField(), v2.getField()); + v2.clear(); + } + } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java index 1b81c6b209fbb..5f7863c6f6177 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import java.nio.charset.StandardCharsets; @@ -794,6 +795,22 @@ public void testNullableType() { } } + @Test + public void testGetTransferPairWithField() { + try (BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 1000000, 1000000); + LargeVarCharVector v1 = new LargeVarCharVector("v1", childAllocator1)) { + v1.allocateNew(); + v1.setSafe(4094, "hello world".getBytes(), 0, 11); + v1.setValueCount(4001); + + TransferPair tp = v1.getTransferPair(v1.getField(), allocator); + tp.transfer(); + LargeVarCharVector v2 = (LargeVarCharVector) tp.getTo(); + assertSame(v1.getField(), v2.getField()); + v2.clear(); + } + } + private void populateLargeVarcharVector(final LargeVarCharVector vector, int valueCount, String[] values) { for (int i = 0; i < valueCount; i += 3) { final String s = String.format("%010d", i); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index f0f19058eef20..84149ffbf97f4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -1115,6 +1116,26 @@ public void testIsEmpty() { } } + @Test + public void testGetTransferPairWithField() { + try (ListVector fromVector = ListVector.empty("input", allocator)) { + UnionListWriter writer = fromVector.getWriter(); + writer.allocate(); + writer.setPosition(0); // optional + writer.startList(); + writer.bigInt().writeBigInt(1); + writer.bigInt().writeBigInt(2); + writer.bigInt().writeBigInt(3); + writer.endList(); + writer.setValueCount(1); + final TransferPair transferPair = fromVector.getTransferPair(fromVector.getField(), + allocator); + final ListVector toVector = (ListVector) transferPair.getTo(); + // Field inside a new vector created by reusing a field should be the same in memory as the original field. + assertSame(toVector.getField(), fromVector.getField()); + } + } + private void writeIntValues(UnionListWriter writer, int[] values) { writer.startList(); for (int v: values) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index d60d5611a5f7b..5c8fd55ec98dc 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -1133,4 +1133,26 @@ public void testGetTransferPair() { vector.clear(); } } + + @Test + public void testGetTransferPairWithField() { + try (MapVector mapVector = MapVector.empty("mapVector", allocator, false)) { + + FieldType type = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); + AddOrGetResult addResult = mapVector.addOrGetVector(type); + FieldType keyType = new FieldType(false, MinorType.BIGINT.getType(), null, null); + FieldType valueType = FieldType.nullable(MinorType.FLOAT8.getType()); + addResult.getVector().addOrGet(MapVector.KEY_NAME, keyType, BigIntVector.class); + addResult.getVector().addOrGet(MapVector.VALUE_NAME, valueType, Float8Vector.class); + mapVector.allocateNew(); + mapVector.setValueCount(0); + + assertEquals(-1, mapVector.getLastSet()); + TransferPair tp = mapVector.getTransferPair(mapVector.getField(), allocator); + tp.transfer(); + MapVector toVector = (MapVector) tp.getTo(); + assertSame(toVector.getField(), mapVector.getField()); + toVector.clear(); + } + } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index b4c30480000c8..b51c96946a7ae 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -29,11 +29,16 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; +import org.apache.arrow.vector.complex.impl.NullableStructWriter; +import org.apache.arrow.vector.complex.writer.Float8Writer; +import org.apache.arrow.vector.complex.writer.IntWriter; import org.apache.arrow.vector.holders.ComplexHolder; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -282,4 +287,42 @@ public void testAddChildVectorsWithDuplicatedFieldNamesForConflictPolicyReplace( } } + @Test + public void testGetTransferPair() { + try (final StructVector fromVector = simpleStructVector("s1", allocator)) { + TransferPair tp = fromVector.getTransferPair(fromVector.getField(), allocator); + final StructVector toVector = (StructVector) tp.getTo(); + // Field inside a new vector created by reusing a field should be the same in memory as the original field. + assertSame(toVector.getField(), fromVector.getField()); + toVector.clear(); + } + } + + private StructVector simpleStructVector(String name, BufferAllocator allocator) { + final String INT_COL = "struct_int_child"; + final String FLT_COL = "struct_flt_child"; + StructVector structVector = StructVector.empty(name, allocator); + final int size = 6; // number of structs + + NullableStructWriter structWriter = structVector.getWriter(); + structVector.addOrGet( + INT_COL, FieldType.nullable(Types.MinorType.INT.getType()), IntVector.class); + structVector.addOrGet( + FLT_COL, FieldType.nullable(Types.MinorType.INT.getType()), IntVector.class); + structVector.allocateNew(); + IntWriter intWriter = structWriter.integer(INT_COL); + Float8Writer float8Writer = structWriter.float8(FLT_COL); + + for (int i = 0; i < size; i++) { + structWriter.setPosition(i); + structWriter.start(); + intWriter.writeInt(i); + float8Writer.writeFloat8(i * .1); + structWriter.end(); + } + + structWriter.setValueCount(size); + + return structVector; + } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java index 7a445750a63de..ce3fb2cdf0ea1 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java @@ -40,6 +40,7 @@ import org.junit.Test; public class TestVectorSchemaRoot { + private BufferAllocator allocator; @Before