From 4968625c72d5d5be7b1a2d332e79da588b65f8fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Wed, 23 Aug 2023 09:31:43 +0100 Subject: [PATCH 01/11] Add Duration type definition --- js/src/enum.ts | 7 ++++++- js/src/type.ts | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/js/src/enum.ts b/js/src/enum.ts index f5856bc06afbe..4e207dd37cec1 100644 --- a/js/src/enum.ts +++ b/js/src/enum.ts @@ -137,7 +137,7 @@ export enum MessageHeader { * nested type consisting of other data types, or another data type (e.g. a * timestamp encoded as an int64). * - * **Note**: Only enum values 0-17 (NONE through Map) are written to an Arrow + * **Note**: Only enum values 0-18 (NONE through Duration) are written to an Arrow * IPC payload. * * The rest of the values are specified here so TypeScript can narrow the type @@ -174,6 +174,7 @@ export enum Type { FixedSizeBinary = 15, /** Fixed-size binary. Each value occupies the same number of bytes */ FixedSizeList = 16, /** Fixed-size list. Each value occupies the same number of bytes */ Map = 17, /** Map of named logical types */ + Duration = 18, /** Measure of elapsed time in either seconds, miliseconds, microseconds or nanoseconds. */ Dictionary = -1, /** Dictionary aka Category type */ Int8 = -2, @@ -201,6 +202,10 @@ export enum Type { SparseUnion = -24, IntervalDayTime = -25, IntervalYearMonth = -26, + DurationSecond = -27, + DurationMillisecond = -28, + DurationMicrosecond = -29, + DurationNanosecond = -30 } export enum BufferType { diff --git a/js/src/type.ts b/js/src/type.ts index 1dc90c47cbd10..34bbf45bca728 100644 --- a/js/src/type.ts +++ b/js/src/type.ts @@ -63,6 +63,7 @@ export abstract class DataType { construct /** @ignore */ export class IntervalYearMonth extends Interval_ { constructor() { super(IntervalUnit.YEAR_MONTH); } } +/** @ignore */ +type Durations = Type.Duration | Type.DurationSecond | Type.DurationMillisecond | Type.DurationMicrosecond | Type.DurationNanosecond; +/** @ignore */ +export interface Duration extends DataType { + TArray: BigInt64Array; + TValue: bigint; + ArrayType: BigInt64Array; +} + +/** @ignore */ +export class Duration extends DataType { + constructor(public readonly unit: TimeUnit) { + super(); + } + public get typeId() { return Type.Duration as T; } + public toString() { return `Duration<${TimeUnit[this.unit]}>`; } + protected static [Symbol.toStringTag] = ((proto: Duration) => { + (proto).unit = null; + (proto).ArrayType = BigInt64Array; + return proto[Symbol.toStringTag] = 'Duration'; + })(Duration.prototype); +} + +/** @ignore */ +export class DurationSecond extends Duration { constructor() { super(TimeUnit.SECOND); }} +/** @ignore */ +export class DurationMillisecond extends Duration { constructor() { super(TimeUnit.MILLISECOND); }} +/** @ignore */ +export class DurationMicrosecond extends Duration { constructor() { super(TimeUnit.MICROSECOND); }} +/** @ignore */ +export class DurationNanosecond extends Duration { constructor() { super(TimeUnit.NANOSECOND); }} + + /** @ignore */ export interface List extends DataType { TArray: Array; From 5bb7a797ee85e5d5f8c04c39b2d50c31f0876d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Wed, 23 Aug 2023 13:52:18 +0100 Subject: [PATCH 02/11] Update builders and visitors with Duration --- js/src/Arrow.dom.ts | 2 ++ js/src/Arrow.ts | 2 ++ js/src/builder.ts | 6 ++-- js/src/builder/duration.ts | 46 +++++++++++++++++++++++++++ js/src/data.ts | 11 +++++++ js/src/interfaces.ts | 16 ++++++++++ js/src/visitor.ts | 27 +++++++++++++++- js/src/visitor/builderctor.ts | 6 ++++ js/src/visitor/bytelength.ts | 5 ++- js/src/visitor/get.ts | 30 +++++++++++++++++ js/src/visitor/indexof.ts | 11 +++++++ js/src/visitor/iterator.ts | 11 +++++++ js/src/visitor/jsontypeassembler.ts | 3 ++ js/src/visitor/jsonvectorassembler.ts | 6 +++- js/src/visitor/set.ts | 31 ++++++++++++++++++ js/src/visitor/typeassembler.ts | 6 ++++ js/src/visitor/typecomparator.ts | 18 +++++++++++ js/src/visitor/typector.ts | 5 +++ js/src/visitor/vectorassembler.ts | 6 ++-- js/src/visitor/vectorloader.ts | 3 ++ 20 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 js/src/builder/duration.ts diff --git a/js/src/Arrow.dom.ts b/js/src/Arrow.dom.ts index 2fdef60c1fb55..451bf6acb6186 100644 --- a/js/src/Arrow.dom.ts +++ b/js/src/Arrow.dom.ts @@ -59,6 +59,7 @@ export { Union, DenseUnion, SparseUnion, Dictionary, Interval, IntervalDayTime, IntervalYearMonth, + Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond, FixedSizeList, Map_, MapRow, Table, makeTable, tableFromArrays, @@ -86,6 +87,7 @@ export { FixedSizeListBuilder, FloatBuilder, Float16Builder, Float32Builder, Float64Builder, IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder, + DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder, IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder, ListBuilder, MapBuilder, diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts index 4a6394c266b1b..714861e764ccb 100644 --- a/js/src/Arrow.ts +++ b/js/src/Arrow.ts @@ -48,6 +48,7 @@ export { Union, DenseUnion, SparseUnion, Dictionary, Interval, IntervalDayTime, IntervalYearMonth, + Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond, FixedSizeList, Map_ } from './type.js'; @@ -75,6 +76,7 @@ export { IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint export { TimeBuilder, TimeSecondBuilder, TimeMillisecondBuilder, TimeMicrosecondBuilder, TimeNanosecondBuilder } from './builder/time.js'; export { TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder } from './builder/timestamp.js'; export { IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder } from './builder/interval.js'; +export { DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder } from './builder/duration.js'; export { Utf8Builder } from './builder/utf8.js'; export { BinaryBuilder } from './builder/binary.js'; export { ListBuilder } from './builder/list.js'; diff --git a/js/src/builder.ts b/js/src/builder.ts index 90fe3ddcc9477..93510eedf84ff 100644 --- a/js/src/builder.ts +++ b/js/src/builder.ts @@ -21,7 +21,7 @@ import { MapRow, kKeys } from './row/map.js'; import { DataType, strideForType, Float, Int, Decimal, FixedSizeBinary, - Date_, Time, Timestamp, Interval, + Date_, Time, Timestamp, Interval, Duration, Utf8, Binary, List, Map_, } from './type.js'; import { createIsValidFunction } from './builder/valid.js'; @@ -290,7 +290,7 @@ export abstract class Builder { } else if (valueOffsets = _offsets?.flush(length)) { // Variable-width primitives (Binary, Utf8), and Lists // Binary, Utf8 data = _values?.flush(_offsets.last()); - } else { // Fixed-width primitives (Int, Float, Decimal, Time, Timestamp, and Interval) + } else { // Fixed-width primitives (Int, Float, Decimal, Time, Timestamp, Duration and Interval) data = _values?.flush(length); } @@ -342,7 +342,7 @@ export abstract class Builder { (Builder.prototype as any)._isValid = () => true; /** @ignore */ -export abstract class FixedWidthBuilder extends Builder { +export abstract class FixedWidthBuilder extends Builder { constructor(opts: BuilderOptions) { super(opts); this._values = new DataBufferBuilder(new this.ArrayType(0), this.stride); diff --git a/js/src/builder/duration.ts b/js/src/builder/duration.ts new file mode 100644 index 0000000000000..3af770284cd4f --- /dev/null +++ b/js/src/builder/duration.ts @@ -0,0 +1,46 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { FixedWidthBuilder } from '../builder.js'; +import { Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond } from '../type.js'; +import { setDuration, setDurationSecond, setDurationMillisecond, setDurationMicrosecond, setDurationNanosecond } from '../visitor/set.js'; + +/** @ignore */ +export class DurationBuilder extends FixedWidthBuilder { } + +(DurationBuilder.prototype as any)._setValue = setDuration; + +/** @ignore */ +export class DurationSecondBuilder extends DurationBuilder { } + +(DurationSecond.prototype as any)._setValue = setDurationSecond; + +/** @ignore */ +export class DurationMillisecondBuilder extends DurationBuilder { } + +(DurationMillisecond.prototype as any)._setValue = setDurationMillisecond; + +/** @ignore */ +export class DurationMicrosecondBuilder extends DurationBuilder { } + +(DurationMicrosecond.prototype as any)._setValue = setDurationMicrosecond; + +/** @ignore */ +export class DurationNanosecondBuilder extends DurationBuilder { } + +(DurationNanosecond.prototype as any)._setValue = setDurationNanosecond; diff --git a/js/src/data.ts b/js/src/data.ts index dc423cdb01e1c..1e9df71cff8a7 100644 --- a/js/src/data.ts +++ b/js/src/data.ts @@ -257,6 +257,7 @@ import { Int, Date_, Interval, + Duration, Time, Timestamp, Union, DenseUnion, SparseUnion, @@ -390,6 +391,13 @@ class MakeDataVisitor extends Visitor { const { ['length']: length = data.length / strideForType(type), ['nullCount']: nullCount = props['nullBitmap'] ? -1 : 0, } = props; return new Data(type, offset, length, nullCount, [undefined, data, nullBitmap]); } + public visitDuration(props: DurationDataProps) { + const { ['type']: type, ['offset']: offset = 0 } = props; + const nullBitmap = toUint8Array(props['nullBitmap']); + const data = toArrayBufferView(type.ArrayType, props['data']); + const { ['length']: length = data.length, ['nullCount']: nullCount = props['nullBitmap'] ? -1 : 0, } = props; + return new Data(type, offset, length, nullCount, [undefined, data, nullBitmap]); + } public visitFixedSizeList(props: FixedSizeListDataProps) { const { ['type']: type, ['offset']: offset = 0, ['child']: child = new MakeDataVisitor().visit({ type: type.valueType }) } = props; const nullBitmap = toUint8Array(props['nullBitmap']); @@ -424,6 +432,7 @@ interface Date_DataProps extends DataProps_ { data?: DataBuf interface TimeDataProps extends DataProps_ { data?: DataBuffer } interface TimestampDataProps extends DataProps_ { data?: DataBuffer } interface IntervalDataProps extends DataProps_ { data?: DataBuffer } +interface DurationDataProps extends DataProps_ { data?: DataBuffer } interface FixedSizeBinaryDataProps extends DataProps_ { data?: DataBuffer } interface BinaryDataProps extends DataProps_ { valueOffsets: ValueOffsetsBuffer; data?: DataBuffer } interface Utf8DataProps extends DataProps_ { valueOffsets: ValueOffsetsBuffer; data?: DataBuffer } @@ -446,6 +455,7 @@ export type DataProps = ( T extends Time /* */ ? TimeDataProps : T extends Timestamp /* */ ? TimestampDataProps : T extends Interval /* */ ? IntervalDataProps : + T extends Duration /* */ ? DurationDataProps : T extends FixedSizeBinary /* */ ? FixedSizeBinaryDataProps : T extends Binary /* */ ? BinaryDataProps : T extends Utf8 /* */ ? Utf8DataProps : @@ -471,6 +481,7 @@ export function makeData(props: Date_DataProps): Data; export function makeData(props: TimeDataProps): Data; export function makeData(props: TimestampDataProps): Data; export function makeData(props: IntervalDataProps): Data; +export function makeData(props: DurationDataProps): Data; export function makeData(props: FixedSizeBinaryDataProps): Data; export function makeData(props: BinaryDataProps): Data; export function makeData(props: Utf8DataProps): Data; diff --git a/js/src/interfaces.ts b/js/src/interfaces.ts index 8d61295919046..95c5adbb2a25e 100644 --- a/js/src/interfaces.ts +++ b/js/src/interfaces.ts @@ -31,6 +31,7 @@ import type { IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, import type { TimeBuilder, TimeSecondBuilder, TimeMillisecondBuilder, TimeMicrosecondBuilder, TimeNanosecondBuilder } from './builder/time.js'; import type { TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder } from './builder/timestamp.js'; import type { IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder } from './builder/interval.js'; +import type { DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder } from './builder/duration.js'; import type { Utf8Builder } from './builder/utf8.js'; import type { BinaryBuilder } from './builder/binary.js'; import type { ListBuilder } from './builder/list.js'; @@ -222,6 +223,11 @@ export type TypeToDataType = { [Type.Interval]: type.Interval; [Type.IntervalDayTime]: type.IntervalDayTime; [Type.IntervalYearMonth]: type.IntervalYearMonth; + [Type.Duration]: type.Duration; + [Type.DurationSecond]: type.DurationSecond; + [Type.DurationMillisecond]: type.DurationMillisecond; + [Type.DurationMicrosecond]: type.DurationMicrosecond; + [Type.DurationNanosecond]: type.DurationNanosecond; [Type.Map]: type.Map_; [Type.List]: type.List; [Type.Struct]: type.Struct; @@ -270,6 +276,11 @@ type TypeToBuilder = { [Type.Interval]: IntervalBuilder; [Type.IntervalDayTime]: IntervalDayTimeBuilder; [Type.IntervalYearMonth]: IntervalYearMonthBuilder; + [Type.Duration]: DurationBuilder; + [Type.DurationSecond]: DurationBuilder; + [Type.DurationMillisecond]: DurationMillisecondBuilder; + [Type.DurationMicrosecond]: DurationMicrosecondBuilder; + [Type.DurationNanosecond]: DurationNanosecondBuilder; [Type.Map]: MapBuilder; [Type.List]: ListBuilder; [Type.Struct]: StructBuilder; @@ -318,6 +329,11 @@ type DataTypeToBuilder = { [Type.Interval]: T extends type.Interval ? IntervalBuilder : never; [Type.IntervalDayTime]: T extends type.IntervalDayTime ? IntervalDayTimeBuilder : never; [Type.IntervalYearMonth]: T extends type.IntervalYearMonth ? IntervalYearMonthBuilder : never; + [Type.Duration]: T extends type.Duration ? DurationBuilder: never; + [Type.DurationSecond]: T extends type.DurationSecond ? DurationSecondBuilder : never; + [Type.DurationMillisecond]: T extends type.DurationMillisecond ? DurationMillisecondBuilder : never; + [Type.DurationMicrosecond]: T extends type.DurationMicrosecond ? DurationMicrosecondBuilder: never; + [Type.DurationNanosecond]: T extends type.DurationNanosecond ? DurationNanosecondBuilder: never; [Type.Map]: T extends type.Map_ ? MapBuilder : never; [Type.List]: T extends type.List ? ListBuilder : never; [Type.Struct]: T extends type.Struct ? StructBuilder : never; diff --git a/js/src/visitor.ts b/js/src/visitor.ts index 3be50a6d3eacf..c63640b038e47 100644 --- a/js/src/visitor.ts +++ b/js/src/visitor.ts @@ -16,7 +16,7 @@ // under the License. import { Type, Precision, DateUnit, TimeUnit, IntervalUnit, UnionMode } from './enum.js'; -import { DataType, Float, Int, Date_, Interval, Time, Timestamp, Union, } from './type.js'; +import { DataType, Float, Int, Date_, Interval, Time, Timestamp, Union, Duration } from './type.js'; export abstract class Visitor { public visitMany(nodes: any[], ...args: any[][]) { @@ -47,6 +47,7 @@ export abstract class Visitor { public visitUnion(_node: any, ..._args: any[]): any { return null; } public visitDictionary(_node: any, ..._args: any[]): any { return null; } public visitInterval(_node: any, ..._args: any[]): any { return null; } + public visitDuration(_node: any, ... _args: any[]): any { return null; } public visitFixedSizeList(_node: any, ..._args: any[]): any { return null; } public visitMap(_node: any, ..._args: any[]): any { return null; } } @@ -113,6 +114,11 @@ function getVisitFnByTypeId(visitor: Visitor, dtype: Type, throwIfNotFound = tru case Type.Interval: fn = visitor.visitInterval; break; case Type.IntervalDayTime: fn = visitor.visitIntervalDayTime || visitor.visitInterval; break; case Type.IntervalYearMonth: fn = visitor.visitIntervalYearMonth || visitor.visitInterval; break; + case Type.Duration: fn = visitor.visitDuration; break; + case Type.DurationSecond: fn = visitor.visitDurationSecond || visitor.visitDuration; break; + case Type.DurationMillisecond: fn = visitor.visitDurationMillisecond || visitor.visitDuration; break; + case Type.DurationMicrosecond: fn = visitor.visitDurationMicrosecond || visitor.visitDuration; break; + case Type.DurationNanosecond: fn = visitor.visitDurationNanosecond || visitor.visitDuration; break; case Type.FixedSizeList: fn = visitor.visitFixedSizeList; break; case Type.Map: fn = visitor.visitMap; break; } @@ -180,6 +186,15 @@ function inferDType(type: T): Type { } // @ts-ignore return Type.Interval; + case Type.Duration: + switch ((type as any as Duration).unit) { + case TimeUnit.SECOND: return Type.DurationSecond; + case TimeUnit.MILLISECOND: return Type.DurationMillisecond; + case TimeUnit.MICROSECOND: return Type.DurationMicrosecond; + case TimeUnit.NANOSECOND: return Type.DurationNanosecond; + } + // @ts-ignore + return Type.Duration; case Type.Map: return Type.Map; case Type.List: return Type.List; case Type.Struct: return Type.Struct; @@ -239,6 +254,11 @@ export interface Visitor { visitInterval(node: any, ...args: any[]): any; visitIntervalDayTime?(node: any, ...args: any[]): any; visitIntervalYearMonth?(node: any, ...args: any[]): any; + visitDuration(node: any, ...args: any[]): any; + visitDurationSecond(node: any, ...args: any[]): any; + visitDurationMillisecond(node: any, ...args: any[]): any; + visitDurationMicrosecond(node: any, ...args: any[]): any; + visitDurationNanosecond(node: any, ...args: any[]): any; visitFixedSizeList(node: any, ...args: any[]): any; visitMap(node: any, ...args: any[]): any; } @@ -270,3 +290,8 @@ export interface Visitor { (Visitor.prototype as any).visitSparseUnion = null; (Visitor.prototype as any).visitIntervalDayTime = null; (Visitor.prototype as any).visitIntervalYearMonth = null; +(Visitor.prototype as any).visitDuration = null; +(Visitor.prototype as any).visitDurationSecond = null; +(Visitor.prototype as any).visitDurationMillisecond = null; +(Visitor.prototype as any).visitDurationMicrosecond = null; +(Visitor.prototype as any).visitDurationNanosecond = null; diff --git a/js/src/visitor/builderctor.ts b/js/src/visitor/builderctor.ts index 9ce9ae4d4a797..2d20f2a8efd5c 100644 --- a/js/src/visitor/builderctor.ts +++ b/js/src/visitor/builderctor.ts @@ -30,6 +30,7 @@ import { FixedSizeBinaryBuilder } from '../builder/fixedsizebinary.js'; import { FixedSizeListBuilder } from '../builder/fixedsizelist.js'; import { FloatBuilder, Float16Builder, Float32Builder, Float64Builder } from '../builder/float.js'; import { IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder } from '../builder/interval.js'; +import { DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder } from '../builder/duration.js'; import { IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder } from '../builder/int.js'; import { ListBuilder } from '../builder/list.js'; import { MapBuilder } from '../builder/map.js'; @@ -91,6 +92,11 @@ export class GetBuilderCtor extends Visitor { public visitInterval() { return IntervalBuilder; } public visitIntervalDayTime() { return IntervalDayTimeBuilder; } public visitIntervalYearMonth() { return IntervalYearMonthBuilder; } + public visitDuration() { return DurationBuilder; } + public visitDurationSecond() { return DurationSecondBuilder; } + public visitDurationMillisecond() { return DurationMillisecondBuilder; } + public visitDurationMicrosecond() { return DurationMicrosecondBuilder; } + public visistDurationNanosecond() { return DurationNanosecondBuilder; } public visitFixedSizeList() { return FixedSizeListBuilder; } public visitMap() { return MapBuilder; } } diff --git a/js/src/visitor/bytelength.ts b/js/src/visitor/bytelength.ts index 862808ad54ee9..72d6148a52fd8 100644 --- a/js/src/visitor/bytelength.ts +++ b/js/src/visitor/bytelength.ts @@ -25,7 +25,7 @@ import { TypeToDataType } from '../interfaces.js'; import { Type, TimeUnit, UnionMode } from '../enum.js'; import { DataType, Dictionary, - Float, Int, Date_, Interval, Time, Timestamp, + Float, Int, Date_, Interval, Time, Timestamp, Duration, Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, Union, DenseUnion, SparseUnion, } from '../type.js'; @@ -75,6 +75,9 @@ export class GetByteLengthVisitor extends Visitor { public visitInterval(data: Data, _: number) { return (data.type.unit + 1) * 4; } + public visitDuration(____: Data, _: number) { + return 8; + } public visitStruct(data: Data, i: number) { return data.children.reduce((total, child) => total + instance.visit(child, i), 0); } diff --git a/js/src/visitor/get.ts b/js/src/visitor/get.ts index 12f8325470bac..5aaaedf51a37e 100644 --- a/js/src/visitor/get.ts +++ b/js/src/visitor/get.ts @@ -34,6 +34,7 @@ import { Interval, IntervalDayTime, IntervalYearMonth, Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond, Union, DenseUnion, SparseUnion, } from '../type.js'; @@ -84,6 +85,11 @@ export interface GetVisitor extends Visitor { visitInterval(data: Data, index: number): T['TValue'] | null; visitIntervalDayTime(data: Data, index: number): T['TValue'] | null; visitIntervalYearMonth(data: Data, index: number): T['TValue'] | null; + visitDuration(data: Data, index: number): T['TValue'] | null; + visitDurationSecond(data: Data, index: number): T['TValue'] | null; + visitDurationMillisecond(data: Data, index: number): T['TValue'] | null; + visitDurationMicrosecond(data: Data, index: number): T['TValue'] | null; + visitDurationNanosecond(data: Data, index: number): T['TValue'] | null; visitFixedSizeList(data: Data, index: number): T['TValue'] | null; visitMap(data: Data, index: number): T['TValue'] | null; } @@ -279,6 +285,25 @@ const getIntervalYearMonth = ({ values }: Data, return int32s; }; +/** @ignore */ +const getDurationSecond = ({ values }: Data, index: number): T['TValue'] => values[index]; +/** @ignore */ +const getDurationMillisecond = ({ values }: Data, index: number): T['TValue'] => values[index]; +/** @ignore */ +const getDurationMicrosecond = ({ values }: Data, index: number): T['TValue'] => values[index]; +/** @ignore */ +const getDurationNanosecond = ({ values }: Data, index: number): T['TValue'] => values[index]; +/* istanbul ignore next */ +/** @ignore */ +const getDuration = (data: Data, index: number): T['TValue'] => { + switch (data.type.unit) { + case TimeUnit.SECOND: return getDurationSecond(data as Data, index); + case TimeUnit.MILLISECOND: return getDurationMillisecond(data as Data, index); + case TimeUnit.MICROSECOND: return getDurationMicrosecond(data as Data, index); + case TimeUnit.NANOSECOND: return getDurationNanosecond(data as Data, index); + } +}; + /** @ignore */ const getFixedSizeList = (data: Data, index: number): T['TValue'] => { const { stride, children } = data; @@ -328,6 +353,11 @@ GetVisitor.prototype.visitDictionary = wrapGet(getDictionary); GetVisitor.prototype.visitInterval = wrapGet(getInterval); GetVisitor.prototype.visitIntervalDayTime = wrapGet(getIntervalDayTime); GetVisitor.prototype.visitIntervalYearMonth = wrapGet(getIntervalYearMonth); +GetVisitor.prototype.visitDuration = wrapGet(getDuration); +GetVisitor.prototype.visitDurationSecond = wrapGet(getDurationSecond); +GetVisitor.prototype.visitDurationMillisecond = wrapGet(getDurationMillisecond); +GetVisitor.prototype.visitDurationMicrosecond = wrapGet(getDurationMicrosecond); +GetVisitor.prototype.visitDurationNanosecond = wrapGet(getDurationNanosecond); GetVisitor.prototype.visitFixedSizeList = wrapGet(getFixedSizeList); GetVisitor.prototype.visitMap = wrapGet(getMap); diff --git a/js/src/visitor/indexof.ts b/js/src/visitor/indexof.ts index 654134c6dff04..28dcff20d3bd3 100644 --- a/js/src/visitor/indexof.ts +++ b/js/src/visitor/indexof.ts @@ -31,6 +31,7 @@ import { Interval, IntervalDayTime, IntervalYearMonth, Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond, Union, DenseUnion, SparseUnion, } from '../type.js'; @@ -81,6 +82,11 @@ export interface IndexOfVisitor extends Visitor { visitInterval(data: Data, value: T['TValue'] | null, index?: number): number; visitIntervalDayTime(data: Data, value: T['TValue'] | null, index?: number): number; visitIntervalYearMonth(data: Data, value: T['TValue'] | null, index?: number): number; + visitDuration(data: Data, value: T['TValue'] | null, index?: number): number; + visitDurationSecond(data: Data, value: T['TValue'] | null, index?: number): number; + visitDurationMillisecond(data: Data, value: T['TValue'] | null, index?: number): number; + visitDurationMicrosecond(data: Data, value: T['TValue'] | null, index?: number): number; + visitDurationNanosecond(data: Data, value: T['TValue'] | null, index?: number): number; visitFixedSizeList(data: Data, value: T['TValue'] | null, index?: number): number; visitMap(data: Data, value: T['TValue'] | null, index?: number): number; } @@ -191,6 +197,11 @@ IndexOfVisitor.prototype.visitDictionary = indexOfValue; IndexOfVisitor.prototype.visitInterval = indexOfValue; IndexOfVisitor.prototype.visitIntervalDayTime = indexOfValue; IndexOfVisitor.prototype.visitIntervalYearMonth = indexOfValue; +IndexOfVisitor.prototype.visitDuration = indexOfValue; +IndexOfVisitor.prototype.visitDurationSecond = indexOfValue; +IndexOfVisitor.prototype.visitDurationMillisecond = indexOfValue; +IndexOfVisitor.prototype.visitDurationMicrosecond = indexOfValue; +IndexOfVisitor.prototype.visitDurationNanosecond = indexOfValue; IndexOfVisitor.prototype.visitFixedSizeList = indexOfValue; IndexOfVisitor.prototype.visitMap = indexOfValue; diff --git a/js/src/visitor/iterator.ts b/js/src/visitor/iterator.ts index 48021a78e86f6..e38bb907695d0 100644 --- a/js/src/visitor/iterator.ts +++ b/js/src/visitor/iterator.ts @@ -28,6 +28,7 @@ import { Interval, IntervalDayTime, IntervalYearMonth, Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond, Union, DenseUnion, SparseUnion, } from '../type.js'; import { ChunkedIterator } from '../util/chunk.js'; @@ -79,6 +80,11 @@ export interface IteratorVisitor extends Visitor { visitInterval(vector: Vector): IterableIterator; visitIntervalDayTime(vector: Vector): IterableIterator; visitIntervalYearMonth(vector: Vector): IterableIterator; + visitDuration(vector: Vector): IterableIterator; + visitDurationSecond(vector: Vector): IterableIterator; + visitDurationMillisecond(vector: Vector): IterableIterator; + visitDurationMicrosecond(vector: Vector): IterableIterator; + visitDurationNanosecond(vector: Vector): IterableIterator; visitFixedSizeList(vector: Vector): IterableIterator; visitMap(vector: Vector): IterableIterator; } @@ -177,6 +183,11 @@ IteratorVisitor.prototype.visitDictionary = vectorIterator; IteratorVisitor.prototype.visitInterval = vectorIterator; IteratorVisitor.prototype.visitIntervalDayTime = vectorIterator; IteratorVisitor.prototype.visitIntervalYearMonth = vectorIterator; +IteratorVisitor.prototype.visitDuration = vectorIterator; +IteratorVisitor.prototype.visitDurationSecond = vectorIterator; +IteratorVisitor.prototype.visitDurationMillisecond = vectorIterator; +IteratorVisitor.prototype.visitDurationMicrosecond = vectorIterator; +IteratorVisitor.prototype.visitDurationNanosecond = vectorIterator; IteratorVisitor.prototype.visitFixedSizeList = vectorIterator; IteratorVisitor.prototype.visitMap = vectorIterator; diff --git a/js/src/visitor/jsontypeassembler.ts b/js/src/visitor/jsontypeassembler.ts index d83edfc24fbd8..6e6cfb07413c3 100644 --- a/js/src/visitor/jsontypeassembler.ts +++ b/js/src/visitor/jsontypeassembler.ts @@ -63,6 +63,9 @@ export class JSONTypeAssembler extends Visitor { public visitInterval({ typeId, unit }: T) { return { 'name': ArrowType[typeId].toLowerCase(), 'unit': IntervalUnit[unit] }; } + public visitDuration({ typeId, unit }: T) { + return { 'name': ArrowType[typeId].toLocaleLowerCase(), 'unit': TimeUnit[unit]}; + } public visitList({ typeId }: T) { return { 'name': ArrowType[typeId].toLowerCase() }; } diff --git a/js/src/visitor/jsonvectorassembler.ts b/js/src/visitor/jsonvectorassembler.ts index 7a617f4afe2c4..55a6b4e2ea390 100644 --- a/js/src/visitor/jsonvectorassembler.ts +++ b/js/src/visitor/jsonvectorassembler.ts @@ -26,7 +26,7 @@ import { UnionMode, DateUnit, TimeUnit } from '../enum.js'; import { BitIterator, getBit, getBool } from '../util/bit.js'; import { DataType, - Float, Int, Date_, Interval, Time, Timestamp, Union, + Float, Int, Date_, Interval, Time, Timestamp, Union, Duration, Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, IntArray, } from '../type.js'; @@ -52,6 +52,7 @@ export interface JSONVectorAssembler extends Visitor { visitStruct(data: Data): { children: any[] }; visitUnion(data: Data): { children: any[]; TYPE_ID: number[] }; visitInterval(data: Data): { DATA: number[] }; + visitDuration(data: Data): { DATA: string[] }; visitFixedSizeList(data: Data): { children: any[] }; visitMap(data: Data): { children: any[] }; } @@ -146,6 +147,9 @@ export class JSONVectorAssembler extends Visitor { public visitInterval(data: Data) { return { 'DATA': [...data.values] }; } + public visitDuration(data: Data) { + return { 'DATA': [...bigNumsToStrings(data.values, 2)]}; + } public visitFixedSizeList(data: Data) { return { 'children': this.visitMany(data.type.children, data.children) diff --git a/js/src/visitor/set.ts b/js/src/visitor/set.ts index c2d4319911afe..1a0eddc556899 100644 --- a/js/src/visitor/set.ts +++ b/js/src/visitor/set.ts @@ -32,6 +32,7 @@ import { Interval, IntervalDayTime, IntervalYearMonth, Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond, Union, DenseUnion, SparseUnion, } from '../type.js'; @@ -82,6 +83,11 @@ export interface SetVisitor extends Visitor { visitInterval(data: Data, index: number, value: T['TValue']): void; visitIntervalDayTime(data: Data, index: number, value: T['TValue']): void; visitIntervalYearMonth(data: Data, index: number, value: T['TValue']): void; + visitDuration(data: Data, index: number, value: T['TValue']): void; + visitDurationSecond(data: Data, index: number, value: T['TValue']): void; + visitDurationMillisecond(data: Data, index: number, value: T['TValue']): void; + visitDurationMicrosecond(data: Data, index: number, value: T['TValue']): void; + visitDurationNanosecond(data: Data, index: number, value: T['TValue']): void; visitFixedSizeList(data: Data, index: number, value: T['TValue']): void; visitMap(data: Data, index: number, value: T['TValue']): void; } @@ -308,6 +314,26 @@ export const setIntervalDayTime = ({ values }: Data({ values }: Data, index: number, value: T['TValue']): void => { values[index] = (value[0] * 12) + (value[1] % 12); }; +/** @ignore */ +export const setDurationSecond = ({ values }: Data, index: number, value: T['TValue']): void => { values[index] = value; }; +/** @ignore */ +export const setDurationMillisecond = ({ values }: Data, index: number, value: T['TValue']): void => { values[index] = value; }; +/** @ignore */ +export const setDurationMicrosecond = ({ values }: Data, index: number, value: T['TValue']): void => { values[index] = value; }; +/** @ignore */ +export const setDurationNanosecond = ({ values }: Data, index: number, value: T['TValue']): void => { values[index] = value; }; +/* istanbul ignore next */ +/** @ignore */ +export const setDuration = (data: Data, index: number, value: T['TValue']): void => { + switch (data.type.unit) { + case TimeUnit.SECOND: return setDurationSecond(data as Data, index, value as DurationSecond['TValue']); + case TimeUnit.MILLISECOND: return setDurationMillisecond(data as Data, index, value as DurationMillisecond['TValue']); + case TimeUnit.MICROSECOND: return setDurationMicrosecond(data as Data, index, value as DurationMicrosecond['TValue']); + case TimeUnit.NANOSECOND: return setDurationNanosecond(data as Data, index, value as DurationNanosecond['TValue']); + } +}; + + /** @ignore */ const setFixedSizeList = (data: Data, index: number, value: T['TValue']): void => { const { stride } = data; @@ -364,6 +390,11 @@ SetVisitor.prototype.visitDictionary = wrapSet(setDictionary); SetVisitor.prototype.visitInterval = wrapSet(setIntervalValue); SetVisitor.prototype.visitIntervalDayTime = wrapSet(setIntervalDayTime); SetVisitor.prototype.visitIntervalYearMonth = wrapSet(setIntervalYearMonth); +SetVisitor.prototype.visitDuration = wrapSet(setDuration); +SetVisitor.prototype.visitDurationSecond = wrapSet(setDurationSecond); +SetVisitor.prototype.visitDurationMillisecond = wrapSet(setDurationMillisecond); +SetVisitor.prototype.visitDurationMicrosecond = wrapSet(setDurationMicrosecond); +SetVisitor.prototype.visitDurationNanosecond = wrapSet(setDurationNanosecond); SetVisitor.prototype.visitFixedSizeList = wrapSet(setFixedSizeList); SetVisitor.prototype.visitMap = wrapSet(setMap); diff --git a/js/src/visitor/typeassembler.ts b/js/src/visitor/typeassembler.ts index c84e3930f64f5..c2262d20531b9 100644 --- a/js/src/visitor/typeassembler.ts +++ b/js/src/visitor/typeassembler.ts @@ -32,6 +32,7 @@ import { Date } from '../fb/date.js'; import { Time } from '../fb/time.js'; import { Timestamp } from '../fb/timestamp.js'; import { Interval } from '../fb/interval.js'; +import { Duration } from '../fb/duration.js'; import { List } from '../fb/list.js'; import { Struct_ as Struct } from '../fb/struct-.js'; import { Union } from '../fb/union.js'; @@ -109,6 +110,11 @@ export class TypeAssembler extends Visitor { Interval.addUnit(b, node.unit); return Interval.endInterval(b); } + public visitDuration(node: T, b: Builder) { + Duration.startDuration(b); + Duration.addUnit(b, node.unit); + return Duration.endDuration(b); + } public visitList(_node: T, b: Builder) { List.startList(b); return List.endList(b); diff --git a/js/src/visitor/typecomparator.ts b/js/src/visitor/typecomparator.ts index a77c4020961ce..1de8e218dae4f 100644 --- a/js/src/visitor/typecomparator.ts +++ b/js/src/visitor/typecomparator.ts @@ -28,6 +28,7 @@ import { Interval, IntervalDayTime, IntervalYearMonth, Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond, Union, DenseUnion, SparseUnion, } from '../type.js'; @@ -77,6 +78,11 @@ export interface TypeComparator extends Visitor { visitInterval(type: T, other?: DataType | null): other is T; visitIntervalDayTime(type: T, other?: DataType | null): other is T; visitIntervalYearMonth(type: T, other?: DataType | null): other is T; + visitDuration(type: T, other?: DataType | null): other is T; + visitDurationSecond(type: T, other?: DataType | null): other is T; + visitDurationMillisecond(type: T, other?: DataType | null): other is T; + visitDurationMicrosecond(type: T, other?: DataType | null): other is T; + visitDurationNanosecond(type: T, other?: DataType | null): other is T; visitFixedSizeList(type: T, other?: DataType | null): other is T; visitMap(type: T, other?: DataType | null): other is T; } @@ -202,6 +208,13 @@ function compareInterval(type: T, other?: DataType | null): ); } +function compareDuration(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.unit === other.unit + ); +} + function compareFixedSizeList(type: T, other?: DataType | null): other is T { return (type === other) || ( compareConstructor(type, other) && @@ -261,6 +274,11 @@ TypeComparator.prototype.visitDictionary = compareDictionary; TypeComparator.prototype.visitInterval = compareInterval; TypeComparator.prototype.visitIntervalDayTime = compareInterval; TypeComparator.prototype.visitIntervalYearMonth = compareInterval; +TypeComparator.prototype.visitDuration = compareDuration; +TypeComparator.prototype.visitDurationSecond = compareDuration; +TypeComparator.prototype.visitDurationMillisecond = compareDuration; +TypeComparator.prototype.visitDurationMicrosecond = compareDuration; +TypeComparator.prototype.visitDurationNanosecond = compareDuration; TypeComparator.prototype.visitFixedSizeList = compareFixedSizeList; TypeComparator.prototype.visitMap = compareMap; diff --git a/js/src/visitor/typector.ts b/js/src/visitor/typector.ts index c825a61dbadfb..077f66592fbfb 100644 --- a/js/src/visitor/typector.ts +++ b/js/src/visitor/typector.ts @@ -74,6 +74,11 @@ export class GetDataTypeConstructor extends Visitor { public visitInterval() { return type.Interval; } public visitIntervalDayTime() { return type.IntervalDayTime; } public visitIntervalYearMonth() { return type.IntervalYearMonth; } + public visitDuration() { return type.Duration; } + public visitDurationSecond() { return type.DurationSecond; } + public visitDurationMillisecond() { return type.DurationMillisecond; } + public visitDurationMicrosecond() { return type.DurationMicrosecond; } + public visitDurationNanosecond() { return type.DurationNanosecond; } public visitFixedSizeList() { return type.FixedSizeList; } public visitMap() { return type.Map_; } } diff --git a/js/src/visitor/vectorassembler.ts b/js/src/visitor/vectorassembler.ts index dbf778c4c3631..949463272e718 100644 --- a/js/src/visitor/vectorassembler.ts +++ b/js/src/visitor/vectorassembler.ts @@ -26,7 +26,7 @@ import { packBools, truncateBitmap } from '../util/bit.js'; import { BufferRegion, FieldNode } from '../ipc/metadata/message.js'; import { DataType, Dictionary, - Float, Int, Date_, Interval, Time, Timestamp, Union, + Float, Int, Date_, Interval, Time, Timestamp, Union, Duration, Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, } from '../type.js'; @@ -51,6 +51,7 @@ export interface VectorAssembler extends Visitor { visitStruct(data: Data): this; visitUnion(data: Data): this; visitInterval(data: Data): this; + visitDuration(data: Data): this; visitFixedSizeList(data: Data): this; visitMap(data: Data): this; } @@ -195,7 +196,7 @@ function assembleBoolVector(this: VectorAssembler, data: Data } /** @ignore */ -function assembleFlatVector(this: VectorAssembler, data: Data) { +function assembleFlatVector(this: VectorAssembler, data: Data) { return addBuffer.call(this, data.values.subarray(0, data.length * data.stride)); } @@ -243,5 +244,6 @@ VectorAssembler.prototype.visitList = assembleListVector; VectorAssembler.prototype.visitStruct = assembleNestedVector; VectorAssembler.prototype.visitUnion = assembleUnion; VectorAssembler.prototype.visitInterval = assembleFlatVector; +VectorAssembler.prototype.visitDuration = assembleFlatVector; VectorAssembler.prototype.visitFixedSizeList = assembleListVector; VectorAssembler.prototype.visitMap = assembleListVector; diff --git a/js/src/visitor/vectorloader.ts b/js/src/visitor/vectorloader.ts index cb4bc2829274f..6fbe803e66cf8 100644 --- a/js/src/visitor/vectorloader.ts +++ b/js/src/visitor/vectorloader.ts @@ -115,6 +115,9 @@ export class VectorLoader extends Visitor { public visitInterval(type: T, { length, nullCount } = this.nextFieldNode()) { return makeData({ type, length, nullCount, nullBitmap: this.readNullBitmap(type, nullCount), data: this.readData(type) }); } + public visitDuration(type: T, { length, nullCount } = this.nextFieldNode()) { + return makeData({ type, length, nullCount, nullBitmap: this.readNullBitmap(type, nullCount), data: this.readData(type) }); + } public visitFixedSizeList(type: T, { length, nullCount } = this.nextFieldNode()) { return makeData({ type, length, nullCount, nullBitmap: this.readNullBitmap(type, nullCount), 'child': this.visit(type.children[0]) }); } From 8752659f985f45528f3908e048bd8e25c82bc14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Wed, 23 Aug 2023 15:27:20 +0100 Subject: [PATCH 03/11] Add support for Duration to IPC --- js/src/ipc/metadata/message.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/js/src/ipc/metadata/message.ts b/js/src/ipc/metadata/message.ts index 6465d3d064720..27c9b92d6897b 100644 --- a/js/src/ipc/metadata/message.ts +++ b/js/src/ipc/metadata/message.ts @@ -36,6 +36,7 @@ import { Date as _Date } from '../../fb/date.js'; import { Time as _Time } from '../../fb/time.js'; import { Timestamp as _Timestamp } from '../../fb/timestamp.js'; import { Interval as _Interval } from '../../fb/interval.js'; +import { Duration as _Duration } from '../../fb/duration.js'; import { Union as _Union } from '../../fb/union.js'; import { FixedSizeBinary as _FixedSizeBinary } from '../../fb/fixed-size-binary.js'; import { FixedSizeList as _FixedSizeList } from '../../fb/fixed-size-list.js'; @@ -57,7 +58,7 @@ import { DataType, Dictionary, TimeBitWidth, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, Union, - Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, + Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, Duration, } from '../../type.js'; /** @@ -466,6 +467,10 @@ function decodeFieldType(f: _Field, children?: Field[]): DataType { const t = f.type(new _Interval())!; return new Interval(t.unit()); } + case Type['Duration']: { + const t = f.type(new _Duration())!; + return new Duration(t.unit()); + } case Type['Union']: { const t = f.type(new _Union())!; return new Union(t.mode(), t.typeIdsArray() || [], children || []); From ddb3f93f45607340e3e746441c21f04032980254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Wed, 23 Aug 2023 15:31:30 +0100 Subject: [PATCH 04/11] Update JavaScript DataType support --- docs/source/status.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/status.rst b/docs/source/status.rst index 36c29fcdc4da6..f2a360d8248ea 100644 --- a/docs/source/status.rst +++ b/docs/source/status.rst @@ -46,7 +46,7 @@ Data Types +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Decimal128 | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ -| Decimal256 | ✓ | ✓ | ✓ | | ✓ | ✓ | ✓ | | +| Decimal256 | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Date32/64 | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ @@ -54,9 +54,9 @@ Data Types +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Timestamp | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ -| Duration | ✓ | ✓ | ✓ | | | ✓ | ✓ | | +| Duration | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ -| Interval | ✓ | ✓ | ✓ | | | ✓ | ✓ | | +| Interval | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Fixed Size Binary | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ From 9918a4556259d7aca52d253a91e25c77cba63c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Wed, 23 Aug 2023 18:11:39 +0100 Subject: [PATCH 05/11] Fix JSON Duration handling --- js/src/ipc/metadata/json.ts | 6 +++++- js/src/visitor/vectorloader.ts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/js/src/ipc/metadata/json.ts b/js/src/ipc/metadata/json.ts index e5995110f084b..f1f306730ddba 100644 --- a/js/src/ipc/metadata/json.ts +++ b/js/src/ipc/metadata/json.ts @@ -22,7 +22,7 @@ import { DataType, Dictionary, TimeBitWidth, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, Union, - Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, + Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, Duration, } from '../../type.js'; import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message.js'; @@ -185,6 +185,10 @@ function typeFromJSON(f: any, children?: Field[]): DataType { const t = f['type']; return new Interval(IntervalUnit[t['unit']] as any); } + case 'duration': { + const t = f['type']; + return new Duration(TimeUnit[t['unit']] as any); + } case 'union': { const t = f['type']; const [m, ...ms] = (t['mode'] + '').toLowerCase(); diff --git a/js/src/visitor/vectorloader.ts b/js/src/visitor/vectorloader.ts index 6fbe803e66cf8..db34edad9a1c1 100644 --- a/js/src/visitor/vectorloader.ts +++ b/js/src/visitor/vectorloader.ts @@ -160,7 +160,7 @@ export class JSONVectorLoader extends VectorLoader { const { sources } = this; if (DataType.isTimestamp(type)) { return toArrayBufferView(Uint8Array, Int64.convertArray(sources[offset] as string[])); - } else if ((DataType.isInt(type) || DataType.isTime(type)) && type.bitWidth === 64) { + } else if ((DataType.isInt(type) || DataType.isTime(type)) && type.bitWidth === 64 || DataType.isDuration(type)) { return toArrayBufferView(Uint8Array, Int64.convertArray(sources[offset] as string[])); } else if (DataType.isDate(type) && type.unit === DateUnit.MILLISECOND) { return toArrayBufferView(Uint8Array, Int64.convertArray(sources[offset] as string[])); From de272fc7fc8563f054e3f25eb92554523ef968de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Wed, 23 Aug 2023 18:11:52 +0100 Subject: [PATCH 06/11] Extend tests to Duration --- js/test/data/tables.ts | 3 ++- js/test/generate-test-data.ts | 20 +++++++++++++++++++- js/test/unit/builders/builder-tests.ts | 4 ++++ js/test/unit/generated-data-tests.ts | 4 ++++ js/test/unit/visitor-tests.ts | 11 +++++++++++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/js/test/data/tables.ts b/js/test/data/tables.ts index e4d859e0a69b7..28aed7e4feccf 100644 --- a/js/test/data/tables.ts +++ b/js/test/data/tables.ts @@ -30,7 +30,8 @@ const valueVectorGeneratorNames = [ 'float16', 'float32', 'float64', 'utf8', 'binary', 'fixedSizeBinary', 'dateDay', 'dateMillisecond', 'timestampSecond', 'timestampMillisecond', 'timestampMicrosecond', 'timestampNanosecond', 'timeSecond', 'timeMillisecond', 'timeMicrosecond', 'timeNanosecond', 'decimal', - 'dictionary', 'intervalDayTime', 'intervalYearMonth' + 'dictionary', 'intervalDayTime', 'intervalYearMonth', + 'durationSecond', 'durationMillisecond', 'durationMicrosecond', 'durationNanosecond', ]; const vectorGeneratorNames = [...valueVectorGeneratorNames, ...listVectorGeneratorNames, ...nestedVectorGeneratorNames]; diff --git a/js/test/generate-test-data.ts b/js/test/generate-test-data.ts index a03b22c54c770..15fb715a31f95 100644 --- a/js/test/generate-test-data.ts +++ b/js/test/generate-test-data.ts @@ -36,6 +36,7 @@ import { Union, DenseUnion, SparseUnion, Dictionary, Interval, IntervalDayTime, IntervalYearMonth, + Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond, FixedSizeList, Map_, DateUnit, TimeUnit, UnionMode, @@ -58,6 +59,7 @@ interface TestDataVectorGenerator extends Visitor { visit(type: T, length?: number, nullCount?: number): GeneratedVector; visit(type: T, length?: number, nullCount?: number): GeneratedVector; visit(type: T, length?: number, nullCount?: number): GeneratedVector; + visit(type: T, length?: number, nullCount?: number): GeneratedVector; visit(type: T, length?: number, nullCount?: number, child?: Vector): GeneratedVector; visit(type: T, length?: number, nullCount?: number, child?: Vector): GeneratedVector; visit(type: T, length?: number, nullCount?: number, dictionary?: Vector): GeneratedVector; @@ -84,6 +86,7 @@ interface TestDataVectorGenerator extends Visitor { visitUnion: typeof generateUnion; visitDictionary: typeof generateDictionary; visitInterval: typeof generateInterval; + visitDuration: typeof generateDuration; visitFixedSizeList: typeof generateFixedSizeList; visitMap: typeof generateMap; } @@ -108,6 +111,7 @@ TestDataVectorGenerator.prototype.visitStruct = generateStruct; TestDataVectorGenerator.prototype.visitUnion = generateUnion; TestDataVectorGenerator.prototype.visitDictionary = generateDictionary; TestDataVectorGenerator.prototype.visitInterval = generateInterval; +TestDataVectorGenerator.prototype.visitDuration = generateDuration; TestDataVectorGenerator.prototype.visitFixedSizeList = generateFixedSizeList; TestDataVectorGenerator.prototype.visitMap = generateMap; @@ -230,11 +234,15 @@ export const sparseUnion = (length = 100, nullCount = Math.trunc(length * 0.2), export const dictionary = (length = 100, nullCount = Math.trunc(length * 0.2), dict: T = new Utf8(), keys: TKey = new Int32()) => vectorGenerator.visit(new Dictionary(dict, keys), length, nullCount); export const intervalDayTime = (length = 100, nullCount = Math.trunc(length * 0.2)) => vectorGenerator.visit(new IntervalDayTime(), length, nullCount); export const intervalYearMonth = (length = 100, nullCount = Math.trunc(length * 0.2)) => vectorGenerator.visit(new IntervalYearMonth(), length, nullCount); +export const durationSecond = (length = 100, nullCount = Math.trunc(length * 0.2)) => vectorGenerator.visit(new DurationSecond(), length, nullCount); +export const durationMillisecond = (length = 100, nullCount = Math.trunc(length * 0.2)) => vectorGenerator.visit(new DurationMillisecond(), length, nullCount); +export const durationMicrosecond = (length = 100, nullCount = Math.trunc(length * 0.2)) => vectorGenerator.visit(new DurationMicrosecond(), length, nullCount); +export const durationNanosecond = (length = 100, nullCount = Math.trunc(length * 0.2)) => vectorGenerator.visit(new DurationNanosecond(), length, nullCount); export const fixedSizeList = (length = 100, nullCount = Math.trunc(length * 0.2), listSize = 2, child = defaultListChild) => vectorGenerator.visit(new FixedSizeList(listSize, child), length, nullCount); export const map = (length = 100, nullCount = Math.trunc(length * 0.2), child: Field> = defaultMapChild()) => vectorGenerator.visit(new Map_(child), length, nullCount); export const vecs = { - null_, bool, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, dateDay, dateMillisecond, timestampSecond, timestampMillisecond, timestampMicrosecond, timestampNanosecond, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, decimal, list, struct, denseUnion, sparseUnion, dictionary, intervalDayTime, intervalYearMonth, fixedSizeList, map + null_, bool, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, dateDay, dateMillisecond, timestampSecond, timestampMillisecond, timestampMicrosecond, timestampNanosecond, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, decimal, list, struct, denseUnion, sparseUnion, dictionary, intervalDayTime, intervalYearMonth, fixedSizeList, map, durationSecond, durationMillisecond, durationMicrosecond, durationNanosecond } as { [k: string]: (...args: any[]) => any }; function generateNull(this: TestDataVectorGenerator, type: T, length = 100): GeneratedVector { @@ -421,6 +429,16 @@ function generateInterval(this: TestDataVectorGenerator, typ return { values, vector: new Vector([makeData({ type, length, nullCount, nullBitmap, data })]) }; } +function generateDuration(this: TestDataVectorGenerator, type: T, length = 100, nullCount = Math.trunc(length * 0.2)): GeneratedVector { + const nullBitmap = createBitmap(length, nullCount); + const multiple = type.unit === TimeUnit.NANOSECOND ? 1000000000 : + type.unit === TimeUnit.MICROSECOND ? 1000000 : + type.unit === TimeUnit.MILLISECOND ? 1000 : 1; + const values: bigint[] = []; + const data = createTime64(length, nullBitmap, multiple, values); + return { values: () => values, vector: new Vector([makeData({ type, length, nullCount, nullBitmap, data })]) }; +} + function generateList(this: TestDataVectorGenerator, type: T, length = 100, nullCount = Math.trunc(length * 0.2), child = this.visit(type.children[0].type, length * 3, nullCount * 3)): GeneratedVector { const childVec = child.vector; const nullBitmap = createBitmap(length, nullCount); diff --git a/js/test/unit/builders/builder-tests.ts b/js/test/unit/builders/builder-tests.ts index a73183a7a5d47..b261e4f815e3a 100644 --- a/js/test/unit/builders/builder-tests.ts +++ b/js/test/unit/builders/builder-tests.ts @@ -64,6 +64,10 @@ describe('Generated Test Data', () => { describe('DictionaryBuilder', () => { validateBuilder(generate.dictionary); }); describe('IntervalDayTimeBuilder', () => { validateBuilder(generate.intervalDayTime); }); describe('IntervalYearMonthBuilder', () => { validateBuilder(generate.intervalYearMonth); }); + describe('DurationSecondBuilder', () => { validateBuilder(generate.durationSecond); }); + describe('DurationMillisecondBuilder', () => { validateBuilder(generate.durationMillisecond); }); + describe('DurationMicrosecondBuilder', () => { validateBuilder(generate.durationMicrosecond); }); + describe('DurationNanosecondBuilder', () => { validateBuilder(generate.durationNanosecond); }); describe('FixedSizeListBuilder', () => { validateBuilder(generate.fixedSizeList); }); describe('MapBuilder', () => { validateBuilder(generate.map); }); }); diff --git a/js/test/unit/generated-data-tests.ts b/js/test/unit/generated-data-tests.ts index 90cf0d598aa6f..d64c7c188d3ed 100644 --- a/js/test/unit/generated-data-tests.ts +++ b/js/test/unit/generated-data-tests.ts @@ -58,6 +58,10 @@ describe('Generated Test Data', () => { describe('Dictionary', () => { validateVector(generate.dictionary()); }); describe('IntervalDayTime', () => { validateVector(generate.intervalDayTime()); }); describe('IntervalYearMonth', () => { validateVector(generate.intervalYearMonth()); }); + describe('DurationSecond', () => { validateVector(generate.durationSecond()); }); + describe('DurationMillisecond', () => { validateVector(generate.durationMillisecond()); }); + describe('DurationMicrosecond', () => { validateVector(generate.durationMicrosecond()); }); + describe('DurationNanosecond', () => { validateVector(generate.durationNanosecond()); }); describe('FixedSizeList', () => { validateVector(generate.fixedSizeList()); }); describe('Map', () => { validateVector(generate.map()); }); }); diff --git a/js/test/unit/visitor-tests.ts b/js/test/unit/visitor-tests.ts index 645fcc60f8d90..8a7ba1ed778aa 100644 --- a/js/test/unit/visitor-tests.ts +++ b/js/test/unit/visitor-tests.ts @@ -25,6 +25,7 @@ import { Interval, IntervalDayTime, IntervalYearMonth, Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond, Union, DenseUnion, SparseUnion, } from 'apache-arrow'; @@ -46,6 +47,7 @@ class BasicVisitor extends Visitor { public visitUnion(type: T) { return (this.type = type); } public visitDictionary(type: T) { return (this.type = type); } public visitInterval(type: T) { return (this.type = type); } + public visitDuration(type: T) { return (this.type = type); } public visitFixedSizeList(type: T) { return (this.type = type); } public visitMap(type: T) { return (this.type = type); } } @@ -86,6 +88,10 @@ class FeatureVisitor extends Visitor { public visitDictionary(type: T) { return (this.type = type); } public visitIntervalDayTime(type: T) { return (this.type = type); } public visitIntervalYearMonth(type: T) { return (this.type = type); } + public visitDurationSecond(type: T) { return (this.type = type); } + public visitDurationMillisecond(type: T) { return (this.type = type); } + public visitDurationMicrosecond(type: T) { return (this.type = type); } + public visitDurationNanosecond(type: T) { return (this.type = type); } public visitFixedSizeList(type: T) { return (this.type = type); } public visitMap(type: T) { return (this.type = type); } } @@ -109,6 +115,7 @@ describe('Visitor', () => { test(`visits Union types`, () => validateBasicVisitor(new Union(0, [] as any[], [] as any[]))); test(`visits Dictionary types`, () => validateBasicVisitor(new Dictionary(null as any, null as any))); test(`visits Interval types`, () => validateBasicVisitor(new Interval(0))); + test(`visits Duration types`, () => validateBasicVisitor(new Duration(0))); test(`visits FixedSizeList types`, () => validateBasicVisitor(new FixedSizeList(2, null as any))); test(`visits Map types`, () => validateBasicVisitor(new Map_(new Field('', new Struct<{ key: Utf8; value: Int }>([ new Field('key', new Utf8()), new Field('value', new Int8()) @@ -158,6 +165,10 @@ describe('Visitor', () => { test(`visits IntervalDayTime types`, () => validateFeatureVisitor(new IntervalDayTime())); test(`visits IntervalYearMonth types`, () => validateFeatureVisitor(new IntervalYearMonth())); test(`visits FixedSizeList types`, () => validateFeatureVisitor(new FixedSizeList(2, null as any))); + test(`visits DurationSecond types`, () => validateFeatureVisitor(new DurationSecond())); + test(`visits DurationMillisecond types`, () => validateFeatureVisitor(new DurationMillisecond())); + test(`visits DurationMicrosecond types`, () => validateFeatureVisitor(new DurationMicrosecond())); + test(`visits DurationNanosecond types`, () => validateFeatureVisitor(new DurationNanosecond())); test(`visits Map types`, () => validateFeatureVisitor(new Map_(new Field('', new Struct<{ key: Utf8; value: Int }>([ new Field('key', new Utf8()), new Field('value', new Int8()) ] as any[]))))); From 975be021bebd4bb6ff2cf2b9286948282c8ffd03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Thu, 24 Aug 2023 09:31:12 +0100 Subject: [PATCH 07/11] Fix DurationBuilder prototypes --- js/src/builder/duration.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/js/src/builder/duration.ts b/js/src/builder/duration.ts index 3af770284cd4f..968899ea55b91 100644 --- a/js/src/builder/duration.ts +++ b/js/src/builder/duration.ts @@ -28,19 +28,19 @@ export class DurationBuilder extends /** @ignore */ export class DurationSecondBuilder extends DurationBuilder { } -(DurationSecond.prototype as any)._setValue = setDurationSecond; +(DurationSecondBuilder.prototype as any)._setValue = setDurationSecond; /** @ignore */ export class DurationMillisecondBuilder extends DurationBuilder { } -(DurationMillisecond.prototype as any)._setValue = setDurationMillisecond; +(DurationMillisecondBuilder.prototype as any)._setValue = setDurationMillisecond; /** @ignore */ export class DurationMicrosecondBuilder extends DurationBuilder { } -(DurationMicrosecond.prototype as any)._setValue = setDurationMicrosecond; +(DurationMicrosecondBuilder.prototype as any)._setValue = setDurationMicrosecond; /** @ignore */ export class DurationNanosecondBuilder extends DurationBuilder { } -(DurationNanosecond.prototype as any)._setValue = setDurationNanosecond; +(DurationNanosecondBuilder.prototype as any)._setValue = setDurationNanosecond; From ce241ee9708a5234ac35998db7333c74e90968a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Thu, 24 Aug 2023 16:54:04 +0100 Subject: [PATCH 08/11] Mark that MonthDayNano is not supported --- docs/source/status.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/status.rst b/docs/source/status.rst index f2a360d8248ea..d2979704f76a0 100644 --- a/docs/source/status.rst +++ b/docs/source/status.rst @@ -56,7 +56,7 @@ Data Types +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Duration | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ -| Interval | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | | +| Interval | ✓ | ✓ | ✓ | ✓ (3) | | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Fixed Size Binary | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ @@ -103,6 +103,7 @@ Notes: * \(1) Float16 support in C# is only available when targeting .NET 6+. * \(2) Nested dictionaries not supported +* \(3) IntervalMonthDayNano not supported .. seealso:: The :ref:`format_columnar` specification. From 4aa2c2b01fa70d96191f8fc46e3f96bc4fc2c8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Thu, 24 Aug 2023 16:55:44 +0100 Subject: [PATCH 09/11] Remove integration skips for duration/interval --- dev/archery/archery/integration/datagen.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py index 5ac32da56a8de..2ba17262c0451 100644 --- a/dev/archery/archery/integration/datagen.py +++ b/dev/archery/archery/integration/datagen.py @@ -1805,12 +1805,10 @@ def _temp_path(): generate_datetime_case(), generate_duration_case() - .skip_tester('C#') - .skip_tester('JS'), # TODO(ARROW-5239): Intervals + JS + .skip_category('C#'), generate_interval_case() - .skip_tester('C#') - .skip_tester('JS'), # TODO(ARROW-5239): Intervals + JS + .skip_category('C#'), generate_month_day_nano_interval_case() .skip_tester('C#') From c2739625c524ff521d1bc4055a6a4fb7f77b4e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Necas?= Date: Thu, 24 Aug 2023 18:03:21 +0100 Subject: [PATCH 10/11] Revert changes to intervals --- dev/archery/archery/integration/datagen.py | 3 ++- docs/source/status.rst | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py index 2ba17262c0451..e941990eaa80a 100644 --- a/dev/archery/archery/integration/datagen.py +++ b/dev/archery/archery/integration/datagen.py @@ -1808,7 +1808,8 @@ def _temp_path(): .skip_category('C#'), generate_interval_case() - .skip_category('C#'), + .skip_category('C#') + .skip_category('JS'), # TODO(ARROW-5239): Intervals + JS generate_month_day_nano_interval_case() .skip_tester('C#') diff --git a/docs/source/status.rst b/docs/source/status.rst index d2979704f76a0..dda284c2771d1 100644 --- a/docs/source/status.rst +++ b/docs/source/status.rst @@ -56,7 +56,7 @@ Data Types +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Duration | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ -| Interval | ✓ | ✓ | ✓ | ✓ (3) | | ✓ | ✓ | | +| Interval | ✓ | ✓ | ✓ | | | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Fixed Size Binary | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ @@ -103,7 +103,6 @@ Notes: * \(1) Float16 support in C# is only available when targeting .NET 6+. * \(2) Nested dictionaries not supported -* \(3) IntervalMonthDayNano not supported .. seealso:: The :ref:`format_columnar` specification. From 50c7e5a66d7c01978d87ad5d3ee87568aac786a1 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 21 Sep 2023 15:49:45 -0700 Subject: [PATCH 11/11] update name skip_category -> skip_tester --- dev/archery/archery/integration/datagen.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py index e941990eaa80a..8ae39ae589f96 100644 --- a/dev/archery/archery/integration/datagen.py +++ b/dev/archery/archery/integration/datagen.py @@ -1805,11 +1805,11 @@ def _temp_path(): generate_datetime_case(), generate_duration_case() - .skip_category('C#'), + .skip_tester('C#'), generate_interval_case() - .skip_category('C#') - .skip_category('JS'), # TODO(ARROW-5239): Intervals + JS + .skip_tester('C#') + .skip_tester('JS'), # TODO(ARROW-5239): Intervals + JS generate_month_day_nano_interval_case() .skip_tester('C#')