From 305f4383ba0cc0eff92ac5f97031a7a926c61f58 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 7 Nov 2023 02:08:39 +0900 Subject: [PATCH 01/57] GH-38562: [Packaging] Add support for Ubuntu 23.10 (#38563) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Ubuntu 23.10 released on 2023-10. ### What changes are included in this PR? Build packages for Ubuntu 23.10. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * Closes: #38562 Authored-by: Sutou Kouhei Signed-off-by: Raúl Cumplido --- dev/release/binary-task.rb | 1 + dev/release/verify-release-candidate.sh | 4 +- .../apt/ubuntu-mantic/Dockerfile | 41 +++++++++ .../apache-arrow/apt/ubuntu-mantic-arm64/from | 18 ++++ .../apache-arrow/apt/ubuntu-mantic/Dockerfile | 85 +++++++++++++++++++ dev/tasks/linux-packages/package-task.rb | 4 + dev/tasks/tasks.yml | 3 +- 7 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-mantic/Dockerfile create mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-mantic-arm64/from create mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-mantic/Dockerfile diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb index e7b618ba3348d..519c8339a4dba 100644 --- a/dev/release/binary-task.rb +++ b/dev/release/binary-task.rb @@ -1089,6 +1089,7 @@ def available_apt_targets ["ubuntu", "focal", "main"], ["ubuntu", "jammy", "main"], ["ubuntu", "lunar", "main"], + ["ubuntu", "mantic", "main"], ] end diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 49997fbf4ea91..05a7498a85180 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -196,7 +196,9 @@ test_apt() { "ubuntu:jammy" \ "arm64v8/ubuntu:jammy" \ "ubuntu:lunar" \ - "arm64v8/ubuntu:lunar"; do \ + "arm64v8/ubuntu:lunar" \ + "ubuntu:mantic" \ + "arm64v8/ubuntu:mantic"; do \ case "${target}" in arm64v8/*) if [ "$(arch)" = "aarch64" -o -e /usr/bin/qemu-aarch64-static ]; then diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-mantic/Dockerfile b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-mantic/Dockerfile new file mode 100644 index 0000000000000..b5a61282b30fc --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-mantic/Dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:mantic + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + gnupg \ + lsb-release && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-mantic-arm64/from b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-mantic-arm64/from new file mode 100644 index 0000000000000..247faef234794 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-mantic-arm64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/ubuntu:mantic diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-mantic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-mantic/Dockerfile new file mode 100644 index 0000000000000..9e90e08d26513 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-mantic/Dockerfile @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=ubuntu:mantic +FROM ${FROM} + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + clang \ + clang-tools \ + cmake \ + debhelper \ + devscripts \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgmock-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + libgtest-dev \ + liblz4-dev \ + libmlir-15-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + llvm-dev \ + lsb-release \ + meson \ + mlir-15-tools \ + ninja-build \ + nlohmann-json3-dev \ + pkg-config \ + protobuf-compiler-grpc \ + python3-dev \ + python3-pip \ + python3-setuptools \ + rapidjson-dev \ + tzdata \ + valac \ + zlib1g-dev && \ + if apt list | grep -q '^libcuda1'; then \ + apt install -y -V ${quiet} nvidia-cuda-toolkit; \ + else \ + :; \ + fi && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb index bb5d70b64f2b0..da281d0ee2cf9 100644 --- a/dev/tasks/linux-packages/package-task.rb +++ b/dev/tasks/linux-packages/package-task.rb @@ -271,12 +271,16 @@ def apt_targets_default # "debian-bullseye-arm64", "debian-bookworm", # "debian-bookworm-arm64", + "debian-trixie", + # "debian-trixie-arm64", "ubuntu-focal", # "ubuntu-focal-arm64", "ubuntu-jammy", # "ubuntu-jammy-arm64", "ubuntu-lunar", # "ubuntu-lunar-arm64", + "ubuntu-mantic", + # "ubuntu-mantic-arm64", ] end diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 0c62200e9b1ab..b8d2f6c0a0086 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -557,7 +557,8 @@ tasks: "debian-trixie", "ubuntu-focal", "ubuntu-jammy", - "ubuntu-lunar"] %} + "ubuntu-lunar", + "ubuntu-mantic"] %} {% for architecture in ["amd64", "arm64"] %} {{ target }}-{{ architecture }}: ci: github From b55d8d5664480f63c2be75f0e18eeb006b640427 Mon Sep 17 00:00:00 2001 From: Jeremy Aguilon Date: Mon, 6 Nov 2023 12:48:35 -0500 Subject: [PATCH 02/57] GH-38381: [C++][Acero] Create a sorted merge node (#38380) ### Rationale for this change This is an implementation of a node that can merge N sorted inputs (only in ascending order for a first pass). Where possible I have shared components with `asof_join_node.cc`. Full description/use case is described in https://github.com/apache/arrow/issues/38381 ### What changes are included in this PR? * Take out relevant guts of asofjoin to stream data top to bottom/consume in a non blocking manner * Implement a sorted merger ### Are these changes tested? Basic test added. Locally I have tested this on 100+ gigabytes of parquet, sharded across 50+ files. Happy to add a benchmark test on top of the basic test, but submitting now for code feedback. ### Are there any user-facing changes? Yes, `sorted_merge` is now an exposed declaration Lead-authored-by: Jeremy Aguilon Co-authored-by: jeremy Co-authored-by: Weston Pace Signed-off-by: Weston Pace --- cpp/src/arrow/acero/CMakeLists.txt | 6 +- cpp/src/arrow/acero/asof_join_node.cc | 432 ++----------- cpp/src/arrow/acero/backpressure_handler.h | 74 +++ .../arrow/acero/concurrent_queue_internal.h | 161 +++++ cpp/src/arrow/acero/exec_plan.cc | 2 + cpp/src/arrow/acero/sorted_merge_node.cc | 609 ++++++++++++++++++ cpp/src/arrow/acero/sorted_merge_node_test.cc | 87 +++ cpp/src/arrow/acero/time_series_util.cc | 63 ++ cpp/src/arrow/acero/time_series_util.h | 31 + cpp/src/arrow/acero/unmaterialized_table.h | 271 ++++++++ 10 files changed, 1357 insertions(+), 379 deletions(-) create mode 100644 cpp/src/arrow/acero/backpressure_handler.h create mode 100644 cpp/src/arrow/acero/concurrent_queue_internal.h create mode 100644 cpp/src/arrow/acero/sorted_merge_node.cc create mode 100644 cpp/src/arrow/acero/sorted_merge_node_test.cc create mode 100644 cpp/src/arrow/acero/time_series_util.cc create mode 100644 cpp/src/arrow/acero/time_series_util.h create mode 100644 cpp/src/arrow/acero/unmaterialized_table.h diff --git a/cpp/src/arrow/acero/CMakeLists.txt b/cpp/src/arrow/acero/CMakeLists.txt index 153413b33cab5..b77d52a23eedb 100644 --- a/cpp/src/arrow/acero/CMakeLists.txt +++ b/cpp/src/arrow/acero/CMakeLists.txt @@ -49,9 +49,11 @@ set(ARROW_ACERO_SRCS project_node.cc query_context.cc sink_node.cc + sorted_merge_node.cc source_node.cc swiss_join.cc task_util.cc + time_series_util.cc tpch_node.cc union_node.cc util.cc) @@ -173,11 +175,13 @@ add_arrow_acero_test(hash_join_node_test SOURCES hash_join_node_test.cc add_arrow_acero_test(pivot_longer_node_test SOURCES pivot_longer_node_test.cc test_nodes.cc) -# asof_join_node uses std::thread internally +# asof_join_node and sorted_merge_node use std::thread internally # and doesn't use ThreadPool so it will # be broken if threading is turned off if(ARROW_ENABLE_THREADING) add_arrow_acero_test(asof_join_node_test SOURCES asof_join_node_test.cc test_nodes.cc) + add_arrow_acero_test(sorted_merge_node_test SOURCES sorted_merge_node_test.cc + test_nodes.cc) endif() add_arrow_acero_test(tpch_node_test SOURCES tpch_node_test.cc) diff --git a/cpp/src/arrow/acero/asof_join_node.cc b/cpp/src/arrow/acero/asof_join_node.cc index d19d2db299cba..4a3b6b199c4c0 100644 --- a/cpp/src/arrow/acero/asof_join_node.cc +++ b/cpp/src/arrow/acero/asof_join_node.cc @@ -16,6 +16,8 @@ // under the License. #include "arrow/acero/asof_join_node.h" +#include "arrow/acero/backpressure_handler.h" +#include "arrow/acero/concurrent_queue_internal.h" #include #include @@ -30,6 +32,7 @@ #include "arrow/acero/exec_plan.h" #include "arrow/acero/options.h" +#include "arrow/acero/unmaterialized_table.h" #ifndef NDEBUG #include "arrow/acero/options_internal.h" #endif @@ -41,6 +44,7 @@ #ifndef NDEBUG #include "arrow/compute/function_internal.h" #endif +#include "arrow/acero/time_series_util.h" #include "arrow/compute/key_hash.h" #include "arrow/compute/light_array.h" #include "arrow/record_batch.h" @@ -122,92 +126,12 @@ struct TolType { typedef uint64_t row_index_t; typedef int col_index_t; -// normalize the value to 64-bits while preserving ordering of values -template ::value, bool> = true> -static inline uint64_t time_value(T t) { - uint64_t bias = std::is_signed::value ? (uint64_t)1 << (8 * sizeof(T) - 1) : 0; - return t < 0 ? static_cast(t + bias) : static_cast(t); -} - // indicates normalization of a key value template ::value, bool> = true> static inline uint64_t key_value(T t) { return static_cast(t); } -/** - * Simple implementation for an unbound concurrent queue - */ -template -class ConcurrentQueue { - public: - T Pop() { - std::unique_lock lock(mutex_); - cond_.wait(lock, [&] { return !queue_.empty(); }); - return PopUnlocked(); - } - - T PopUnlocked() { - auto item = queue_.front(); - queue_.pop(); - return item; - } - - void Push(const T& item) { - std::unique_lock lock(mutex_); - return PushUnlocked(item); - } - - void PushUnlocked(const T& item) { - queue_.push(item); - cond_.notify_one(); - } - - void Clear() { - std::unique_lock lock(mutex_); - ClearUnlocked(); - } - - void ClearUnlocked() { queue_ = std::queue(); } - - std::optional TryPop() { - std::unique_lock lock(mutex_); - return TryPopUnlocked(); - } - - std::optional TryPopUnlocked() { - // Try to pop the oldest value from the queue (or return nullopt if none) - if (queue_.empty()) { - return std::nullopt; - } else { - auto item = queue_.front(); - queue_.pop(); - return item; - } - } - - bool Empty() const { - std::unique_lock lock(mutex_); - return queue_.empty(); - } - - // Un-synchronized access to front - // For this to be "safe": - // 1) the caller logically guarantees that queue is not empty - // 2) pop/try_pop cannot be called concurrently with this - const T& UnsyncFront() const { return queue_.front(); } - - size_t UnsyncSize() const { return queue_.size(); } - - protected: - std::mutex& GetMutex() { return mutex_; } - - private: - std::queue queue_; - mutable std::mutex mutex_; - std::condition_variable cond_; -}; - class AsofJoinNode; #ifndef NDEBUG @@ -547,104 +471,6 @@ class BackpressureController : public BackpressureControl { std::atomic& backpressure_counter_; }; -class BackpressureHandler { - private: - BackpressureHandler(ExecNode* input, size_t low_threshold, size_t high_threshold, - std::unique_ptr backpressure_control) - : input_(input), - low_threshold_(low_threshold), - high_threshold_(high_threshold), - backpressure_control_(std::move(backpressure_control)) {} - - public: - static Result Make( - ExecNode* input, size_t low_threshold, size_t high_threshold, - std::unique_ptr backpressure_control) { - if (low_threshold >= high_threshold) { - return Status::Invalid("low threshold (", low_threshold, - ") must be less than high threshold (", high_threshold, ")"); - } - if (backpressure_control == NULLPTR) { - return Status::Invalid("null backpressure control parameter"); - } - BackpressureHandler backpressure_handler(input, low_threshold, high_threshold, - std::move(backpressure_control)); - return std::move(backpressure_handler); - } - - void Handle(size_t start_level, size_t end_level) { - if (start_level < high_threshold_ && end_level >= high_threshold_) { - backpressure_control_->Pause(); - } else if (start_level > low_threshold_ && end_level <= low_threshold_) { - backpressure_control_->Resume(); - } - } - - Status ForceShutdown() { - // It may be unintuitive to call Resume() here, but this is to avoid a deadlock. - // Since acero's executor won't terminate if any one node is paused, we need to - // force resume the node before stopping production. - backpressure_control_->Resume(); - return input_->StopProducing(); - } - - private: - ExecNode* input_; - size_t low_threshold_; - size_t high_threshold_; - std::unique_ptr backpressure_control_; -}; - -template -class BackpressureConcurrentQueue : public ConcurrentQueue { - private: - struct DoHandle { - explicit DoHandle(BackpressureConcurrentQueue& queue) - : queue_(queue), start_size_(queue_.UnsyncSize()) {} - - ~DoHandle() { - size_t end_size = queue_.UnsyncSize(); - queue_.handler_.Handle(start_size_, end_size); - } - - BackpressureConcurrentQueue& queue_; - size_t start_size_; - }; - - public: - explicit BackpressureConcurrentQueue(BackpressureHandler handler) - : handler_(std::move(handler)) {} - - T Pop() { - std::unique_lock lock(ConcurrentQueue::GetMutex()); - DoHandle do_handle(*this); - return ConcurrentQueue::PopUnlocked(); - } - - void Push(const T& item) { - std::unique_lock lock(ConcurrentQueue::GetMutex()); - DoHandle do_handle(*this); - ConcurrentQueue::PushUnlocked(item); - } - - void Clear() { - std::unique_lock lock(ConcurrentQueue::GetMutex()); - DoHandle do_handle(*this); - ConcurrentQueue::ClearUnlocked(); - } - - std::optional TryPop() { - std::unique_lock lock(ConcurrentQueue::GetMutex()); - DoHandle do_handle(*this); - return ConcurrentQueue::TryPopUnlocked(); - } - - Status ForceShutdown() { return handler_.ForceShutdown(); } - - private: - BackpressureHandler handler_; -}; - class InputState { // InputState correponds to an input // Input record batches are queued up in InputState until processed and @@ -783,29 +609,8 @@ class InputState { } inline OnType GetLatestTime() const { - return GetTime(GetLatestBatch().get(), latest_ref_row_); - } - - inline ByType GetTime(const RecordBatch* batch, row_index_t row) const { - auto data = batch->column_data(time_col_index_); - switch (time_type_id_) { - LATEST_VAL_CASE(INT8, time_value) - LATEST_VAL_CASE(INT16, time_value) - LATEST_VAL_CASE(INT32, time_value) - LATEST_VAL_CASE(INT64, time_value) - LATEST_VAL_CASE(UINT8, time_value) - LATEST_VAL_CASE(UINT16, time_value) - LATEST_VAL_CASE(UINT32, time_value) - LATEST_VAL_CASE(UINT64, time_value) - LATEST_VAL_CASE(DATE32, time_value) - LATEST_VAL_CASE(DATE64, time_value) - LATEST_VAL_CASE(TIME32, time_value) - LATEST_VAL_CASE(TIME64, time_value) - LATEST_VAL_CASE(TIMESTAMP, time_value) - default: - DCHECK(false); - return 0; // cannot happen - } + return GetTime(GetLatestBatch().get(), time_type_id_, time_col_index_, + latest_ref_row_); } #undef LATEST_VAL_CASE @@ -832,7 +637,9 @@ class InputState { have_active_batch &= !queue_.TryPop(); if (have_active_batch) { DCHECK_GT(queue_.UnsyncFront()->num_rows(), 0); // empty batches disallowed - memo_.UpdateTime(GetTime(queue_.UnsyncFront().get(), 0)); // time changed + memo_.UpdateTime(GetTime(queue_.UnsyncFront().get(), time_type_id_, + time_col_index_, + 0)); // time changed } } } @@ -988,35 +795,25 @@ class InputState { std::vector> src_to_dst_; }; +/// Wrapper around UnmaterializedCompositeTable that knows how to emplace +/// the join row-by-row template -struct CompositeReferenceRow { - struct Entry { - arrow::RecordBatch* batch; // can be NULL if there's no value - row_index_t row; - }; - Entry refs[MAX_TABLES]; -}; +class CompositeTableBuilder { + using SliceBuilder = UnmaterializedSliceBuilder; + using CompositeTable = UnmaterializedCompositeTable; -// A table of composite reference rows. Rows maintain pointers to the -// constituent record batches, but the overall table retains shared_ptr -// references to ensure memory remains resident while the table is live. -// -// The main reason for this is that, especially for wide tables, joins -// are effectively row-oriented, rather than column-oriented. Separating -// the join part from the columnar materialization part simplifies the -// logic around data types and increases efficiency. -// -// We don't put the shared_ptr's into the rows for efficiency reasons. -template -class CompositeReferenceTable { public: - NDEBUG_EXPLICIT CompositeReferenceTable(DEBUG_ADD(size_t n_tables, AsofJoinNode* node)) - : DEBUG_ADD(n_tables_(n_tables), node_(node)) { + NDEBUG_EXPLICIT CompositeTableBuilder( + const std::vector>& inputs, + const std::shared_ptr& schema, arrow::MemoryPool* pool, + DEBUG_ADD(size_t n_tables, AsofJoinNode* node)) + : unmaterialized_table(InitUnmaterializedTable(schema, inputs, pool)), + DEBUG_ADD(n_tables_(n_tables), node_(node)) { DCHECK_GE(n_tables_, 1); DCHECK_LE(n_tables_, MAX_TABLES); } - size_t n_rows() const { return rows_.size(); } + size_t n_rows() const { return unmaterialized_table.Size(); } // Adds the latest row from the input state as a new composite reference row // - LHS must have a valid key,timestep,and latest rows @@ -1037,14 +834,16 @@ class CompositeReferenceTable { // On the first row of the batch, we resize the destination. // The destination size is dictated by the size of the LHS batch. row_index_t new_batch_size = lhs_latest_batch->num_rows(); - row_index_t new_capacity = rows_.size() + new_batch_size; - if (rows_.capacity() < new_capacity) rows_.reserve(new_capacity); + row_index_t new_capacity = unmaterialized_table.Size() + new_batch_size; + if (unmaterialized_table.capacity() < new_capacity) { + unmaterialized_table.reserve(new_capacity); + } } - rows_.resize(rows_.size() + 1); - auto& row = rows_.back(); - row.refs[0].batch = lhs_latest_batch.get(); - row.refs[0].row = lhs_latest_row; - AddRecordBatchRef(lhs_latest_batch); + + SliceBuilder new_row{&unmaterialized_table}; + + // Each item represents a portion of the columns of the output table + new_row.AddEntry(lhs_latest_batch, lhs_latest_row, lhs_latest_row + 1); DEBUG_SYNC(node_, "Emplace: key=", key, " lhs_latest_row=", lhs_latest_row, " lhs_latest_time=", lhs_latest_time, DEBUG_MANIP(std::endl)); @@ -1068,100 +867,25 @@ class CompositeReferenceTable { if (tolerance.Accepts(lhs_latest_time, (*opt_entry)->time)) { // Have a valid entry const MemoStore::Entry* entry = *opt_entry; - row.refs[i].batch = entry->batch.get(); - row.refs[i].row = entry->row; - AddRecordBatchRef(entry->batch); + new_row.AddEntry(entry->batch, entry->row, entry->row + 1); continue; } } - row.refs[i].batch = NULL; - row.refs[i].row = 0; + new_row.AddEntry(nullptr, 0, 1); } + new_row.Finalize(); } // Materializes the current reference table into a target record batch - Result> Materialize( - MemoryPool* memory_pool, const std::shared_ptr& output_schema, - const std::vector>& state) { - DCHECK_EQ(state.size(), n_tables_); - - // Don't build empty batches - size_t n_rows = rows_.size(); - if (!n_rows) return NULLPTR; - - // Build the arrays column-by-column from the rows - std::vector> arrays(output_schema->num_fields()); - for (size_t i_table = 0; i_table < n_tables_; ++i_table) { - int n_src_cols = state.at(i_table)->get_schema()->num_fields(); - { - for (col_index_t i_src_col = 0; i_src_col < n_src_cols; ++i_src_col) { - std::optional i_dst_col_opt = - state[i_table]->MapSrcToDst(i_src_col); - if (!i_dst_col_opt) continue; - col_index_t i_dst_col = *i_dst_col_opt; - const auto& src_field = state[i_table]->get_schema()->field(i_src_col); - const auto& dst_field = output_schema->field(i_dst_col); - DCHECK(src_field->type()->Equals(dst_field->type())); - DCHECK_EQ(src_field->name(), dst_field->name()); - const auto& field_type = src_field->type(); - -#define ASOFJOIN_MATERIALIZE_CASE(id) \ - case Type::id: { \ - using T = typename TypeIdTraits::Type; \ - ARROW_ASSIGN_OR_RAISE( \ - arrays.at(i_dst_col), \ - MaterializeColumn(memory_pool, field_type, i_table, i_src_col)); \ - break; \ - } - - switch (field_type->id()) { - ASOFJOIN_MATERIALIZE_CASE(BOOL) - ASOFJOIN_MATERIALIZE_CASE(INT8) - ASOFJOIN_MATERIALIZE_CASE(INT16) - ASOFJOIN_MATERIALIZE_CASE(INT32) - ASOFJOIN_MATERIALIZE_CASE(INT64) - ASOFJOIN_MATERIALIZE_CASE(UINT8) - ASOFJOIN_MATERIALIZE_CASE(UINT16) - ASOFJOIN_MATERIALIZE_CASE(UINT32) - ASOFJOIN_MATERIALIZE_CASE(UINT64) - ASOFJOIN_MATERIALIZE_CASE(FLOAT) - ASOFJOIN_MATERIALIZE_CASE(DOUBLE) - ASOFJOIN_MATERIALIZE_CASE(DATE32) - ASOFJOIN_MATERIALIZE_CASE(DATE64) - ASOFJOIN_MATERIALIZE_CASE(TIME32) - ASOFJOIN_MATERIALIZE_CASE(TIME64) - ASOFJOIN_MATERIALIZE_CASE(TIMESTAMP) - ASOFJOIN_MATERIALIZE_CASE(STRING) - ASOFJOIN_MATERIALIZE_CASE(LARGE_STRING) - ASOFJOIN_MATERIALIZE_CASE(BINARY) - ASOFJOIN_MATERIALIZE_CASE(LARGE_BINARY) - default: - return Status::Invalid("Unsupported data type ", - src_field->type()->ToString(), " for field ", - src_field->name()); - } - -#undef ASOFJOIN_MATERIALIZE_CASE - } - } - } - - // Build the result - DCHECK_LE(n_rows, (uint64_t)std::numeric_limits::max()); - std::shared_ptr r = - arrow::RecordBatch::Make(output_schema, (int64_t)n_rows, arrays); - return r; + Result>> Materialize() { + return unmaterialized_table.Materialize(); } // Returns true if there are no rows - bool empty() const { return rows_.empty(); } + bool empty() const { return unmaterialized_table.Empty(); } private: - // Contains shared_ptr refs for all RecordBatches referred to by the contents of rows_ - std::unordered_map> _ptr2ref; - - // Row table references - std::vector> rows_; + CompositeTable unmaterialized_table; // Total number of tables in the composite table size_t n_tables_; @@ -1171,70 +895,20 @@ class CompositeReferenceTable { AsofJoinNode* node_; #endif - // Adds a RecordBatch ref to the mapping, if needed - void AddRecordBatchRef(const std::shared_ptr& ref) { - if (!_ptr2ref.count((uintptr_t)ref.get())) _ptr2ref[(uintptr_t)ref.get()] = ref; - } - - template ::BuilderType> - enable_if_boolean static BuilderAppend( - Builder& builder, const std::shared_ptr& source, row_index_t row) { - if (source->IsNull(row)) { - builder.UnsafeAppendNull(); - return Status::OK(); - } - builder.UnsafeAppend(bit_util::GetBit(source->template GetValues(1), row)); - return Status::OK(); - } - - template ::BuilderType> - enable_if_t::value && !is_boolean_type::value, - Status> static BuilderAppend(Builder& builder, - const std::shared_ptr& source, - row_index_t row) { - if (source->IsNull(row)) { - builder.UnsafeAppendNull(); - return Status::OK(); - } - using CType = typename TypeTraits::CType; - builder.UnsafeAppend(source->template GetValues(1)[row]); - return Status::OK(); - } - - template ::BuilderType> - enable_if_base_binary static BuilderAppend( - Builder& builder, const std::shared_ptr& source, row_index_t row) { - if (source->IsNull(row)) { - return builder.AppendNull(); - } - using offset_type = typename Type::offset_type; - const uint8_t* data = source->buffers[2]->data(); - const offset_type* offsets = source->GetValues(1); - const offset_type offset0 = offsets[row]; - const offset_type offset1 = offsets[row + 1]; - return builder.Append(data + offset0, offset1 - offset0); - } - - template ::BuilderType> - Result> MaterializeColumn(MemoryPool* memory_pool, - const std::shared_ptr& type, - size_t i_table, col_index_t i_col) { - ARROW_ASSIGN_OR_RAISE(auto a_builder, MakeBuilder(type, memory_pool)); - Builder& builder = *checked_cast(a_builder.get()); - ARROW_RETURN_NOT_OK(builder.Reserve(rows_.size())); - for (row_index_t i_row = 0; i_row < rows_.size(); ++i_row) { - const auto& ref = rows_[i_row].refs[i_table]; - if (ref.batch) { - Status st = - BuilderAppend(builder, ref.batch->column_data(i_col), ref.row); - ARROW_RETURN_NOT_OK(st); - } else { - builder.UnsafeAppendNull(); + static CompositeTable InitUnmaterializedTable( + const std::shared_ptr& schema, + const std::vector>& inputs, arrow::MemoryPool* pool) { + std::unordered_map> dst_to_src; + for (size_t i = 0; i < inputs.size(); i++) { + auto& input = inputs[i]; + for (int src = 0; src < input->get_schema()->num_fields(); src++) { + auto dst = input->MapSrcToDst(src); + if (dst.has_value()) { + dst_to_src[dst.value()] = std::make_pair(static_cast(i), src); + } } } - std::shared_ptr result; - ARROW_RETURN_NOT_OK(builder.Finish(&result)); - return result; + return CompositeTable{schema, inputs.size(), dst_to_src, pool}; } }; @@ -1279,7 +953,9 @@ class AsofJoinNode : public ExecNode { auto& lhs = *state_.at(0); // Construct new target table if needed - CompositeReferenceTable dst(DEBUG_ADD(state_.size(), this)); + CompositeTableBuilder dst(state_, output_schema_, + plan()->query_context()->memory_pool(), + DEBUG_ADD(state_.size(), this)); // Generate rows into the dst table until we either run out of data or hit the row // limit, or run out of input @@ -1318,8 +994,8 @@ class AsofJoinNode : public ExecNode { if (dst.empty()) { return NULLPTR; } else { - return dst.Materialize(plan()->query_context()->memory_pool(), output_schema(), - state_); + ARROW_ASSIGN_OR_RAISE(auto out, dst.Materialize()); + return out.has_value() ? out.value() : NULLPTR; } } diff --git a/cpp/src/arrow/acero/backpressure_handler.h b/cpp/src/arrow/acero/backpressure_handler.h new file mode 100644 index 0000000000000..178272315d7fb --- /dev/null +++ b/cpp/src/arrow/acero/backpressure_handler.h @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include "arrow/acero/exec_plan.h" +#include "arrow/acero/options.h" + +#include + +namespace arrow::acero { + +class BackpressureHandler { + private: + BackpressureHandler(ExecNode* input, size_t low_threshold, size_t high_threshold, + std::unique_ptr backpressure_control) + : input_(input), + low_threshold_(low_threshold), + high_threshold_(high_threshold), + backpressure_control_(std::move(backpressure_control)) {} + + public: + static Result Make( + ExecNode* input, size_t low_threshold, size_t high_threshold, + std::unique_ptr backpressure_control) { + if (low_threshold >= high_threshold) { + return Status::Invalid("low threshold (", low_threshold, + ") must be less than high threshold (", high_threshold, ")"); + } + if (backpressure_control == NULLPTR) { + return Status::Invalid("null backpressure control parameter"); + } + BackpressureHandler backpressure_handler(input, low_threshold, high_threshold, + std::move(backpressure_control)); + return std::move(backpressure_handler); + } + + void Handle(size_t start_level, size_t end_level) { + if (start_level < high_threshold_ && end_level >= high_threshold_) { + backpressure_control_->Pause(); + } else if (start_level > low_threshold_ && end_level <= low_threshold_) { + backpressure_control_->Resume(); + } + } + + Status ForceShutdown() { + // It may be unintuitive to call Resume() here, but this is to avoid a deadlock. + // Since acero's executor won't terminate if any one node is paused, we need to + // force resume the node before stopping production. + backpressure_control_->Resume(); + return input_->StopProducing(); + } + + private: + ExecNode* input_; + size_t low_threshold_; + size_t high_threshold_; + std::unique_ptr backpressure_control_; +}; + +} // namespace arrow::acero diff --git a/cpp/src/arrow/acero/concurrent_queue_internal.h b/cpp/src/arrow/acero/concurrent_queue_internal.h new file mode 100644 index 0000000000000..f530394187299 --- /dev/null +++ b/cpp/src/arrow/acero/concurrent_queue_internal.h @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include "arrow/acero/backpressure_handler.h" + +namespace arrow::acero { + +/** + * Simple implementation for a thread safe blocking unbound multi-consumer / + * multi-producer concurrent queue + */ +template +class ConcurrentQueue { + public: + // Pops the last item from the queue. Must be called on a non-empty queue + // + T Pop() { + std::unique_lock lock(mutex_); + cond_.wait(lock, [&] { return !queue_.empty(); }); + return PopUnlocked(); + } + + // Pops the last item from the queue, or returns a nullopt if empty + // + std::optional TryPop() { + std::unique_lock lock(mutex_); + return TryPopUnlocked(); + } + + // Pushes an item to the queue + // + void Push(const T& item) { + std::unique_lock lock(mutex_); + return PushUnlocked(item); + } + + // Clears the queue + // + void Clear() { + std::unique_lock lock(mutex_); + ClearUnlocked(); + } + + bool Empty() const { + std::unique_lock lock(mutex_); + return queue_.empty(); + } + + // Un-synchronized access to front + // For this to be "safe": + // 1) the caller logically guarantees that queue is not empty + // 2) pop/try_pop cannot be called concurrently with this + const T& UnsyncFront() const { return queue_.front(); } + + size_t UnsyncSize() const { return queue_.size(); } + + protected: + std::mutex& GetMutex() { return mutex_; } + + T PopUnlocked() { + auto item = queue_.front(); + queue_.pop(); + return item; + } + + void PushUnlocked(const T& item) { + queue_.push(item); + cond_.notify_one(); + } + + void ClearUnlocked() { queue_ = std::queue(); } + + std::optional TryPopUnlocked() { + // Try to pop the oldest value from the queue (or return nullopt if none) + if (queue_.empty()) { + return std::nullopt; + } else { + auto item = queue_.front(); + queue_.pop(); + return item; + } + } + std::queue queue_; + + private: + mutable std::mutex mutex_; + std::condition_variable cond_; +}; + +template +class BackpressureConcurrentQueue : public ConcurrentQueue { + private: + struct DoHandle { + explicit DoHandle(BackpressureConcurrentQueue& queue) + : queue_(queue), start_size_(queue_.UnsyncSize()) {} + + ~DoHandle() { + // unsynced access is safe since DoHandle is internally only used when the + // lock is held + size_t end_size = queue_.UnsyncSize(); + queue_.handler_.Handle(start_size_, end_size); + } + + BackpressureConcurrentQueue& queue_; + size_t start_size_; + }; + + public: + explicit BackpressureConcurrentQueue(BackpressureHandler handler) + : handler_(std::move(handler)) {} + + T Pop() { + std::unique_lock lock(ConcurrentQueue::GetMutex()); + DoHandle do_handle(*this); + return ConcurrentQueue::PopUnlocked(); + } + + void Push(const T& item) { + std::unique_lock lock(ConcurrentQueue::GetMutex()); + DoHandle do_handle(*this); + ConcurrentQueue::PushUnlocked(item); + } + + void Clear() { + std::unique_lock lock(ConcurrentQueue::GetMutex()); + DoHandle do_handle(*this); + ConcurrentQueue::ClearUnlocked(); + } + + std::optional TryPop() { + std::unique_lock lock(ConcurrentQueue::GetMutex()); + DoHandle do_handle(*this); + return ConcurrentQueue::TryPopUnlocked(); + } + + Status ForceShutdown() { return handler_.ForceShutdown(); } + + private: + BackpressureHandler handler_; +}; + +} // namespace arrow::acero diff --git a/cpp/src/arrow/acero/exec_plan.cc b/cpp/src/arrow/acero/exec_plan.cc index 541e5fed6206b..97119726d4b17 100644 --- a/cpp/src/arrow/acero/exec_plan.cc +++ b/cpp/src/arrow/acero/exec_plan.cc @@ -1114,6 +1114,7 @@ void RegisterAggregateNode(ExecFactoryRegistry*); void RegisterSinkNode(ExecFactoryRegistry*); void RegisterHashJoinNode(ExecFactoryRegistry*); void RegisterAsofJoinNode(ExecFactoryRegistry*); +void RegisterSortedMergeNode(ExecFactoryRegistry*); } // namespace internal @@ -1132,6 +1133,7 @@ ExecFactoryRegistry* default_exec_factory_registry() { internal::RegisterSinkNode(this); internal::RegisterHashJoinNode(this); internal::RegisterAsofJoinNode(this); + internal::RegisterSortedMergeNode(this); } Result GetFactory(const std::string& factory_name) override { diff --git a/cpp/src/arrow/acero/sorted_merge_node.cc b/cpp/src/arrow/acero/sorted_merge_node.cc new file mode 100644 index 0000000000000..f3b934eda186b --- /dev/null +++ b/cpp/src/arrow/acero/sorted_merge_node.cc @@ -0,0 +1,609 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include "arrow/acero/concurrent_queue_internal.h" +#include "arrow/acero/exec_plan.h" +#include "arrow/acero/options.h" +#include "arrow/acero/query_context.h" +#include "arrow/acero/time_series_util.h" +#include "arrow/acero/unmaterialized_table.h" +#include "arrow/acero/util.h" +#include "arrow/array/builder_base.h" +#include "arrow/result.h" +#include "arrow/type_fwd.h" +#include "arrow/util/logging.h" + +namespace { +template +struct Defer { + Callable callable; + explicit Defer(Callable callable_) : callable(std::move(callable_)) {} + ~Defer() noexcept { callable(); } +}; + +std::vector GetInputLabels( + const arrow::acero::ExecNode::NodeVector& inputs) { + std::vector labels(inputs.size()); + for (size_t i = 0; i < inputs.size(); i++) { + labels[i] = "input_" + std::to_string(i) + "_label"; + } + return labels; +} + +template +inline typename T::const_iterator std_find(const T& container, const V& val) { + return std::find(container.begin(), container.end(), val); +} + +template +inline bool std_has(const T& container, const V& val) { + return container.end() != std_find(container, val); +} + +} // namespace + +namespace arrow::acero { + +namespace { + +// Each slice is associated with a single input source, so we only need 1 record +// batch per slice +using SingleRecordBatchSliceBuilder = arrow::acero::UnmaterializedSliceBuilder<1>; +using SingleRecordBatchCompositeTable = arrow::acero::UnmaterializedCompositeTable<1>; + +using row_index_t = uint64_t; +using time_unit_t = uint64_t; +using col_index_t = int; + +constexpr bool kNewTask = true; +constexpr bool kPoisonPill = false; + +class BackpressureController : public BackpressureControl { + public: + BackpressureController(ExecNode* node, ExecNode* output, + std::atomic& backpressure_counter) + : node_(node), output_(output), backpressure_counter_(backpressure_counter) {} + + void Pause() override { node_->PauseProducing(output_, ++backpressure_counter_); } + void Resume() override { node_->ResumeProducing(output_, ++backpressure_counter_); } + + private: + ExecNode* node_; + ExecNode* output_; + std::atomic& backpressure_counter_; +}; + +/// InputState correponds to an input. Input record batches are queued up in InputState +/// until processed and turned into output record batches. +class InputState { + public: + InputState(size_t index, BackpressureHandler handler, + const std::shared_ptr& schema, const int time_col_index) + : index_(index), + queue_(std::move(handler)), + schema_(schema), + time_col_index_(time_col_index), + time_type_id_(schema_->fields()[time_col_index_]->type()->id()) {} + + template + static arrow::Result Make(size_t index, arrow::acero::ExecNode* input, + arrow::acero::ExecNode* output, + std::atomic& backpressure_counter, + const std::shared_ptr& schema, + const col_index_t time_col_index) { + constexpr size_t low_threshold = 4, high_threshold = 8; + std::unique_ptr backpressure_control = + std::make_unique(input, output, backpressure_counter); + ARROW_ASSIGN_OR_RAISE(auto handler, + BackpressureHandler::Make(input, low_threshold, high_threshold, + std::move(backpressure_control))); + return PtrType(new InputState(index, std::move(handler), schema, time_col_index)); + } + + bool IsTimeColumn(col_index_t i) const { + DCHECK_LT(i, schema_->num_fields()); + return (i == time_col_index_); + } + + // Gets the latest row index, assuming the queue isn't empty + row_index_t GetLatestRow() const { return latest_ref_row_; } + + bool Empty() const { + // cannot be empty if ref row is >0 -- can avoid slow queue lock + // below + if (latest_ref_row_ > 0) { + return false; + } + return queue_.Empty(); + } + + size_t index() const { return index_; } + + int total_batches() const { return total_batches_; } + + // Gets latest batch (precondition: must not be empty) + const std::shared_ptr& GetLatestBatch() const { + return queue_.UnsyncFront(); + } + +#define LATEST_VAL_CASE(id, val) \ + case arrow::Type::id: { \ + using T = typename arrow::TypeIdTraits::Type; \ + using CType = typename arrow::TypeTraits::CType; \ + return val(data->GetValues(1)[row]); \ + } + + inline time_unit_t GetLatestTime() const { + return GetTime(GetLatestBatch().get(), time_type_id_, time_col_index_, + latest_ref_row_); + } + +#undef LATEST_VAL_CASE + + bool Finished() const { return batches_processed_ == total_batches_; } + + void Advance(SingleRecordBatchSliceBuilder& builder) { + // Advance the row until a new time is encountered or the record batch + // ends. This will return a range of {-1, -1} and a nullptr if there is + // no input + bool active = + (latest_ref_row_ > 0 /*short circuit the lock on the queue*/) || !queue_.Empty(); + + if (!active) { + return; + } + + row_index_t start = latest_ref_row_; + row_index_t end = latest_ref_row_; + time_unit_t startTime = GetLatestTime(); + std::shared_ptr batch = queue_.UnsyncFront(); + auto rows_in_batch = (row_index_t)batch->num_rows(); + + while (GetLatestTime() == startTime) { + end = ++latest_ref_row_; + if (latest_ref_row_ >= rows_in_batch) { + // hit the end of the batch, need to get the next batch if + // possible. + ++batches_processed_; + latest_ref_row_ = 0; + active &= !queue_.TryPop(); + if (active) { + DCHECK_GT(queue_.UnsyncFront()->num_rows(), + 0); // empty batches disallowed, sanity check + } + break; + } + } + builder.AddEntry(batch, start, end); + } + + arrow::Status Push(const std::shared_ptr& rb) { + if (rb->num_rows() > 0) { + queue_.Push(rb); + } else { + ++batches_processed_; // don't enqueue empty batches, just record + // as processed + } + return arrow::Status::OK(); + } + + const std::shared_ptr& get_schema() const { return schema_; } + + void set_total_batches(int n) { total_batches_ = n; } + + private: + size_t index_; + // Pending record batches. The latest is the front. Batches cannot be empty. + BackpressureConcurrentQueue> queue_; + // Schema associated with the input + std::shared_ptr schema_; + // Total number of batches (only int because InputFinished uses int) + std::atomic total_batches_{-1}; + // Number of batches processed so far (only int because InputFinished uses + // int) + std::atomic batches_processed_{0}; + // Index of the time col + col_index_t time_col_index_; + // Type id of the time column + arrow::Type::type time_type_id_; + // Index of the latest row reference within; if >0 then queue_ cannot be + // empty Must be < queue_.front()->num_rows() if queue_ is non-empty + row_index_t latest_ref_row_ = 0; + // Time of latest row + time_unit_t latest_time_ = std::numeric_limits::lowest(); +}; + +struct InputStateComparator { + bool operator()(const std::shared_ptr& lhs, + const std::shared_ptr& rhs) const { + // True if lhs is ahead of time of rhs + if (lhs->Finished()) { + return false; + } + if (rhs->Finished()) { + return false; + } + time_unit_t lFirst = lhs->GetLatestTime(); + time_unit_t rFirst = rhs->GetLatestTime(); + return lFirst > rFirst; + } +}; + +class SortedMergeNode : public ExecNode { + static constexpr int64_t kTargetOutputBatchSize = 1024 * 1024; + + public: + SortedMergeNode(arrow::acero::ExecPlan* plan, + std::vector inputs, + std::shared_ptr output_schema, + arrow::Ordering new_ordering) + : ExecNode(plan, inputs, GetInputLabels(inputs), std::move(output_schema)), + ordering_(std::move(new_ordering)), + input_counter(inputs_.size()), + output_counter(inputs_.size()), + process_thread() { + SetLabel("sorted_merge"); + } + + ~SortedMergeNode() override { + process_queue.Push( + kPoisonPill); // poison pill + // We might create a temporary (such as to inspect the output + // schema), in which case there isn't anything to join + if (process_thread.joinable()) { + process_thread.join(); + } + } + + static arrow::Result Make( + arrow::acero::ExecPlan* plan, std::vector inputs, + const arrow::acero::ExecNodeOptions& options) { + RETURN_NOT_OK(ValidateExecNodeInputs(plan, inputs, static_cast(inputs.size()), + "SortedMergeNode")); + + if (inputs.size() < 1) { + return Status::Invalid("Constructing a `SortedMergeNode` with < 1 inputs"); + } + + const auto schema = inputs.at(0)->output_schema(); + for (const auto& input : inputs) { + if (!input->output_schema()->Equals(schema)) { + return Status::Invalid( + "SortedMergeNode input schemas must all " + "match, first schema " + "was: ", + schema->ToString(), " got schema: ", input->output_schema()->ToString()); + } + } + + const auto& order_options = + arrow::internal::checked_cast(options); + + if (order_options.ordering.is_implicit() || order_options.ordering.is_unordered()) { + return Status::Invalid("`ordering` must be an explicit non-empty ordering"); + } + + std::shared_ptr output_schema = inputs[0]->output_schema(); + return plan->EmplaceNode( + plan, std::move(inputs), std::move(output_schema), order_options.ordering); + } + + const char* kind_name() const override { return "SortedMergeNode"; } + + const arrow::Ordering& ordering() const override { return ordering_; } + + arrow::Status Init() override { + ARROW_CHECK(ordering_.sort_keys().size() == 1) << "Only one sort key supported"; + + auto inputs = this->inputs(); + for (size_t i = 0; i < inputs.size(); i++) { + ExecNode* input = inputs[i]; + const auto& schema = input->output_schema(); + + const auto& sort_key = ordering_.sort_keys()[0]; + if (sort_key.order != arrow::compute::SortOrder::Ascending) { + return Status::NotImplemented("Only ascending sort order is supported"); + } + + const FieldRef& ref = sort_key.target; + auto match_res = ref.FindOne(*schema); + if (!match_res.ok()) { + return Status::Invalid("Bad sort key : ", match_res.status().message()); + } + ARROW_ASSIGN_OR_RAISE(auto match, match_res); + ARROW_DCHECK(match.indices().size() == 1); + + ARROW_ASSIGN_OR_RAISE(auto input_state, + InputState::Make>( + i, input, this, backpressure_counter, schema, + std::move(match.indices()[0]))); + state.push_back(std::move(input_state)); + } + return Status::OK(); + } + + arrow::Status InputReceived(arrow::acero::ExecNode* input, + arrow::ExecBatch batch) override { + ARROW_DCHECK(std_has(inputs_, input)); + const size_t index = std_find(inputs_, input) - inputs_.begin(); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr rb, + batch.ToRecordBatch(output_schema_)); + + // Push into the queue. Note that we don't need to lock since + // InputState's ConcurrentQueue manages locking + input_counter[index] += rb->num_rows(); + ARROW_RETURN_NOT_OK(state[index]->Push(rb)); + process_queue.Push(kNewTask); + return Status::OK(); + } + + arrow::Status InputFinished(arrow::acero::ExecNode* input, int total_batches) override { + ARROW_DCHECK(std_has(inputs_, input)); + { + std::lock_guard guard(gate); + ARROW_DCHECK(std_has(inputs_, input)); + size_t k = std_find(inputs_, input) - inputs_.begin(); + state.at(k)->set_total_batches(total_batches); + } + // Trigger a final process call for stragglers + process_queue.Push(kNewTask); + return Status::OK(); + } + + arrow::Status StartProducing() override { + ARROW_ASSIGN_OR_RAISE(process_task, plan_->query_context()->BeginExternalTask( + "SortedMergeNode::ProcessThread")); + if (!process_task.is_valid()) { + // Plan has already aborted. Do not start process thread + return Status::OK(); + } + process_thread = std::thread(&SortedMergeNode::StartPoller, this); + return Status::OK(); + } + + arrow::Status StopProducingImpl() override { + process_queue.Clear(); + process_queue.Push(kPoisonPill); + return Status::OK(); + } + + // handled by the backpressure controller + void PauseProducing(arrow::acero::ExecNode* output, int32_t counter) override {} + void ResumeProducing(arrow::acero::ExecNode* output, int32_t counter) override {} + + protected: + std::string ToStringExtra(int indent) const override { + std::stringstream ss; + ss << "ordering=" << ordering_.ToString(); + return ss.str(); + } + + private: + void EndFromProcessThread(arrow::Status st = arrow::Status::OK()) { + ARROW_CHECK(!cleanup_started); + for (size_t i = 0; i < input_counter.size(); ++i) { + ARROW_CHECK(input_counter[i] == output_counter[i]) + << input_counter[i] << " != " << output_counter[i]; + } + + ARROW_UNUSED( + plan_->query_context()->executor()->Spawn([this, st = std::move(st)]() mutable { + Defer cleanup([this, &st]() { process_task.MarkFinished(st); }); + if (st.ok()) { + st = output_->InputFinished(this, batches_produced); + } + })); + } + + bool CheckEnded() { + bool all_finished = true; + for (const auto& s : state) { + all_finished &= s->Finished(); + } + if (all_finished) { + EndFromProcessThread(); + return false; + } + return true; + } + + /// Streams the input states in sorted order until we run out of input + arrow::Result> getNextBatch() { + DCHECK(!state.empty()); + for (const auto& s : state) { + if (s->Empty() && !s->Finished()) { + return nullptr; // not enough data, wait + } + } + + std::vector> heap = state; + // filter out finished states + heap.erase(std::remove_if( + heap.begin(), heap.end(), + [](const std::shared_ptr& s) { return s->Finished(); }), + heap.end()); + + // If any are Empty(), then return early since we don't have enough data + if (std::any_of(heap.begin(), heap.end(), + [](const std::shared_ptr& s) { return s->Empty(); })) { + return nullptr; + } + + // Currently we only support one sort key + const auto sort_col = *ordering_.sort_keys().at(0).target.name(); + const auto comp = InputStateComparator(); + std::make_heap(heap.begin(), heap.end(), comp); + + // Each slice only has one record batch with the same schema as the output + std::unordered_map> output_col_to_src; + for (int i = 0; i < output_schema_->num_fields(); i++) { + output_col_to_src[i] = std::make_pair(0, i); + } + SingleRecordBatchCompositeTable output(output_schema(), 1, + std::move(output_col_to_src), + plan()->query_context()->memory_pool()); + + // Generate rows until we run out of data or we exceed the target output + // size + bool waiting_for_more_data = false; + while (!waiting_for_more_data && !heap.empty() && + output.Size() < kTargetOutputBatchSize) { + std::pop_heap(heap.begin(), heap.end(), comp); + + auto& next_item = heap.back(); + time_unit_t latest_time = std::numeric_limits::min(); + time_unit_t new_time = next_item->GetLatestTime(); + ARROW_CHECK(new_time >= latest_time) + << "Input state " << next_item->index() + << " has out of order data. newTime=" << new_time + << " latestTime=" << latest_time; + + latest_time = new_time; + SingleRecordBatchSliceBuilder builder{&output}; + next_item->Advance(builder); + + if (builder.Size() > 0) { + output_counter[next_item->index()] += builder.Size(); + builder.Finalize(); + } + if (next_item->Finished()) { + heap.pop_back(); + } else if (next_item->Empty()) { + // We've run out of data on one of the inputs + waiting_for_more_data = true; + continue; // skip the unnecessary make_heap + } + std::make_heap(heap.begin(), heap.end(), comp); + } + + // Emit the batch + if (output.Size() == 0) { + return nullptr; + } + + ARROW_ASSIGN_OR_RAISE(auto maybe_rb, output.Materialize()); + return maybe_rb.value_or(nullptr); + } + /// Gets a batch. Returns true if there is more data to process, false if we + /// are done or an error occurred + bool PollOnce() { + std::lock_guard guard(gate); + if (!CheckEnded()) { + return false; + } + + // Process batches while we have data + for (;;) { + Result> result = getNextBatch(); + + if (result.ok()) { + auto out_rb = *result; + if (!out_rb) { + break; + } + ExecBatch out_b(*out_rb); + out_b.index = batches_produced++; + Status st = output_->InputReceived(this, std::move(out_b)); + if (!st.ok()) { + ARROW_LOG(FATAL) << "Error in output_::InputReceived: " << st.ToString(); + EndFromProcessThread(std::move(st)); + } + } else { + EndFromProcessThread(result.status()); + return false; + } + } + + // Report to the output the total batch count, if we've already + // finished everything (there are two places where this can happen: + // here and InputFinished) + // + // It may happen here in cases where InputFinished was called before + // we were finished producing results (so we didn't know the output + // size at that time) + if (!CheckEnded()) { + return false; + } + + // There is no more we can do now but there is still work remaining + // for later when more data arrives. + return true; + } + + void EmitBatches() { + while (true) { + // Implementation note: If the queue is empty, we will block here + if (process_queue.Pop() == kPoisonPill) { + EndFromProcessThread(); + } + // Either we're out of data or something went wrong + if (!PollOnce()) { + return; + } + } + } + + /// The entry point for processThread + static void StartPoller(SortedMergeNode* node) { node->EmitBatches(); } + + arrow::Ordering ordering_; + + // Each input state corresponds to an input (e.g. a parquet data file) + std::vector> state; + std::vector input_counter; + std::vector output_counter; + std::mutex gate; + + std::atomic cleanup_started{false}; + + // Backpressure counter common to all input states + std::atomic backpressure_counter; + + std::atomic batches_produced{0}; + + // Queue to trigger processing of a given input. False acts as a poison pill + ConcurrentQueue process_queue; + // Once StartProducing is called, we initialize this thread to poll the + // input states and emit batches + std::thread process_thread; + arrow::Future<> process_task; + + // Map arg index --> completion counter + std::vector counter_; + // Map arg index --> data + std::vector accumulation_queue_; + std::mutex mutex_; + std::atomic total_batches_{0}; +}; + +} // namespace + +namespace internal { +void RegisterSortedMergeNode(ExecFactoryRegistry* registry) { + DCHECK_OK(registry->AddFactory("sorted_merge", SortedMergeNode::Make)); +} +} // namespace internal + +} // namespace arrow::acero diff --git a/cpp/src/arrow/acero/sorted_merge_node_test.cc b/cpp/src/arrow/acero/sorted_merge_node_test.cc new file mode 100644 index 0000000000000..55446d631d90c --- /dev/null +++ b/cpp/src/arrow/acero/sorted_merge_node_test.cc @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "arrow/acero/exec_plan.h" +#include "arrow/acero/map_node.h" +#include "arrow/acero/options.h" +#include "arrow/acero/test_nodes.h" +#include "arrow/array/builder_base.h" +#include "arrow/array/concatenate.h" +#include "arrow/compute/ordering.h" +#include "arrow/result.h" +#include "arrow/scalar.h" +#include "arrow/table.h" +#include "arrow/testing/generator.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/type.h" +#include "arrow/type_fwd.h" + +namespace arrow::acero { + +std::shared_ptr TestTable(int start, int step, int rows_per_batch, + int num_batches) { + return gen::Gen({{"timestamp", gen::Step(start, step, /*signed_int=*/true)}, + {"str", gen::Random(utf8())}}) + ->FailOnError() + ->Table(rows_per_batch, num_batches); +} + +TEST(SortedMergeNode, Basic) { + auto table1 = TestTable( + /*start=*/0, + /*step=*/2, + /*rows_per_batch=*/2, + /*num_batches=*/3); + auto table2 = TestTable( + /*start=*/1, + /*step=*/2, + /*rows_per_batch=*/3, + /*num_batches=*/2); + auto table3 = TestTable( + /*start=*/3, + /*step=*/3, + /*rows_per_batch=*/6, + /*num_batches=*/1); + std::vector src_decls; + src_decls.emplace_back(Declaration("table_source", TableSourceNodeOptions(table1))); + src_decls.emplace_back(Declaration("table_source", TableSourceNodeOptions(table2))); + src_decls.emplace_back(Declaration("table_source", TableSourceNodeOptions(table3))); + + auto ops = OrderByNodeOptions(compute::Ordering({compute::SortKey("timestamp")})); + + Declaration sorted_merge{"sorted_merge", src_decls, ops}; + // We can't use threads for sorted merging since it relies on + // ascending deterministic order of timestamps + ASSERT_OK_AND_ASSIGN(auto output, + DeclarationToTable(sorted_merge, /*use_threads=*/false)); + ASSERT_EQ(output->num_rows(), 18); + + ASSERT_OK_AND_ASSIGN(auto expected_ts_builder, + MakeBuilder(int32(), default_memory_pool())); + for (auto i : {0, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9, 9, 10, 11, 12, 15, 18}) { + ASSERT_OK(expected_ts_builder->AppendScalar(*MakeScalar(i))); + } + ASSERT_OK_AND_ASSIGN(auto expected_ts, expected_ts_builder->Finish()); + auto output_col = output->column(0); + ASSERT_OK_AND_ASSIGN(auto output_ts, Concatenate(output_col->chunks())); + + AssertArraysEqual(*expected_ts, *output_ts); +} + +} // namespace arrow::acero diff --git a/cpp/src/arrow/acero/time_series_util.cc b/cpp/src/arrow/acero/time_series_util.cc new file mode 100644 index 0000000000000..71133fef47306 --- /dev/null +++ b/cpp/src/arrow/acero/time_series_util.cc @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/array/data.h" + +#include "arrow/acero/time_series_util.h" +#include "arrow/util/logging.h" + +namespace arrow::acero { + +template ::value, bool>> +inline uint64_t NormalizeTime(T t) { + uint64_t bias = + std::is_signed::value ? static_cast(1) << (8 * sizeof(T) - 1) : 0; + return t < 0 ? static_cast(t + bias) : static_cast(t); +} + +uint64_t GetTime(const RecordBatch* batch, Type::type time_type, int col, uint64_t row) { +#define LATEST_VAL_CASE(id, val) \ + case Type::id: { \ + using T = typename TypeIdTraits::Type; \ + using CType = typename TypeTraits::CType; \ + return val(data->GetValues(1)[row]); \ + } + + auto data = batch->column_data(col); + switch (time_type) { + LATEST_VAL_CASE(INT8, NormalizeTime) + LATEST_VAL_CASE(INT16, NormalizeTime) + LATEST_VAL_CASE(INT32, NormalizeTime) + LATEST_VAL_CASE(INT64, NormalizeTime) + LATEST_VAL_CASE(UINT8, NormalizeTime) + LATEST_VAL_CASE(UINT16, NormalizeTime) + LATEST_VAL_CASE(UINT32, NormalizeTime) + LATEST_VAL_CASE(UINT64, NormalizeTime) + LATEST_VAL_CASE(DATE32, NormalizeTime) + LATEST_VAL_CASE(DATE64, NormalizeTime) + LATEST_VAL_CASE(TIME32, NormalizeTime) + LATEST_VAL_CASE(TIME64, NormalizeTime) + LATEST_VAL_CASE(TIMESTAMP, NormalizeTime) + default: + DCHECK(false); + return 0; // cannot happen + } + +#undef LATEST_VAL_CASE +} + +} // namespace arrow::acero diff --git a/cpp/src/arrow/acero/time_series_util.h b/cpp/src/arrow/acero/time_series_util.h new file mode 100644 index 0000000000000..97707f43bf20b --- /dev/null +++ b/cpp/src/arrow/acero/time_series_util.h @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/record_batch.h" +#include "arrow/type_traits.h" + +namespace arrow::acero { + +// normalize the value to unsigned 64-bits while preserving ordering of values +template ::value, bool> = true> +uint64_t NormalizeTime(T t); + +uint64_t GetTime(const RecordBatch* batch, Type::type time_type, int col, uint64_t row); + +} // namespace arrow::acero diff --git a/cpp/src/arrow/acero/unmaterialized_table.h b/cpp/src/arrow/acero/unmaterialized_table.h new file mode 100644 index 0000000000000..05d6c866936e0 --- /dev/null +++ b/cpp/src/arrow/acero/unmaterialized_table.h @@ -0,0 +1,271 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include "arrow/array/builder_base.h" +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/memory_pool.h" +#include "arrow/record_batch.h" +#include "arrow/type_traits.h" +#include "arrow/util/logging.h" + +namespace arrow::acero { + +/// Lightweight representation of a cell of an unmaterialized table. +/// +struct CompositeEntry { + RecordBatch* batch; + uint64_t start; + uint64_t end; +}; + +// Forward declare the builder +template +class UnmaterializedSliceBuilder; + +/// A table of composite reference rows. Rows maintain pointers to the +/// constituent record batches, but the overall table retains shared_ptr +/// references to ensure memory remains resident while the table is live. +/// +/// The main reason for this is that, especially for wide tables, some operations +/// such as sorted_merge or asof_join are effectively row-oriented, rather than +/// column-oriented. Separating the join part from the columnar materialization +/// part simplifies the logic around data types and increases efficiency. +/// +/// We don't put the shared_ptr's into the rows for efficiency reasons. Use +/// UnmaterializedSliceBuilder to add ranges of record batches to this table +template +class UnmaterializedCompositeTable { + public: + UnmaterializedCompositeTable( + const std::shared_ptr& output_schema, size_t num_composite_tables, + std::unordered_map> output_col_to_src_, + arrow::MemoryPool* pool_ = arrow::default_memory_pool()) + : schema(output_schema), + num_composite_tables(num_composite_tables), + output_col_to_src(std::move(output_col_to_src_)), + pool{pool_} {} + + // Shallow wrappers around std::vector for performance + inline size_t capacity() { return slices.capacity(); } + inline void reserve(size_t num_slices) { slices.reserve(num_slices); } + + inline size_t Size() const { return num_rows; } + inline size_t Empty() const { return num_rows == 0; } + + Result>> Materialize() { + // Don't build empty batches + if (Empty()) { + return std::nullopt; + } + DCHECK_LE(Size(), (uint64_t)std::numeric_limits::max()); + std::vector> arrays(schema->num_fields()); + +#define MATERIALIZE_CASE(id) \ + case arrow::Type::id: { \ + using T = typename arrow::TypeIdTraits::Type; \ + ARROW_ASSIGN_OR_RAISE(arrays.at(i_col), materializeColumn(field_type, i_col)); \ + break; \ + } + + // Build the arrays column-by-column from the rows + for (int i_col = 0; i_col < schema->num_fields(); ++i_col) { + const std::shared_ptr& field = schema->field(i_col); + const auto& field_type = field->type(); + + switch (field_type->id()) { + MATERIALIZE_CASE(BOOL) + MATERIALIZE_CASE(INT8) + MATERIALIZE_CASE(INT16) + MATERIALIZE_CASE(INT32) + MATERIALIZE_CASE(INT64) + MATERIALIZE_CASE(UINT8) + MATERIALIZE_CASE(UINT16) + MATERIALIZE_CASE(UINT32) + MATERIALIZE_CASE(UINT64) + MATERIALIZE_CASE(FLOAT) + MATERIALIZE_CASE(DOUBLE) + MATERIALIZE_CASE(DATE32) + MATERIALIZE_CASE(DATE64) + MATERIALIZE_CASE(TIME32) + MATERIALIZE_CASE(TIME64) + MATERIALIZE_CASE(TIMESTAMP) + MATERIALIZE_CASE(STRING) + MATERIALIZE_CASE(LARGE_STRING) + MATERIALIZE_CASE(BINARY) + MATERIALIZE_CASE(LARGE_BINARY) + default: + return arrow::Status::Invalid("Unsupported data type ", + field->type()->ToString(), " for field ", + field->name()); + } + } + +#undef MATERIALIZE_CASE + + std::shared_ptr r = + arrow::RecordBatch::Make(schema, (int64_t)num_rows, arrays); + return r; + } + + private: + struct UnmaterializedSlice { + CompositeEntry components[MAX_COMPOSITE_TABLES]; + size_t num_components; + + inline int64_t Size() const { + if (num_components == 0) { + return 0; + } + return components[0].end - components[0].start; + } + }; + + // Mapping from an output column ID to a source table ID and column ID + std::shared_ptr schema; + size_t num_composite_tables; + std::unordered_map> output_col_to_src; + + arrow::MemoryPool* pool; + + /// A map from address of a record batch to the record batch. Used to + /// maintain the lifetime of the record batch in case it goes out of scope + /// by the main exec node thread + std::unordered_map> ptr2Ref = {}; + std::vector slices; + + size_t num_rows = 0; + + // for AddRecordBatchRef/AddSlice and access to UnmaterializedSlice + friend class UnmaterializedSliceBuilder; + + void AddRecordBatchRef(const std::shared_ptr& ref) { + ptr2Ref[(uintptr_t)ref.get()] = ref; + } + void AddSlice(const UnmaterializedSlice& slice) { + slices.push_back(slice); + num_rows += slice.Size(); + } + + template ::BuilderType> + enable_if_boolean static BuilderAppend( + Builder& builder, const std::shared_ptr& source, uint64_t row) { + if (source->IsNull(row)) { + builder.UnsafeAppendNull(); + return Status::OK(); + } + builder.UnsafeAppend(bit_util::GetBit(source->template GetValues(1), row)); + return Status::OK(); + } + + template ::BuilderType> + enable_if_t::value && !is_boolean_type::value, + Status> static BuilderAppend(Builder& builder, + const std::shared_ptr& source, + uint64_t row) { + if (source->IsNull(row)) { + builder.UnsafeAppendNull(); + return Status::OK(); + } + using CType = typename TypeTraits::CType; + builder.UnsafeAppend(source->template GetValues(1)[row]); + return Status::OK(); + } + + template ::BuilderType> + enable_if_base_binary static BuilderAppend( + Builder& builder, const std::shared_ptr& source, uint64_t row) { + if (source->IsNull(row)) { + return builder.AppendNull(); + } + using offset_type = typename Type::offset_type; + const uint8_t* data = source->buffers[2]->data(); + const offset_type* offsets = source->GetValues(1); + const offset_type offset0 = offsets[row]; + const offset_type offset1 = offsets[row + 1]; + return builder.Append(data + offset0, offset1 - offset0); + } + + template ::BuilderType> + arrow::Result> materializeColumn( + const std::shared_ptr& type, int i_col) { + ARROW_ASSIGN_OR_RAISE(auto builderPtr, arrow::MakeBuilder(type, pool)); + Builder& builder = *arrow::internal::checked_cast(builderPtr.get()); + ARROW_RETURN_NOT_OK(builder.Reserve(num_rows)); + + const auto& [table_index, column_index] = output_col_to_src[i_col]; + + for (const auto& unmaterialized_slice : slices) { + const auto& [batch, start, end] = unmaterialized_slice.components[table_index]; + if (batch) { + for (uint64_t rowNum = start; rowNum < end; ++rowNum) { + arrow::Status st = BuilderAppend( + builder, batch->column_data(column_index), rowNum); + ARROW_RETURN_NOT_OK(st); + } + } else { + for (uint64_t rowNum = start; rowNum < end; ++rowNum) { + ARROW_RETURN_NOT_OK(builder.AppendNull()); + } + } + } + std::shared_ptr result; + ARROW_RETURN_NOT_OK(builder.Finish(&result)); + return Result{std::move(result)}; + } +}; + +/// A builder class that can append blocks of data to a row. A "slice" +/// is built by horizontally concatenating record batches. +template +class UnmaterializedSliceBuilder { + public: + explicit UnmaterializedSliceBuilder( + UnmaterializedCompositeTable* table_) + : table(table_) {} + + void AddEntry(std::shared_ptr rb, uint64_t start, uint64_t end) { + if (rb) { + table->AddRecordBatchRef(rb); + } + if (slice.num_components) { + size_t last_index = slice.num_components - 1; + DCHECK_EQ(slice.components[last_index].end - slice.components[last_index].start, + end - start) + << "Slices should be the same length. "; + } + slice.components[slice.num_components++] = CompositeEntry{rb.get(), start, end}; + } + + void Finalize() { table->AddSlice(slice); } + int64_t Size() { return slice.Size(); } + + private: + using TUnmaterializedCompositeTable = + UnmaterializedCompositeTable; + using TUnmaterializedSlice = + typename TUnmaterializedCompositeTable::UnmaterializedSlice; + + TUnmaterializedCompositeTable* table; + TUnmaterializedSlice slice{}; +}; + +} // namespace arrow::acero From 02d8bd26ef5a8a6f840d4cc98f669ea81c534487 Mon Sep 17 00:00:00 2001 From: Laurent Goujon Date: Mon, 6 Nov 2023 11:34:40 -0800 Subject: [PATCH 03/57] GH-38578: [Java][FlightSQL] Remove joda usage from flight-sql library (#38579) ### Rationale for this change [joda](https://www.joda.org/joda-time/) is a very popular date/time manipulation library for java but the introduction of `java.time` package makes it obsolete and author actually recommends using `java.time` over `joda` > Note that from Java SE 8 onwards, users are asked to migrate to java.time (JSR-310) - a core part of the JDK which replaces this project. ### What changes are included in this PR? Change include the use of `java.time` classes over `joda` classes and the removal of `joda` library as a dependency ### Are these changes tested? As there is no behavior change, it is covered by the existing tests * Closes: #38578 Authored-by: Laurent Goujon Signed-off-by: David Li --- java/flight/flight-sql-jdbc-core/pom.xml | 6 -- ...ArrowFlightJdbcIntervalVectorAccessor.java | 7 +- .../jdbc/utils/IntervalStringUtils.java | 15 ++-- ...wFlightJdbcIntervalVectorAccessorTest.java | 74 ++++++++++--------- 4 files changed, 49 insertions(+), 53 deletions(-) diff --git a/java/flight/flight-sql-jdbc-core/pom.xml b/java/flight/flight-sql-jdbc-core/pom.xml index 8a0e62bde8471..cbeaa88f1e2f7 100644 --- a/java/flight/flight-sql-jdbc-core/pom.xml +++ b/java/flight/flight-sql-jdbc-core/pom.xml @@ -136,12 +136,6 @@ bcpkix-jdk15on 1.61 - - - joda-time - joda-time - 2.10.14 - diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessor.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessor.java index 283dc9160a9e9..21d1c15712cdb 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessor.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessor.java @@ -22,6 +22,8 @@ import static org.apache.arrow.vector.util.DateUtility.yearsToMonths; import java.sql.SQLException; +import java.time.Duration; +import java.time.Period; import java.util.function.IntSupplier; import org.apache.arrow.driver.jdbc.accessor.ArrowFlightJdbcAccessor; @@ -31,7 +33,6 @@ import org.apache.arrow.vector.IntervalYearVector; import org.apache.arrow.vector.holders.NullableIntervalDayHolder; import org.apache.arrow.vector.holders.NullableIntervalYearHolder; -import org.joda.time.Period; /** * Accessor for the Arrow type {@link IntervalDayVector}. @@ -62,7 +63,7 @@ public ArrowFlightJdbcIntervalVectorAccessor(IntervalDayVector vector, } else { final int days = holder.days; final int millis = holder.milliseconds; - return formatIntervalDay(new Period().plusDays(days).plusMillis(millis)); + return formatIntervalDay(Duration.ofDays(days).plusMillis(millis)); } }; objectClass = java.time.Duration.class; @@ -89,7 +90,7 @@ public ArrowFlightJdbcIntervalVectorAccessor(IntervalYearVector vector, final int interval = holder.value; final int years = (interval / yearsToMonths); final int months = (interval % yearsToMonths); - return formatIntervalYear(new Period().plusYears(years).plusMonths(months)); + return formatIntervalYear(Period.ofYears(years).plusMonths(months)); } }; objectClass = java.time.Period.class; diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/IntervalStringUtils.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/IntervalStringUtils.java index 05643274ac348..fdf6c508d93b0 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/IntervalStringUtils.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/IntervalStringUtils.java @@ -17,8 +17,10 @@ package org.apache.arrow.driver.jdbc.utils; +import java.time.Duration; +import java.time.Period; + import org.apache.arrow.vector.util.DateUtility; -import org.joda.time.Period; /** * Utility class to format periods similar to Oracle's representation @@ -36,7 +38,7 @@ private IntervalStringUtils( ) {} * For example, the string "+21-02" defines an interval of 21 years and 2 months. */ public static String formatIntervalYear(final Period p) { - long months = p.getYears() * (long) DateUtility.yearsToMonths + p.getMonths(); + long months = p.toTotalMonths(); boolean neg = false; if (months < 0) { months = -months; @@ -53,8 +55,8 @@ public static String formatIntervalYear(final Period p) { * For example, the string "-001 18:25:16.766" defines an interval of * - 1 day 18 hours 25 minutes 16 seconds and 766 milliseconds. */ - public static String formatIntervalDay(final Period p) { - long millis = p.getDays() * (long) DateUtility.daysToStandardMillis + millisFromPeriod(p); + public static String formatIntervalDay(final Duration d) { + long millis = d.toMillis(); boolean neg = false; if (millis < 0) { @@ -76,9 +78,4 @@ public static String formatIntervalDay(final Period p) { return String.format("%c%03d %02d:%02d:%02d.%03d", neg ? '-' : '+', days, hours, minutes, seconds, millis); } - - public static int millisFromPeriod(Period period) { - return period.getHours() * DateUtility.hoursToMillis + period.getMinutes() * DateUtility.minutesToMillis + - period.getSeconds() * DateUtility.secondsToMillis + period.getMillis(); - } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessorTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessorTest.java index ea228692202a7..322b7d40bd6e1 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessorTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessorTest.java @@ -21,7 +21,6 @@ import static org.apache.arrow.driver.jdbc.utils.IntervalStringUtils.formatIntervalYear; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; -import static org.joda.time.Period.parse; import java.time.Duration; import java.time.Period; @@ -142,57 +141,62 @@ private String getStringOnVector(ValueVector vector, int index) { if (object == null) { return null; } else if (vector instanceof IntervalDayVector) { - return formatIntervalDay(parse(object)); + return formatIntervalDay(Duration.parse(object)); } else if (vector instanceof IntervalYearVector) { - return formatIntervalYear(parse(object)); + return formatIntervalYear(Period.parse(object)); } return null; } @Test public void testShouldGetIntervalYear( ) { - Assert.assertEquals("-002-00", formatIntervalYear(parse("P-2Y"))); - Assert.assertEquals("-001-01", formatIntervalYear(parse("P-1Y-1M"))); - Assert.assertEquals("-001-02", formatIntervalYear(parse("P-1Y-2M"))); - Assert.assertEquals("-002-03", formatIntervalYear(parse("P-2Y-3M"))); - Assert.assertEquals("-002-04", formatIntervalYear(parse("P-2Y-4M"))); - Assert.assertEquals("-011-01", formatIntervalYear(parse("P-11Y-1M"))); - Assert.assertEquals("+002-00", formatIntervalYear(parse("P+2Y"))); - Assert.assertEquals("+001-01", formatIntervalYear(parse("P+1Y1M"))); - Assert.assertEquals("+001-02", formatIntervalYear(parse("P+1Y2M"))); - Assert.assertEquals("+002-03", formatIntervalYear(parse("P+2Y3M"))); - Assert.assertEquals("+002-04", formatIntervalYear(parse("P+2Y4M"))); - Assert.assertEquals("+011-01", formatIntervalYear(parse("P+11Y1M"))); + Assert.assertEquals("-002-00", formatIntervalYear(Period.parse("P-2Y"))); + Assert.assertEquals("-001-01", formatIntervalYear(Period.parse("P-1Y-1M"))); + Assert.assertEquals("-001-02", formatIntervalYear(Period.parse("P-1Y-2M"))); + Assert.assertEquals("-002-03", formatIntervalYear(Period.parse("P-2Y-3M"))); + Assert.assertEquals("-002-04", formatIntervalYear(Period.parse("P-2Y-4M"))); + Assert.assertEquals("-011-01", formatIntervalYear(Period.parse("P-11Y-1M"))); + Assert.assertEquals("+002-00", formatIntervalYear(Period.parse("P+2Y"))); + Assert.assertEquals("+001-01", formatIntervalYear(Period.parse("P+1Y1M"))); + Assert.assertEquals("+001-02", formatIntervalYear(Period.parse("P+1Y2M"))); + Assert.assertEquals("+002-03", formatIntervalYear(Period.parse("P+2Y3M"))); + Assert.assertEquals("+002-04", formatIntervalYear(Period.parse("P+2Y4M"))); + Assert.assertEquals("+011-01", formatIntervalYear(Period.parse("P+11Y1M"))); } @Test public void testShouldGetIntervalDay( ) { - Assert.assertEquals("-001 00:00:00.000", formatIntervalDay(parse("PT-24H"))); - Assert.assertEquals("+001 00:00:00.000", formatIntervalDay(parse("PT+24H"))); - Assert.assertEquals("-000 01:00:00.000", formatIntervalDay(parse("PT-1H"))); - Assert.assertEquals("-000 01:00:00.001", formatIntervalDay(parse("PT-1H-0M-00.001S"))); - Assert.assertEquals("-000 01:01:01.000", formatIntervalDay(parse("PT-1H-1M-1S"))); - Assert.assertEquals("-000 02:02:02.002", formatIntervalDay(parse("PT-2H-2M-02.002S"))); - Assert.assertEquals("-000 23:59:59.999", formatIntervalDay(parse("PT-23H-59M-59.999S"))); - Assert.assertEquals("-000 11:59:00.100", formatIntervalDay(parse("PT-11H-59M-00.100S"))); - Assert.assertEquals("-000 05:02:03.000", formatIntervalDay(parse("PT-5H-2M-3S"))); - Assert.assertEquals("-000 22:22:22.222", formatIntervalDay(parse("PT-22H-22M-22.222S"))); - Assert.assertEquals("+000 01:00:00.000", formatIntervalDay(parse("PT+1H"))); - Assert.assertEquals("+000 01:00:00.001", formatIntervalDay(parse("PT+1H0M00.001S"))); - Assert.assertEquals("+000 01:01:01.000", formatIntervalDay(parse("PT+1H1M1S"))); - Assert.assertEquals("+000 02:02:02.002", formatIntervalDay(parse("PT+2H2M02.002S"))); - Assert.assertEquals("+000 23:59:59.999", formatIntervalDay(parse("PT+23H59M59.999S"))); - Assert.assertEquals("+000 11:59:00.100", formatIntervalDay(parse("PT+11H59M00.100S"))); - Assert.assertEquals("+000 05:02:03.000", formatIntervalDay(parse("PT+5H2M3S"))); - Assert.assertEquals("+000 22:22:22.222", formatIntervalDay(parse("PT+22H22M22.222S"))); + Assert.assertEquals("-001 00:00:00.000", formatIntervalDay(Duration.parse("PT-24H"))); + Assert.assertEquals("+001 00:00:00.000", formatIntervalDay(Duration.parse("PT+24H"))); + Assert.assertEquals("-000 01:00:00.000", formatIntervalDay(Duration.parse("PT-1H"))); + // "JDK-8054978: java.time.Duration.parse() fails for negative duration with 0 seconds and nanos" not fixed on JDK8 + //Assert.assertEquals("-000 01:00:00.001", formatIntervalDay(Duration.parse("PT-1H-0M-00.001S"))); + Assert.assertEquals("-000 01:00:00.001", formatIntervalDay(Duration.ofHours(-1).minusMillis(1))); + Assert.assertEquals("-000 01:01:01.000", formatIntervalDay(Duration.parse("PT-1H-1M-1S"))); + Assert.assertEquals("-000 02:02:02.002", formatIntervalDay(Duration.parse("PT-2H-2M-02.002S"))); + Assert.assertEquals("-000 23:59:59.999", formatIntervalDay(Duration.parse("PT-23H-59M-59.999S"))); + // "JDK-8054978: java.time.Duration.parse() fails for negative duration with 0 seconds and nanos" not fixed on JDK8 + //Assert.assertEquals("-000 11:59:00.100", formatIntervalDay(Duration.parse("PT-11H-59M-00.100S"))); + Assert.assertEquals("-000 11:59:00.100", + formatIntervalDay(Duration.ofHours(-11).minusMinutes(59).minusMillis(100))); + Assert.assertEquals("-000 05:02:03.000", formatIntervalDay(Duration.parse("PT-5H-2M-3S"))); + Assert.assertEquals("-000 22:22:22.222", formatIntervalDay(Duration.parse("PT-22H-22M-22.222S"))); + Assert.assertEquals("+000 01:00:00.000", formatIntervalDay(Duration.parse("PT+1H"))); + Assert.assertEquals("+000 01:00:00.001", formatIntervalDay(Duration.parse("PT+1H0M00.001S"))); + Assert.assertEquals("+000 01:01:01.000", formatIntervalDay(Duration.parse("PT+1H1M1S"))); + Assert.assertEquals("+000 02:02:02.002", formatIntervalDay(Duration.parse("PT+2H2M02.002S"))); + Assert.assertEquals("+000 23:59:59.999", formatIntervalDay(Duration.parse("PT+23H59M59.999S"))); + Assert.assertEquals("+000 11:59:00.100", formatIntervalDay(Duration.parse("PT+11H59M00.100S"))); + Assert.assertEquals("+000 05:02:03.000", formatIntervalDay(Duration.parse("PT+5H2M3S"))); + Assert.assertEquals("+000 22:22:22.222", formatIntervalDay(Duration.parse("PT+22H22M22.222S"))); } @Test public void testIntervalDayWithJodaPeriodObject() { Assert.assertEquals("+1567 00:00:00.000", - formatIntervalDay(new org.joda.time.Period().plusDays(1567))); + formatIntervalDay(Duration.ofDays(1567))); Assert.assertEquals("-1567 00:00:00.000", - formatIntervalDay(new org.joda.time.Period().minusDays(1567))); + formatIntervalDay(Duration.ofDays(-1567))); } @Test From 4ff1a29aae561cad6851a13666e4375a7645c6ef Mon Sep 17 00:00:00 2001 From: prmoore77 Date: Mon, 6 Nov 2023 14:55:28 -0500 Subject: [PATCH 04/57] GH-38460: [Java][FlightRPC] Add mTLS support for Flight SQL JDBC driver (#38461) ### Rationale for this change I wanted to add additional security capabilities to the Arrow Flight SQL JDBC driver so that it catches up to ADBC. ADBC already supports mTLS - and it is a great security feature. I wanted to bring this to the JDBC driver as well. ### What changes are included in this PR? This PR adds support for mTLS (client certificate verification/authentication) to the Arrow Flight SQL JDBC driver. ### Are these changes tested? Yes, I've added tests of the new mTLS functionality - and have ensured that the change is backward compatible by verifying all existing tests pass. ### Are there any user-facing changes? Yes - but the end-user documentation for the Arrow Flight SQL JDBC driver has been updated in the PR itself. * Closes: #38460 Lead-authored-by: prmoore77 Co-authored-by: David Li Signed-off-by: David Li --- docs/source/java/flight_sql_jdbc_driver.rst | 15 + .../org/apache/arrow/flight/FlightServer.java | 100 +++- .../driver/jdbc/ArrowFlightConnection.java | 3 + .../client/ArrowFlightSqlClientHandler.java | 54 ++- .../utils/ClientAuthenticationUtils.java | 56 ++- .../ArrowFlightConnectionConfigImpl.java | 15 + .../driver/jdbc/ConnectionMutualTlsTest.java | 427 ++++++++++++++++++ .../jdbc/ConnectionTlsRootCertsTest.java | 352 +++++++++++++++ .../driver/jdbc/FlightServerTestRule.java | 22 +- .../jdbc/utils/FlightSqlTestCertificates.java | 10 + 10 files changed, 1045 insertions(+), 9 deletions(-) create mode 100644 java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java create mode 100644 java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsRootCertsTest.java diff --git a/docs/source/java/flight_sql_jdbc_driver.rst b/docs/source/java/flight_sql_jdbc_driver.rst index 8268242302488..34ccfea47f9e3 100644 --- a/docs/source/java/flight_sql_jdbc_driver.rst +++ b/docs/source/java/flight_sql_jdbc_driver.rst @@ -114,6 +114,21 @@ case-sensitive. The supported parameters are: - null - When TLS is enabled, the password for the certificate store + * - tlsRootCerts + - null + - Path to PEM-encoded root certificates for TLS - use this as + an alternative to ``trustStore`` + + * - clientCertificate + - null + - Path to PEM-encoded client mTLS certificate when the Flight + SQL server requires client verification. + + * - clientKey + - null + - Path to PEM-encoded client mTLS key when the Flight + SQL server requires client verification. + * - useEncryption - true - Whether to use TLS (the default is an encrypted connection) diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java index 7f15798f6a326..234c9bdcaacc1 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java @@ -34,6 +34,8 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import javax.net.ssl.SSLException; + import org.apache.arrow.flight.auth.ServerAuthHandler; import org.apache.arrow.flight.auth.ServerAuthInterceptor; import org.apache.arrow.flight.auth2.Auth2Constants; @@ -49,9 +51,14 @@ import io.grpc.Server; import io.grpc.ServerInterceptors; +import io.grpc.netty.GrpcSslContexts; import io.grpc.netty.NettyServerBuilder; import io.netty.channel.EventLoopGroup; import io.netty.channel.ServerChannel; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; + /** * Generic server of flight data that is customized via construction with delegate classes for the @@ -172,6 +179,8 @@ public static final class Builder { private int maxInboundMessageSize = MAX_GRPC_MESSAGE_SIZE; private InputStream certChain; private InputStream key; + private InputStream mTlsCACert; + private SslContext sslContext; private final List> interceptors; // Keep track of inserted interceptors private final Set interceptorKeys; @@ -245,7 +254,25 @@ public FlightServer build() { } if (certChain != null) { - builder.useTransportSecurity(certChain, key); + SslContextBuilder sslContextBuilder = GrpcSslContexts + .forServer(certChain, key); + + if (mTlsCACert != null) { + sslContextBuilder + .clientAuth(ClientAuth.REQUIRE) + .trustManager(mTlsCACert); + } + try { + sslContext = sslContextBuilder.build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } finally { + closeMTlsCACert(); + closeCertChain(); + closeKey(); + } + + builder.sslContext(sslContext); } // Share one executor between the gRPC service, DoPut, and Handshake @@ -306,14 +333,69 @@ public Builder maxInboundMessageSize(int maxMessageSize) { return this; } + /** + * A small utility function to ensure that InputStream attributes. + * are closed if they are not null + * @param stream The InputStream to close (if it is not null). + */ + private void closeInputStreamIfNotNull(InputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException ignored) { + } + } + } + + /** + * A small utility function to ensure that the certChain attribute + * is closed if it is not null. It then sets the attribute to null. + */ + private void closeCertChain() { + closeInputStreamIfNotNull(certChain); + certChain = null; + } + + /** + * A small utility function to ensure that the key attribute + * is closed if it is not null. It then sets the attribute to null. + */ + private void closeKey() { + closeInputStreamIfNotNull(key); + key = null; + } + + /** + * A small utility function to ensure that the mTlsCACert attribute + * is closed if it is not null. It then sets the attribute to null. + */ + private void closeMTlsCACert() { + closeInputStreamIfNotNull(mTlsCACert); + mTlsCACert = null; + } + /** * Enable TLS on the server. * @param certChain The certificate chain to use. * @param key The private key to use. */ public Builder useTls(final File certChain, final File key) throws IOException { + closeCertChain(); this.certChain = new FileInputStream(certChain); + + closeKey(); this.key = new FileInputStream(key); + + return this; + } + + /** + * Enable Client Verification via mTLS on the server. + * @param mTlsCACert The CA certificate to use for verifying clients. + */ + public Builder useMTlsClientVerification(final File mTlsCACert) throws IOException { + closeMTlsCACert(); + this.mTlsCACert = new FileInputStream(mTlsCACert); return this; } @@ -322,9 +404,23 @@ public Builder useTls(final File certChain, final File key) throws IOException { * @param certChain The certificate chain to use. * @param key The private key to use. */ - public Builder useTls(final InputStream certChain, final InputStream key) { + public Builder useTls(final InputStream certChain, final InputStream key) throws IOException { + closeCertChain(); this.certChain = certChain; + + closeKey(); this.key = key; + + return this; + } + + /** + * Enable mTLS on the server. + * @param mTlsCACert The CA certificate to use for verifying clients. + */ + public Builder useMTlsClientVerification(final InputStream mTlsCACert) throws IOException { + closeMTlsCACert(); + this.mTlsCACert = mTlsCACert; return this; } diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index 79bc04d27fe01..fdbb9381c0a55 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -101,6 +101,9 @@ private static ArrowFlightSqlClientHandler createNewClientHandler( .withTrustStorePath(config.getTrustStorePath()) .withTrustStorePassword(config.getTrustStorePassword()) .withSystemTrustStore(config.useSystemTrustStore()) + .withTlsRootCertificates(config.getTlsRootCertificatesPath()) + .withClientCertificate(config.getClientCertificatePath()) + .withClientKey(config.getClientKeyPath()) .withBufferAllocator(allocator) .withEncryption(config.useEncryption()) .withDisableCertificateVerification(config.getDisableCertificateVerification()) diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 66372092b8e99..75e80d45dc669 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -433,6 +433,9 @@ public static final class Builder { private boolean useEncryption; private boolean disableCertificateVerification; private boolean useSystemTrustStore; + private String tlsRootCertificatesPath; + private String clientCertificatePath; + private String clientKeyPath; private BufferAllocator allocator; public Builder() { @@ -457,6 +460,9 @@ private Builder(Builder original) { this.useEncryption = original.useEncryption; this.disableCertificateVerification = original.disableCertificateVerification; this.useSystemTrustStore = original.useSystemTrustStore; + this.tlsRootCertificatesPath = original.tlsRootCertificatesPath; + this.clientCertificatePath = original.clientCertificatePath; + this.clientKeyPath = original.clientKeyPath; this.allocator = original.allocator; } @@ -560,7 +566,42 @@ public Builder withSystemTrustStore(final boolean useSystemTrustStore) { } /** - * Sets the token used in the token authetication. + * Sets the TLS root certificate path as an alternative to using the System + * or other Trust Store. The path must contain a valid PEM file. + * + * @param tlsRootCertificatesPath the TLS root certificate path (if TLS is required). + * @return this instance. + */ + public Builder withTlsRootCertificates(final String tlsRootCertificatesPath) { + this.tlsRootCertificatesPath = tlsRootCertificatesPath; + return this; + } + + /** + * Sets the mTLS client certificate path (if mTLS is required). + * + * @param clientCertificatePath the mTLS client certificate path (if mTLS is required). + * @return this instance. + */ + public Builder withClientCertificate(final String clientCertificatePath) { + this.clientCertificatePath = clientCertificatePath; + return this; + } + + /** + * Sets the mTLS client certificate private key path (if mTLS is required). + * + * @param clientKeyPath the mTLS client certificate private key path (if mTLS is required). + * @return this instance. + */ + public Builder withClientKey(final String clientKeyPath) { + this.clientKeyPath = clientKeyPath; + return this; + } + + /** + * Sets the token used in the token authentication. + * * @param token the token value. * @return this builder instance. */ @@ -660,7 +701,10 @@ public ArrowFlightSqlClientHandler build() throws SQLException { if (disableCertificateVerification) { clientBuilder.verifyServer(false); } else { - if (useSystemTrustStore) { + if (tlsRootCertificatesPath != null) { + clientBuilder.trustedCertificates( + ClientAuthenticationUtils.getTlsRootCertificatesStream(tlsRootCertificatesPath)); + } else if (useSystemTrustStore) { clientBuilder.trustedCertificates( ClientAuthenticationUtils.getCertificateInputStreamFromSystem(trustStorePassword)); } else if (trustStorePath != null) { @@ -668,6 +712,12 @@ public ArrowFlightSqlClientHandler build() throws SQLException { ClientAuthenticationUtils.getCertificateStream(trustStorePath, trustStorePassword)); } } + + if (clientCertificatePath != null && clientKeyPath != null) { + clientBuilder.clientCertificate( + ClientAuthenticationUtils.getClientCertificateStream(clientCertificatePath), + ClientAuthenticationUtils.getClientKeyStream(clientKeyPath)); + } } client = clientBuilder.build(); diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/utils/ClientAuthenticationUtils.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/utils/ClientAuthenticationUtils.java index 6d9880bd27048..d50dc385a62e1 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/utils/ClientAuthenticationUtils.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/utils/ClientAuthenticationUtils.java @@ -227,14 +227,64 @@ public static InputStream getCertificateStream(final String keyStorePath, final KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); try (final InputStream keyStoreStream = Files - .newInputStream(Paths.get(Preconditions.checkNotNull(keyStorePath)))) { - keyStore.load(keyStoreStream, - Preconditions.checkNotNull(keyStorePass).toCharArray()); + .newInputStream(Paths.get(keyStorePath))) { + keyStore.load(keyStoreStream, keyStorePass.toCharArray()); } return getSingleCertificateInputStream(keyStore); } + /** + * Generates an {@link InputStream} that contains certificates for path-based + * TLS Root Certificates. + * + * @param tlsRootsCertificatesPath The path of the TLS Root Certificates. + * @return a new {code InputStream} containing the certificates. + * @throws GeneralSecurityException on error. + * @throws IOException on error. + */ + public static InputStream getTlsRootCertificatesStream(final String tlsRootsCertificatesPath) + throws GeneralSecurityException, IOException { + Preconditions.checkNotNull(tlsRootsCertificatesPath, "TLS Root certificates path cannot be null!"); + + return Files + .newInputStream(Paths.get(tlsRootsCertificatesPath)); + } + + /** + * Generates an {@link InputStream} that contains certificates for a path-based + * mTLS Client Certificate. + * + * @param clientCertificatePath The path of the mTLS Client Certificate. + * @return a new {code InputStream} containing the certificates. + * @throws GeneralSecurityException on error. + * @throws IOException on error. + */ + public static InputStream getClientCertificateStream(final String clientCertificatePath) + throws GeneralSecurityException, IOException { + Preconditions.checkNotNull(clientCertificatePath, "Client certificate path cannot be null!"); + + return Files + .newInputStream(Paths.get(clientCertificatePath)); + } + + /** + * Generates an {@link InputStream} that contains certificates for a path-based + * mTLS Client Key. + * + * @param clientKeyPath The path of the mTLS Client Key. + * @return a new {code InputStream} containing the certificates. + * @throws GeneralSecurityException on error. + * @throws IOException on error. + */ + public static InputStream getClientKeyStream(final String clientKeyPath) + throws GeneralSecurityException, IOException { + Preconditions.checkNotNull(clientKeyPath, "Client key path cannot be null!"); + + return Files + .newInputStream(Paths.get(clientKeyPath)); + } + private static InputStream getSingleCertificateInputStream(KeyStore keyStore) throws KeyStoreException, IOException, CertificateException { final Enumeration aliases = keyStore.aliases(); diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java index ac338a85d6292..59118e1d6f788 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java @@ -109,6 +109,18 @@ public boolean useSystemTrustStore() { return ArrowFlightConnectionProperty.USE_SYSTEM_TRUST_STORE.getBoolean(properties); } + public String getTlsRootCertificatesPath() { + return ArrowFlightConnectionProperty.TLS_ROOT_CERTS.getString(properties); + } + + public String getClientCertificatePath() { + return ArrowFlightConnectionProperty.CLIENT_CERTIFICATE.getString(properties); + } + + public String getClientKeyPath() { + return ArrowFlightConnectionProperty.CLIENT_KEY.getString(properties); + } + /** * Whether to use TLS encryption. * @@ -175,6 +187,9 @@ public enum ArrowFlightConnectionProperty implements ConnectionProperty { TRUST_STORE("trustStore", null, Type.STRING, false), TRUST_STORE_PASSWORD("trustStorePassword", null, Type.STRING, false), USE_SYSTEM_TRUST_STORE("useSystemTrustStore", true, Type.BOOLEAN, false), + TLS_ROOT_CERTS("tlsRootCerts", null, Type.STRING, false), + CLIENT_CERTIFICATE("clientCertificate", null, Type.STRING, false), + CLIENT_KEY("clientKey", null, Type.STRING, false), THREAD_POOL_SIZE("threadPoolSize", 1, Type.NUMBER, false), TOKEN("token", null, Type.STRING, false); diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java new file mode 100644 index 0000000000000..783e0c41e9269 --- /dev/null +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.driver.jdbc; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; + +import java.io.File; +import java.net.URLEncoder; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +import org.apache.arrow.driver.jdbc.authentication.UserPasswordAuthentication; +import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler; +import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; +import org.apache.arrow.driver.jdbc.utils.FlightSqlTestCertificates; +import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.calcite.avatica.org.apache.http.auth.UsernamePasswordCredentials; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + +/** + * Tests encrypted connections. + */ +public class ConnectionMutualTlsTest { + + @ClassRule + public static final FlightServerTestRule FLIGHT_SERVER_TEST_RULE; + private static final String tlsRootCertsPath; + + private static final String serverMTlsCACertPath; + private static final String clientMTlsCertPath; + private static final String badClientMTlsCertPath; + private static final String clientMTlsKeyPath; + private static final String badClientMTlsKeyPath; + private static final MockFlightSqlProducer PRODUCER = new MockFlightSqlProducer(); + private static final String userTest = "user1"; + private static final String passTest = "pass1"; + + static { + final FlightSqlTestCertificates.CertKeyPair + certKey = FlightSqlTestCertificates.exampleTlsCerts().get(0); + + tlsRootCertsPath = certKey.cert.getPath(); + + final File serverMTlsCACert = FlightSqlTestCertificates.exampleCACert(); + + serverMTlsCACertPath = serverMTlsCACert.getPath(); + + final FlightSqlTestCertificates.CertKeyPair + clientMTlsCertKey = FlightSqlTestCertificates.exampleTlsCerts().get(1); + + clientMTlsCertPath = clientMTlsCertKey.cert.getPath(); + clientMTlsKeyPath = clientMTlsCertKey.key.getPath(); + + badClientMTlsCertPath = clientMTlsCertPath + ".bad"; + badClientMTlsKeyPath = clientMTlsKeyPath + ".bad"; + + UserPasswordAuthentication authentication = new UserPasswordAuthentication.Builder() + .user(userTest, passTest) + .build(); + + FLIGHT_SERVER_TEST_RULE = new FlightServerTestRule.Builder() + .authentication(authentication) + .useEncryption(certKey.cert, certKey.key) + .useMTlsClientVerification(serverMTlsCACert) + .producer(PRODUCER) + .build(); + } + + private BufferAllocator allocator; + + @Before + public void setUp() throws Exception { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @After + public void tearDown() throws Exception { + allocator.getChildAllocators().forEach(BufferAllocator::close); + AutoCloseables.close(allocator); + } + + /** + * Try to instantiate an encrypted FlightClient. + * + * @throws Exception on error. + */ + @Test + public void testGetEncryptedClientAuthenticated() throws Exception { + final UsernamePasswordCredentials credentials = new UsernamePasswordCredentials( + userTest, passTest); + + try (ArrowFlightSqlClientHandler client = + new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withUsername(credentials.getUserName()) + .withPassword(credentials.getPassword()) + .withTlsRootCertificates(tlsRootCertsPath) + .withClientCertificate(clientMTlsCertPath) + .withClientKey(clientMTlsKeyPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()) { + assertNotNull(client); + } + } + + /** + * Try to instantiate an encrypted FlightClient providing a bad mTLS Cert Path. It's expected to + * receive the SQLException. + */ + @Test + public void testGetEncryptedClientWithBadMTlsCertPath() { + final UsernamePasswordCredentials credentials = new UsernamePasswordCredentials( + userTest, passTest); + + assertThrows(SQLException.class, () -> new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withUsername(credentials.getUserName()) + .withPassword(credentials.getPassword()) + .withTlsRootCertificates(tlsRootCertsPath) + .withClientCertificate(badClientMTlsCertPath) + .withClientKey(clientMTlsKeyPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()); + } + + /** + * Try to instantiate an encrypted FlightClient providing a bad mTLS Key Path. It's expected to + * receive the SQLException. + */ + @Test + public void testGetEncryptedClientWithBadMTlsKeyPath() { + final UsernamePasswordCredentials credentials = new UsernamePasswordCredentials( + userTest, passTest); + + assertThrows(SQLException.class, () -> new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withUsername(credentials.getUserName()) + .withPassword(credentials.getPassword()) + .withTlsRootCertificates(tlsRootCertsPath) + .withClientCertificate(clientMTlsCertPath) + .withClientKey(badClientMTlsKeyPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()); + } + + /** + * Try to instantiate an encrypted FlightClient without credentials. + * + * @throws Exception on error. + */ + @Test + public void testGetNonAuthenticatedEncryptedClientNoAuth() throws Exception { + try (ArrowFlightSqlClientHandler client = + new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withTlsRootCertificates(tlsRootCertsPath) + .withClientCertificate(clientMTlsCertPath) + .withClientKey(clientMTlsKeyPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()) { + assertNotNull(client); + } + } + + /** + * Check if an encrypted connection can be established successfully when the + * provided valid credentials and a valid TLS Root Certs path. + * + * @throws Exception on error. + */ + @Test + public void testGetEncryptedConnectionWithValidCredentialsAndTlsRootsPath() throws Exception { + final Properties properties = new Properties(); + + properties.put(ArrowFlightConnectionProperty.HOST.camelName(), "localhost"); + properties.put(ArrowFlightConnectionProperty.PORT.camelName(), + FLIGHT_SERVER_TEST_RULE.getPort()); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), + userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), + passTest); + properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), + tlsRootCertsPath); + properties.put(ArrowFlightConnectionProperty.CLIENT_CERTIFICATE.camelName(), + clientMTlsCertPath); + properties.put(ArrowFlightConnectionProperty.CLIENT_KEY.camelName(), + clientMTlsKeyPath); + + final ArrowFlightJdbcDataSource dataSource = + ArrowFlightJdbcDataSource.createNewDataSource(properties); + try (final Connection connection = dataSource.getConnection()) { + assert connection.isValid(300); + } + } + + /** + * Check if an encrypted connection can be established successfully when not + * providing authentication. + * + * @throws Exception on error. + */ + @Test + public void testGetNonAuthenticatedEncryptedConnection() throws Exception { + final Properties properties = new Properties(); + + properties.put(ArrowFlightConnectionProperty.HOST.camelName(), FLIGHT_SERVER_TEST_RULE.getHost()); + properties.put(ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_RULE.getPort()); + properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), true); + properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + properties.put(ArrowFlightConnectionProperty.CLIENT_CERTIFICATE.camelName(), clientMTlsCertPath); + properties.put(ArrowFlightConnectionProperty.CLIENT_KEY.camelName(), clientMTlsKeyPath); + + final ArrowFlightJdbcDataSource dataSource = ArrowFlightJdbcDataSource.createNewDataSource(properties); + try (final Connection connection = dataSource.getConnection()) { + assert connection.isValid(300); + } + } + + /** + * Check if an encrypted connection can be established successfully when connecting through + * the DriverManager using just a connection url. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueCorrectCastUrlWithDriverManager() throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + final String jdbcUrl = String.format( + "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s" + + "&useEncryption=true&%s=%s&%s=%s&%s=%s", + FLIGHT_SERVER_TEST_RULE.getPort(), + userTest, + passTest, + ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), + URLEncoder.encode(tlsRootCertsPath, "UTF-8"), + ArrowFlightConnectionProperty.CLIENT_CERTIFICATE.camelName(), + URLEncoder.encode(clientMTlsCertPath, "UTF-8"), + ArrowFlightConnectionProperty.CLIENT_KEY.camelName(), + URLEncoder.encode(clientMTlsKeyPath, "UTF-8")); + + try (Connection connection = DriverManager.getConnection(jdbcUrl)) { + Assert.assertTrue(connection.isValid(0)); + } + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * a connection url and properties with String K-V pairs. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueCorrectCastUrlAndPropertiesUsingSetPropertyWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + Properties properties = new Properties(); + + properties.setProperty(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.setProperty(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.setProperty(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "true"); + properties.setProperty(ArrowFlightConnectionProperty.CLIENT_CERTIFICATE.camelName(), clientMTlsCertPath); + properties.setProperty(ArrowFlightConnectionProperty.CLIENT_KEY.camelName(), clientMTlsKeyPath); + + final String jdbcUrl = String.format( + "jdbc:arrow-flight-sql://localhost:%s", + FLIGHT_SERVER_TEST_RULE.getPort()); + + try (Connection connection = DriverManager.getConnection(jdbcUrl, properties)) { + Assert.assertTrue(connection.isValid(0)); + } + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * a connection url and properties with Object K-V pairs. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueCorrectCastUrlAndPropertiesUsingPutWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + Properties properties = new Properties(); + + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), true); + properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + properties.put(ArrowFlightConnectionProperty.CLIENT_CERTIFICATE.camelName(), clientMTlsCertPath); + properties.put(ArrowFlightConnectionProperty.CLIENT_KEY.camelName(), clientMTlsKeyPath); + + final String jdbcUrl = String.format( + "jdbc:arrow-flight-sql://localhost:%s", + FLIGHT_SERVER_TEST_RULE.getPort()); + + try (Connection connection = DriverManager.getConnection(jdbcUrl, properties)) { + Assert.assertTrue(connection.isValid(0)); + } + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * just a connection url and using 0 and 1 as ssl values. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + final String jdbcUrl = String.format( + "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s" + + "&useEncryption=1&useSystemTrustStore=0&%s=%s&%s=%s&%s=%s", + FLIGHT_SERVER_TEST_RULE.getPort(), + userTest, + passTest, + ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), + URLEncoder.encode(tlsRootCertsPath, "UTF-8"), + ArrowFlightConnectionProperty.CLIENT_CERTIFICATE.camelName(), + URLEncoder.encode(clientMTlsCertPath, "UTF-8"), + ArrowFlightConnectionProperty.CLIENT_KEY.camelName(), + URLEncoder.encode(clientMTlsKeyPath, "UTF-8")); + + try (Connection connection = DriverManager.getConnection(jdbcUrl)) { + Assert.assertTrue(connection.isValid(0)); + } + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * a connection url and properties with String K-V pairs and using 0 and 1 as ssl values. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlAndPropertiesUsingSetPropertyWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + Properties properties = new Properties(); + + properties.setProperty(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.setProperty(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.setProperty(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "1"); + properties.setProperty(ArrowFlightConnectionProperty.CLIENT_CERTIFICATE.camelName(), clientMTlsCertPath); + properties.setProperty(ArrowFlightConnectionProperty.CLIENT_KEY.camelName(), clientMTlsKeyPath); + + final String jdbcUrl = String.format("jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()); + + try (Connection connection = DriverManager.getConnection(jdbcUrl, properties)) { + Assert.assertTrue(connection.isValid(0)); + } + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * a connection url and properties with Object K-V pairs and using 0 and 1 as ssl values. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlAndPropertiesUsingPutWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + Properties properties = new Properties(); + + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), 1); + properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + properties.put(ArrowFlightConnectionProperty.CLIENT_CERTIFICATE.camelName(), clientMTlsCertPath); + properties.put(ArrowFlightConnectionProperty.CLIENT_KEY.camelName(), clientMTlsKeyPath); + + final String jdbcUrl = String.format("jdbc:arrow-flight-sql://localhost:%s", + FLIGHT_SERVER_TEST_RULE.getPort()); + + try (Connection connection = DriverManager.getConnection(jdbcUrl, properties)) { + Assert.assertTrue(connection.isValid(0)); + } + } +} diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsRootCertsTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsRootCertsTest.java new file mode 100644 index 0000000000000..333c474824e52 --- /dev/null +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsRootCertsTest.java @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.driver.jdbc; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; + +import java.net.URLEncoder; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +import org.apache.arrow.driver.jdbc.authentication.UserPasswordAuthentication; +import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler; +import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; +import org.apache.arrow.driver.jdbc.utils.FlightSqlTestCertificates; +import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.calcite.avatica.org.apache.http.auth.UsernamePasswordCredentials; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + + +/** + * Tests encrypted connections. + */ +public class ConnectionTlsRootCertsTest { + + @ClassRule + public static final FlightServerTestRule FLIGHT_SERVER_TEST_RULE; + private static final String tlsRootCertsPath; + private static final String badTlsRootCertsPath; + private static final MockFlightSqlProducer PRODUCER = new MockFlightSqlProducer(); + private static final String userTest = "user1"; + private static final String passTest = "pass1"; + + static { + final FlightSqlTestCertificates.CertKeyPair + certKey = FlightSqlTestCertificates.exampleTlsCerts().get(0); + + tlsRootCertsPath = certKey.cert.getPath(); + + badTlsRootCertsPath = certKey.cert.getPath() + ".bad"; + + UserPasswordAuthentication authentication = new UserPasswordAuthentication.Builder() + .user(userTest, passTest) + .build(); + + FLIGHT_SERVER_TEST_RULE = new FlightServerTestRule.Builder() + .authentication(authentication) + .useEncryption(certKey.cert, certKey.key) + .producer(PRODUCER) + .build(); + } + + private BufferAllocator allocator; + + @Before + public void setUp() throws Exception { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @After + public void tearDown() throws Exception { + allocator.getChildAllocators().forEach(BufferAllocator::close); + AutoCloseables.close(allocator); + } + + /** + * Try to instantiate an encrypted FlightClient. + * + * @throws Exception on error. + */ + @Test + public void testGetEncryptedClientAuthenticated() throws Exception { + final UsernamePasswordCredentials credentials = new UsernamePasswordCredentials( + userTest, passTest); + + try (ArrowFlightSqlClientHandler client = + new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withUsername(credentials.getUserName()) + .withPassword(credentials.getPassword()) + .withTlsRootCertificates(tlsRootCertsPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()) { + assertNotNull(client); + } + } + + /** + * Try to instantiate an encrypted FlightClient providing a bad TLS Root Certs Path. It's expected to + * receive the SQLException. + */ + @Test + public void testGetEncryptedClientWithNoCertificateOnKeyStore() { + assertThrows(SQLException.class, () -> new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withTlsRootCertificates(badTlsRootCertsPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()); + } + + /** + * Try to instantiate an encrypted FlightClient without credentials. + * + * @throws Exception on error. + */ + @Test + public void testGetNonAuthenticatedEncryptedClientNoAuth() throws Exception { + try (ArrowFlightSqlClientHandler client = + new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withTlsRootCertificates(tlsRootCertsPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()) { + assertNotNull(client); + } + } + + /** + * Check if an encrypted connection can be established successfully when the + * provided valid credentials and a valid TLS Root Certs path. + * + * @throws Exception on error. + */ + @Test + public void testGetEncryptedConnectionWithValidCredentialsAndTlsRootsPath() throws Exception { + final Properties properties = new Properties(); + + properties.put(ArrowFlightConnectionProperty.HOST.camelName(), "localhost"); + properties.put(ArrowFlightConnectionProperty.PORT.camelName(), + FLIGHT_SERVER_TEST_RULE.getPort()); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), + userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), + passTest); + properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), + tlsRootCertsPath); + + final ArrowFlightJdbcDataSource dataSource = + ArrowFlightJdbcDataSource.createNewDataSource(properties); + try (final Connection connection = dataSource.getConnection()) { + assert connection.isValid(300); + } + } + + /** + * Check if an encrypted connection can be established successfully when not + * providing authentication. + * + * @throws Exception on error. + */ + @Test + public void testGetNonAuthenticatedEncryptedConnection() throws Exception { + final Properties properties = new Properties(); + + properties.put(ArrowFlightConnectionProperty.HOST.camelName(), FLIGHT_SERVER_TEST_RULE.getHost()); + properties.put(ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_RULE.getPort()); + properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), true); + properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + + final ArrowFlightJdbcDataSource dataSource = ArrowFlightJdbcDataSource.createNewDataSource(properties); + try (final Connection connection = dataSource.getConnection()) { + assert connection.isValid(300); + } + } + + /** + * Check if an encrypted connection can be established successfully when connecting through + * the DriverManager using just a connection url. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueCorrectCastUrlWithDriverManager() throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + final Connection connection = DriverManager.getConnection( + String.format( + "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s" + + "&useEncryption=true&%s=%s", + FLIGHT_SERVER_TEST_RULE.getPort(), + userTest, + passTest, + ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), + URLEncoder.encode(tlsRootCertsPath, "UTF-8"))); + Assert.assertTrue(connection.isValid(0)); + connection.close(); + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * a connection url and properties with String K-V pairs. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueCorrectCastUrlAndPropertiesUsingSetPropertyWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + Properties properties = new Properties(); + + properties.setProperty(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.setProperty(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.setProperty(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "true"); + + final Connection connection = DriverManager.getConnection( + String.format( + "jdbc:arrow-flight-sql://localhost:%s", + FLIGHT_SERVER_TEST_RULE.getPort()), + properties); + Assert.assertTrue(connection.isValid(0)); + connection.close(); + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * a connection url and properties with Object K-V pairs. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueCorrectCastUrlAndPropertiesUsingPutWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + Properties properties = new Properties(); + + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), true); + properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + + final Connection connection = DriverManager.getConnection( + String.format( + "jdbc:arrow-flight-sql://localhost:%s", + FLIGHT_SERVER_TEST_RULE.getPort()), + properties); + Assert.assertTrue(connection.isValid(0)); + connection.close(); + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * just a connection url and using 0 and 1 as ssl values. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + final Connection connection = DriverManager.getConnection( + String.format( + "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s" + + "&useEncryption=1&useSystemTrustStore=0&%s=%s", + FLIGHT_SERVER_TEST_RULE.getPort(), + userTest, + passTest, + ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), + URLEncoder.encode(tlsRootCertsPath, "UTF-8"))); + Assert.assertTrue(connection.isValid(0)); + connection.close(); + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * a connection url and properties with String K-V pairs and using 0 and 1 as ssl values. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlAndPropertiesUsingSetPropertyWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + Properties properties = new Properties(); + + properties.setProperty(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.setProperty(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.setProperty(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "1"); + + final Connection connection = DriverManager.getConnection( + String.format("jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), + properties); + Assert.assertTrue(connection.isValid(0)); + connection.close(); + } + + /** + * Check if an encrypted connection can be established successfully when connecting through the DriverManager using + * a connection url and properties with Object K-V pairs and using 0 and 1 as ssl values. + * + * @throws Exception on error. + */ + @Test + public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlAndPropertiesUsingPutWithDriverManager() + throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + DriverManager.registerDriver(driver); + + Properties properties = new Properties(); + + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), 1); + properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); + + final Connection connection = DriverManager.getConnection( + String.format("jdbc:arrow-flight-sql://localhost:%s", + FLIGHT_SERVER_TEST_RULE.getPort()), + properties); + Assert.assertTrue(connection.isValid(0)); + connection.close(); + } +} diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestRule.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestRule.java index 733145892ec3e..df7cbea56ee2f 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestRule.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestRule.java @@ -63,6 +63,7 @@ public class FlightServerTestRule implements TestRule, AutoCloseable { private final FlightSqlProducer producer; private final Authentication authentication; private final CertKeyPair certKeyPair; + private final File mTlsCACert; private final MiddlewareCookie.Factory middlewareCookieFactory = new MiddlewareCookie.Factory(); @@ -71,13 +72,15 @@ private FlightServerTestRule(final Properties properties, final BufferAllocator allocator, final FlightSqlProducer producer, final Authentication authentication, - final CertKeyPair certKeyPair) { + final CertKeyPair certKeyPair, + final File mTlsCACert) { this.properties = Preconditions.checkNotNull(properties); this.config = Preconditions.checkNotNull(config); this.allocator = Preconditions.checkNotNull(allocator); this.producer = Preconditions.checkNotNull(producer); this.authentication = authentication; this.certKeyPair = certKeyPair; + this.mTlsCACert = mTlsCACert; } /** @@ -142,6 +145,9 @@ private FlightServer initiateServer(Location location) throws IOException { if (certKeyPair != null) { builder.useTls(certKeyPair.cert, certKeyPair.key); } + if (mTlsCACert != null) { + builder.useMTlsClientVerification(mTlsCACert); + } return builder.build(); } @@ -212,6 +218,7 @@ public static final class Builder { private FlightSqlProducer producer; private Authentication authentication; private CertKeyPair certKeyPair; + private File mTlsCACert; public Builder() { this.properties = new Properties(); @@ -254,6 +261,17 @@ public Builder useEncryption(final File certChain, final File key) { return this; } + /** + * Enable Client Verification via mTLS on the server. + * + * @param mTlsCACert The CA certificate to use for client verification. + * @return the Builder. + */ + public Builder useMTlsClientVerification(final File mTlsCACert) { + this.mTlsCACert = mTlsCACert; + return this; + } + /** * Builds the {@link FlightServerTestRule} using the provided values. * @@ -262,7 +280,7 @@ public Builder useEncryption(final File certChain, final File key) { public FlightServerTestRule build() { authentication.populateProperties(properties); return new FlightServerTestRule(properties, new ArrowFlightConnectionConfigImpl(properties), - new RootAllocator(Long.MAX_VALUE), producer, authentication, certKeyPair); + new RootAllocator(Long.MAX_VALUE), producer, authentication, certKeyPair, mTlsCACert); } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FlightSqlTestCertificates.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FlightSqlTestCertificates.java index a2b1864c02657..89e27be9f53da 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FlightSqlTestCertificates.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FlightSqlTestCertificates.java @@ -51,6 +51,16 @@ static Path getFlightTestDataRoot() { return getTestDataRoot().resolve("flight"); } + /** + * Create File object with the CA certificate. + * + * @return A File containing the Root CA certificate. + */ + public static File exampleCACert() { + final Path root = getFlightTestDataRoot(); + return root.resolve("root-ca.pem").toFile(); + } + /** * Create CertKeyPair object with the certificates and keys. * From 09e5e6a0020300a73c8e95ec279eea39a52a9ef8 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 6 Nov 2023 13:15:14 -0800 Subject: [PATCH 05/57] GH-36912: [Java] JDBC driver stops consuming roots if it sees an empty root (#38590) ### Rationale for this change The JDBC driver for Flight SQL incorrectly stops iterating on a FlightStream if it encounters an empty root. ### What changes are included in this PR? The driver now waits until FlightStream#next() returns false to determine if a FlightStream is fully-consumed instead of an empty root. ### Are these changes tested? Yes, new tests have been written. ### Are there any user-facing changes? No * Closes: #36912 Authored-by: James Duong Signed-off-by: David Li --- .../jdbc/utils/FlightEndpointDataQueue.java | 13 ++++--- .../arrow/driver/jdbc/ResultSetTest.java | 13 +++++++ .../jdbc/utils/CoreMockedSqlProducers.java | 36 +++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/FlightEndpointDataQueue.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/FlightEndpointDataQueue.java index 71cafd2ec3075..1198d89c40aef 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/FlightEndpointDataQueue.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/FlightEndpointDataQueue.java @@ -104,9 +104,8 @@ private CloseableEndpointStreamPair next(final EndpointStreamSupplier endpointSt futures.remove(future); try { final CloseableEndpointStreamPair endpoint = future.get(); - // Get the next FlightStream with content. - // The stream is non-empty. - if (endpoint.getStream().getRoot().getRowCount() > 0) { + // Get the next FlightStream that has a root with content. + if (endpoint != null) { return endpoint; } } catch (final ExecutionException | InterruptedException | CancellationException e) { @@ -178,8 +177,12 @@ public synchronized void enqueue(final CloseableEndpointStreamPair endpointReque endpointsToClose.add(endpointRequest); futures.add(completionService.submit(() -> { // `FlightStream#next` will block until new data can be read or stream is over. - endpointRequest.getStream().next(); - return endpointRequest; + while (endpointRequest.getStream().next()) { + if (endpointRequest.getStream().getRoot().getRowCount() > 0) { + return endpointRequest; + } + } + return null; })); } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java index e2ac100b8dc36..5b9e269fb3cc8 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java @@ -455,4 +455,17 @@ allocator, forGrpcInsecure("localhost", 0), rootProducer) } } } + + @Test + public void testShouldRunSelectQueryWithEmptyVectorsEmbedded() throws Exception { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery( + CoreMockedSqlProducers.LEGACY_REGULAR_WITH_EMPTY_SQL_CMD)) { + long rowCount = 0; + while (resultSet.next()) { + ++rowCount; + } + assertEquals(2, rowCount); + } + } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java index cf359849a7105..a8e2e7f2e4ce5 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java @@ -64,6 +64,7 @@ public final class CoreMockedSqlProducers { public static final String LEGACY_REGULAR_SQL_CMD = "SELECT * FROM TEST"; public static final String LEGACY_METADATA_SQL_CMD = "SELECT * FROM METADATA"; public static final String LEGACY_CANCELLATION_SQL_CMD = "SELECT * FROM TAKES_FOREVER"; + public static final String LEGACY_REGULAR_WITH_EMPTY_SQL_CMD = "SELECT * FROM TEST_EMPTIES"; private CoreMockedSqlProducers() { // Prevent instantiation. @@ -80,9 +81,44 @@ public static MockFlightSqlProducer getLegacyProducer() { addLegacyRegularSqlCmdSupport(producer); addLegacyMetadataSqlCmdSupport(producer); addLegacyCancellationSqlCmdSupport(producer); + addQueryWithEmbeddedEmptyRoot(producer); return producer; } + private static void addQueryWithEmbeddedEmptyRoot(final MockFlightSqlProducer producer) { + final Schema querySchema = new Schema(ImmutableList.of( + new Field( + "ID", + new FieldType(true, new ArrowType.Int(64, true), + null), + null) + )); + + final List> resultProducers = new ArrayList<>(); + Consumer dataRoot = listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(querySchema, allocator)) { + root.allocateNew(); + root.setRowCount(0); + listener.start(root); + listener.putNext(); // empty root + ((BigIntVector) root.getVector("ID")).setSafe(0, 100L); + root.setRowCount(1); + listener.putNext(); // data root + root.clear(); + root.setRowCount(0); + listener.putNext(); // empty root + ((BigIntVector) root.getVector("ID")).setSafe(0, 100L); + root.setRowCount(1); + listener.putNext(); // data root + } finally { + listener.completed(); + } + }; + resultProducers.add(dataRoot); + producer.addSelectQuery(LEGACY_REGULAR_WITH_EMPTY_SQL_CMD, querySchema, resultProducers); + } + private static void addLegacyRegularSqlCmdSupport(final MockFlightSqlProducer producer) { final Schema querySchema = new Schema(ImmutableList.of( new Field( From c131feae965b510a1c1b045c1b670fbf5a7da9d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 07:09:05 +0900 Subject: [PATCH 06/57] MINOR: [JS] Bump rollup and @rollup/stream in /js (#38595) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [rollup](https://github.com/rollup/rollup) and [@ rollup/stream](https://github.com/rollup/stream). These dependencies needed to be updated together. Updates `rollup` from 3.29.4 to 4.3.0
Release notes

Sourced from rollup's releases.

v4.3.0

4.3.0

2023-11-03

Features

  • Add forceExit CLI flag for situations where the CLI does not exit gracefully (#5195)

Bug Fixes

  • Properly catch errors when removing a temporary config file fails (#5198)

Pull Requests

v4.2.0

4.2.0

2023-10-31

Features

  • Run parsing in multiple threads and introduce parseAstAsync helper function (#5202)

Pull Requests

v4.1.6

4.1.6

2023-10-31

Bug Fixes

  • Fix a bug where emtpy block comments in certain positions could freeze Rollup (#5231)

Pull Requests

v4.1.5

... (truncated)

Changelog

Sourced from rollup's changelog.

4.3.0

2023-11-03

Features

  • Add forceExit CLI flag for situations where the CLI does not exit gracefully (#5195)

Bug Fixes

  • Properly catch errors when removing a temporary config file fails (#5198)

Pull Requests

4.2.0

2023-10-31

Features

  • Run parsing in multiple threads and introduce parseAstAsync helper function (#5202)

Pull Requests

4.1.6

2023-10-31

Bug Fixes

  • Fix a bug where emtpy block comments in certain positions could freeze Rollup (#5231)

Pull Requests

4.1.5

2023-10-28

... (truncated)

Commits

Updates `@ rollup/stream` from 3.0.0 to 3.0.1
Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/package.json | 4 +-- js/yarn.lock | 88 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/js/package.json b/js/package.json index e7ef0f895aa1d..cca333ad3d006 100644 --- a/js/package.json +++ b/js/package.json @@ -68,7 +68,7 @@ "@openpgp/web-stream-tools": "0.0.13", "@rollup/plugin-alias": "5.0.0", "@rollup/plugin-node-resolve": "15.1.0", - "@rollup/stream": "3.0.0", + "@rollup/stream": "3.0.1", "@swc/core": "1.3.82", "@types/benchmark": "2.1.4", "@types/glob": "8.1.0", @@ -106,7 +106,7 @@ "multistream": "4.1.0", "randomatic": "3.1.1", "regenerator-runtime": "0.14.0", - "rollup": "3.29.4", + "rollup": "4.3.0", "rxjs": "7.8.1", "ts-jest": "29.1.1", "ts-node": "10.9.1", diff --git a/js/yarn.lock b/js/yarn.lock index 40d5ef5f16141..b2ed3ed20600b 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1032,10 +1032,70 @@ estree-walker "^2.0.2" picomatch "^2.3.1" -"@rollup/stream@3.0.0": - version "3.0.0" - resolved "https://registry.npmjs.org/@rollup/stream/-/stream-3.0.0.tgz#8578177aa992e1e7d932d7b8b9ac302ce272663e" - integrity sha512-rCctaa32QWDmbO/NyZPE3fKbpi4RHmR3N/mPvCYaig7ieSsDg4uSudKkIsaDHuLuYpvNII8kPwgXs1k077lAPg== +"@rollup/rollup-android-arm-eabi@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.3.0.tgz#8ad8a660b18f1a24ad4a272738a65ac4788a8811" + integrity sha512-/4pns6BYi8MXdwnXM44yoGAcFYVHL/BYlB2q1HXZ6AzH++LaiEVWFpBWQ/glXhbMbv3E3o09igrHFbP/snhAvA== + +"@rollup/rollup-android-arm64@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.3.0.tgz#17b0f412034d14668c8acc8b7cbd8b1c76279599" + integrity sha512-nLO/JsL9idr416vzi3lHm3Xm+QZh4qHij8k3Er13kZr5YhL7/+kBAx84kDmPc7HMexLmwisjDCeDIKNFp8mDlQ== + +"@rollup/rollup-darwin-arm64@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.3.0.tgz#80c4a4dd7b120906d4e655808fb9005784a8bf35" + integrity sha512-dGhVBlllt4iHwTGy21IEoMOTN5wZoid19zEIxsdY29xcEiOEHqzDa7Sqrkh5OE7LKCowL61eFJXxYe/+pYa7ZQ== + +"@rollup/rollup-darwin-x64@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.3.0.tgz#52ad0db40d9b5ae047dfc08e54e4b3f42feaef82" + integrity sha512-h8wRfHeLEbU3NzaP1Oku7BYXCJQiTRr+8U0lklyOQXxXiEpHLL8tk1hFl+tezoRKLcPJD7joKaK74ASsqt3Ekg== + +"@rollup/rollup-linux-arm-gnueabihf@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.3.0.tgz#2ad3d190af01d7fc8704e8e782c4a24006a9f21a" + integrity sha512-wP4VgR/gfV18sylTuym3sxRTkAgUR2vh6YLeX/GEznk5jCYcYSlx585XlcUcl0c8UffIZlRJ09raWSX3JDb4GA== + +"@rollup/rollup-linux-arm64-gnu@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.3.0.tgz#4f7ba42f779f06e93876755b7393c61676e2958a" + integrity sha512-v/14JCYVkqRSJeQbxFx4oUkwVQQw6lFMN7bd4vuARBc3X2lmomkxBsc+BFiIDL/BK+CTx5AOh/k9XmqDnKWRVg== + +"@rollup/rollup-linux-arm64-musl@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.3.0.tgz#64795a09dac02b4d779819509a793b93ba7e4c0d" + integrity sha512-tNhfYqFH5OxtRzfkTOKdgFYlPSZnlDLNW4+leNEvQZhwTJxoTwsZAAhR97l3qVry/kkLyJPBK+Q8EAJLPinDIg== + +"@rollup/rollup-linux-x64-gnu@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.3.0.tgz#00c1ff131ba16881eb1a0ad46b0aa10dcacb010e" + integrity sha512-pw77m8QywdsoFdFOgmc8roF1inBI0rciqzO8ffRUgLoq7+ee9o5eFqtEcS6hHOOplgifAUUisP8cAnwl9nUYPw== + +"@rollup/rollup-linux-x64-musl@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.3.0.tgz#89479dce5e5bf6850fbca92fa7f1637ddd70c9ef" + integrity sha512-tJs7v2MnV2F8w6X1UpPHl/43OfxjUy9SuJ2ZPoxn79v9vYteChVYO/ueLHCpRMmyTUIVML3N9z4azl9ENH8Xxg== + +"@rollup/rollup-win32-arm64-msvc@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.3.0.tgz#1a36aba17c7efe6d61e98b8049e70b40e33b1f45" + integrity sha512-OKGxp6kATQdTyI2DF+e9s+hB3/QZB45b6e+dzcfW1SUqiF6CviWyevhmT4USsMEdP3mlpC9zxLz3Oh+WaTMOSw== + +"@rollup/rollup-win32-ia32-msvc@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.3.0.tgz#a0b1f79afde51e390a7725b7c15ab4e0df780aea" + integrity sha512-DDZ5AH68JJ2ClQFEA1aNnfA7Ybqyeh0644rGbrLOdNehTmzfICHiWSn0OprzYi9HAshTPQvlwrM+bi2kuaIOjQ== + +"@rollup/rollup-win32-x64-msvc@4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.3.0.tgz#0b9bcc159b93c911efb5a2c39ec5d70dd0a589dc" + integrity sha512-dMvGV8p92GQ8jhNlGIKpyhVZPzJlT258pPrM5q2F8lKcc9Iv9BbfdnhX1OfinYWnb9ms5zLw6MlaMnqLfUkKnQ== + +"@rollup/stream@3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@rollup/stream/-/stream-3.0.1.tgz#485452d6f1016ac1b0513060f90ff02aaca3e1c0" + integrity sha512-wdzoakLc9UiPOFa1k17ukfEtvQ0p7JuNFvOZT1DhO5Z5CrTf71An01U9+v+aebYcaLCwy3tLwpCSUF7K7xVN0A== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -6160,11 +6220,23 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" -rollup@3.29.4: - version "3.29.4" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-3.29.4.tgz#4d70c0f9834146df8705bfb69a9a19c9e1109981" - integrity sha512-oWzmBZwvYrU0iJHtDmhsm662rC15FRXmcjCk1xD771dFDx5jJ02ufAQQTn0etB2emNk4J9EZg/yWKpsn9BWGRw== +rollup@4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.3.0.tgz#198e6ae4355899db630d75bc0e17b53f5d0fc20e" + integrity sha512-scIi1NrKLDIYSPK66jjECtII7vIgdAMFmFo8h6qm++I6nN9qDSV35Ku6erzGVqYjx+lj+j5wkusRMr++8SyDZg== optionalDependencies: + "@rollup/rollup-android-arm-eabi" "4.3.0" + "@rollup/rollup-android-arm64" "4.3.0" + "@rollup/rollup-darwin-arm64" "4.3.0" + "@rollup/rollup-darwin-x64" "4.3.0" + "@rollup/rollup-linux-arm-gnueabihf" "4.3.0" + "@rollup/rollup-linux-arm64-gnu" "4.3.0" + "@rollup/rollup-linux-arm64-musl" "4.3.0" + "@rollup/rollup-linux-x64-gnu" "4.3.0" + "@rollup/rollup-linux-x64-musl" "4.3.0" + "@rollup/rollup-win32-arm64-msvc" "4.3.0" + "@rollup/rollup-win32-ia32-msvc" "4.3.0" + "@rollup/rollup-win32-x64-msvc" "4.3.0" fsevents "~2.3.2" run-parallel@^1.1.9: From e90c4cfdfceae662ba451c4f6ef7895a437a7aef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 07:11:18 +0900 Subject: [PATCH 07/57] MINOR: [C#] Bump Google.Protobuf from 3.24.4 to 3.25.0 in /csharp (#38603) Bumps [Google.Protobuf](https://github.com/protocolbuffers/protobuf) from 3.24.4 to 3.25.0.
Commits
  • 6b5d8db Updating version.json and repo version numbers to: 25.0
  • 59a66af Merge pull request #14583 from haberman/cherry-pick-python-memory-leak
  • 1711ebd Fixed Python memory leak in map lookup.
  • 8defef5 Include .inc files directly instead of through a filegroup (#14575)
  • 0a5d6d4 Check that ffi-compiler loads before using it to define tasks. (#14538) (#14560)
  • a4964a4 Merge pull request #14460 from protocolbuffers/25.x-202310181825
  • 08b6c40 Updating version.json to: 25.0-dev
  • ba1aab2 Updating version.json and repo version numbers to: 25.0-rc2
  • 3f05c09 update test runners
  • b4abcef Bump minimum PHP to 8.0
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Google.Protobuf&package-manager=nuget&previous-version=3.24.4&new-version=3.25.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj index d771aebd96796..975bf44db888f 100644 --- a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj +++ b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj @@ -5,7 +5,7 @@ - + From 5164acbf8c102bbfe2ec7011d8fe638b031aac7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 07:11:45 +0900 Subject: [PATCH 08/57] MINOR: [C#] Bump xunit from 2.5.3 to 2.6.1 in /csharp (#38604) Bumps [xunit](https://github.com/xunit/xunit) from 2.5.3 to 2.6.1.
Commits
  • 9f1684f v2.6.1
  • 5d9ef4b Remove unnecessary XUNIT_VALUETASK definition
  • 0266bac Latest assertions
  • fb3c3cf Remove all ValueTask support from assertions
  • 801790c Reconcile AcceptanceTestAssembly and friends with v3
  • f244cc8 More shared assertion tests that shouldn't return ValueTask
  • 262dd72 Shared assertion tests cannot return ValueTask
  • 36bbf77 Bump up to v2.6.1-pre
  • 6a07b95 v2.6.0
  • 5a7297e #2806: Performance improvements in Assert.Equal (v2)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=xunit&package-manager=nuget&previous-version=2.5.3&new-version=2.6.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- .../Apache.Arrow.Compression.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Sql.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj | 2 +- csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj index e63c27d022c24..835d95fc0c521 100644 --- a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj +++ b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj index ed67fafbf8c72..10582500d443c 100644 --- a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj index e4e3c053f943d..b371de9b1eb99 100644 --- a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj index 2835c60da414f..0df097156da34 100644 --- a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj +++ b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj @@ -15,7 +15,7 @@ - + all runtime; build; native; contentfiles; analyzers From 0bc75b7486e51d8939df966b29ee54ebc1083598 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 07:12:24 +0900 Subject: [PATCH 09/57] MINOR: [C#] Bump BenchmarkDotNet from 0.13.9 to 0.13.10 in /csharp (#38606) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet) from 0.13.9 to 0.13.10.
Release notes

Sourced from BenchmarkDotNet's releases.

0.13.10

Full changelog: https://benchmarkdotnet.org/changelog/v0.13.10.html

Highlights

Initial support of .NET 9 and minor bug fixes.

Details

In the v0.13.10 scope, 2 issues were resolved and 3 pull requests were merged. This release includes 10 commits by 4 contributors.

Resolved issues (2)

Merged pull requests (3)

Commits (10)

Contributors (4)

Thank you very much!

Commits
  • 40c4142 Prepare v0.13.10 changelog
  • ae49148 implement full .NET 9 support (#2456)
  • 0a734a9 feat: set RuntimeHostConfigurationOption on generated project (#2453)
  • e170684 Adjust 'Failed to set up high priority' message
  • 7b342f5 Add support for wasm/net9.0 (#2447)
  • e111368 [build] Bump Docfx.App: 2.71.0->2.71.1
  • af9c5c6 Exception handling in DotNetCliGenerator.GetRootDirectory, fix #2436
  • d17c6ad Support Windows 11 23H2 (10.0.22631) in OsBrandStringHelper
  • 2e96d29 Don't show AssemblyInformationalVersion metadata in BDN BrandVersion
  • c27152b Set next BenchmarkDotNet version: 0.13.10
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=BenchmarkDotNet&package-manager=nuget&previous-version=0.13.9&new-version=0.13.10)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- .../test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj b/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj index 5c2c1823fec99..4cbab77e1fe24 100644 --- a/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj +++ b/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj @@ -6,7 +6,7 @@ - + From c73cb13b9f89493f9fb91da854f95aa38bd86cc7 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 6 Nov 2023 23:20:05 +0100 Subject: [PATCH 10/57] GH-38607: [Python] Disable PyExtensionType autoload (#38608) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change PyExtensionType autoload is really a misfeature. It creates PyArrow-specific extension types, though using ExtensionType is almost the same complexity while allowing deserialization from non-PyArrow software. ### What changes are included in this PR? * Disable PyExtensionType autoloading and deprecate PyExtensionType instantiation. * Update the docs to emphasize ExtensionType. ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes. * Closes: #38607 Authored-by: Antoine Pitrou Signed-off-by: Raúl Cumplido --- docs/source/python/extending_types.rst | 149 ++++------ python/pyarrow/tests/test_cffi.py | 48 ++- python/pyarrow/tests/test_extension_type.py | 313 ++++++++++++++------ python/pyarrow/tests/test_pandas.py | 24 +- python/pyarrow/types.pxi | 88 +++--- 5 files changed, 377 insertions(+), 245 deletions(-) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index b9e875ceebc74..ee92cebcb549c 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -68,34 +68,43 @@ message). See the :ref:`format_metadata_extension_types` section of the metadata specification for more details. -Pyarrow allows you to define such extension types from Python. - -There are currently two ways: - -* Subclassing :class:`PyExtensionType`: the (de)serialization is based on pickle. - This is a good option for an extension type that is only used from Python. -* Subclassing :class:`ExtensionType`: this allows to give a custom - Python-independent name and serialized metadata, that can potentially be - recognized by other (non-Python) Arrow implementations such as PySpark. +Pyarrow allows you to define such extension types from Python by subclassing +:class:`ExtensionType` and giving the derived class its own extension name +and serialization mechanism. The extension name and serialized metadata +can potentially be recognized by other (non-Python) Arrow implementations +such as PySpark. For example, we could define a custom UUID type for 128-bit numbers which can -be represented as ``FixedSizeBinary`` type with 16 bytes. -Using the first approach, we create a ``UuidType`` subclass, and implement the -``__reduce__`` method to ensure the class can be properly pickled:: +be represented as ``FixedSizeBinary`` type with 16 bytes:: - class UuidType(pa.PyExtensionType): + class UuidType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.binary(16)) + super().__init__(pa.binary(16), "my_package.uuid") + + def __arrow_ext_serialize__(self): + # Since we don't have a parameterized type, we don't need extra + # metadata to be deserialized + return b'' - def __reduce__(self): - return UuidType, () + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + # Sanity checks, not required but illustrate the method signature. + assert storage_type == pa.binary(16) + assert serialized == b'' + # Return an instance of this subclass given the serialized + # metadata. + return UuidType() + +The special methods ``__arrow_ext_serialize__`` and ``__arrow_ext_deserialize__`` +define the serialization of an extension type instance. For non-parametric +types such as the above, the serialization payload can be left empty. This can now be used to create arrays and tables holding the extension type:: >>> uuid_type = UuidType() >>> uuid_type.extension_name - 'arrow.py_extension_type' + 'my_package.uuid' >>> uuid_type.storage_type FixedSizeBinaryType(fixed_size_binary[16]) @@ -112,8 +121,11 @@ This can now be used to create arrays and tables holding the extension type:: ] This array can be included in RecordBatches, sent over IPC and received in -another Python process. The custom UUID type will be preserved there, as long -as the definition of the class is available (the type can be unpickled). +another Python process. The receiving process must explicitly register the +extension type for deserialization, otherwise it will fall back to the +storage type:: + + >>> pa.register_extension_type(UuidType()) For example, creating a RecordBatch and writing it to a stream using the IPC protocol:: @@ -129,43 +141,12 @@ and then reading it back yields the proper type:: >>> with pa.ipc.open_stream(buf) as reader: ... result = reader.read_all() >>> result.column('ext').type - UuidType(extension) - -We can define the same type using the other option:: - - class UuidType(pa.ExtensionType): - - def __init__(self): - pa.ExtensionType.__init__(self, pa.binary(16), "my_package.uuid") - - def __arrow_ext_serialize__(self): - # since we don't have a parameterized type, we don't need extra - # metadata to be deserialized - return b'' - - @classmethod - def __arrow_ext_deserialize__(self, storage_type, serialized): - # return an instance of this subclass given the serialized - # metadata. - return UuidType() - -This is a slightly longer implementation (you need to implement the special -methods ``__arrow_ext_serialize__`` and ``__arrow_ext_deserialize__``), and the -extension type needs to be registered to be received through IPC (using -:func:`register_extension_type`), but it has -now a unique name:: - - >>> uuid_type = UuidType() - >>> uuid_type.extension_name - 'my_package.uuid' - - >>> pa.register_extension_type(uuid_type) + UuidType(FixedSizeBinaryType(fixed_size_binary[16])) The receiving application doesn't need to be Python but can still recognize -the extension type as a "uuid" type, if it has implemented its own extension -type to receive it. -If the type is not registered in the receiving application, it will fall back -to the storage type. +the extension type as a "my_package.uuid" type, if it has implemented its own +extension type to receive it. If the type is not registered in the receiving +application, it will fall back to the storage type. Parameterized extension type ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -187,7 +168,7 @@ of the given frequency since 1970. # attributes need to be set first before calling # super init (as that calls serialize) self._freq = freq - pa.ExtensionType.__init__(self, pa.int64(), 'my_package.period') + super().__init__(pa.int64(), 'my_package.period') @property def freq(self): @@ -198,7 +179,7 @@ of the given frequency since 1970. @classmethod def __arrow_ext_deserialize__(cls, storage_type, serialized): - # return an instance of this subclass given the serialized + # Return an instance of this subclass given the serialized # metadata. serialized = serialized.decode() assert serialized.startswith("freq=") @@ -209,31 +190,10 @@ Here, we ensure to store all information in the serialized metadata that is needed to reconstruct the instance (in the ``__arrow_ext_deserialize__`` class method), in this case the frequency string. -Note that, once created, the data type instance is considered immutable. If, -in the example above, the ``freq`` parameter would change after instantiation, -the reconstruction of the type instance after IPC will be incorrect. +Note that, once created, the data type instance is considered immutable. In the example above, the ``freq`` parameter is therefore stored in a private attribute with a public read-only property to access it. -Parameterized extension types are also possible using the pickle-based type -subclassing :class:`PyExtensionType`. The equivalent example for the period -data type from above would look like:: - - class PeriodType(pa.PyExtensionType): - - def __init__(self, freq): - self._freq = freq - pa.PyExtensionType.__init__(self, pa.int64()) - - @property - def freq(self): - return self._freq - - def __reduce__(self): - return PeriodType, (self.freq,) - -Also the storage type does not need to be fixed but can be parameterized. - Custom extension array class ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -252,12 +212,16 @@ the data as a 2-D Numpy array ``(N, 3)`` without any copy:: return self.storage.flatten().to_numpy().reshape((-1, 3)) - class Point3DType(pa.PyExtensionType): + class Point3DType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.list_(pa.float32(), 3)) + super().__init__(pa.list_(pa.float32(), 3), "my_package.Point3DType") - def __reduce__(self): - return Point3DType, () + def __arrow_ext_serialize__(self): + return b'' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + return Point3DType() def __arrow_ext_class__(self): return Point3DArray @@ -289,11 +253,8 @@ The additional methods in the extension class are then available to the user:: This array can be sent over IPC, received in another Python process, and the custom -extension array class will be preserved (as long as the definitions of the classes above -are available). - -The same ``__arrow_ext_class__`` specialization can be used with custom types defined -by subclassing :class:`ExtensionType`. +extension array class will be preserved (as long as the receiving process registers +the extension type using :func:`register_extension_type` before reading the IPC data). Custom scalar conversion ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -304,18 +265,24 @@ If you want scalars of your custom extension type to convert to a custom type wh For example, if we wanted the above example 3D point type to return a custom 3D point class instead of a list, we would implement:: + from collections import namedtuple + Point3D = namedtuple("Point3D", ["x", "y", "z"]) class Point3DScalar(pa.ExtensionScalar): def as_py(self) -> Point3D: return Point3D(*self.value.as_py()) - class Point3DType(pa.PyExtensionType): + class Point3DType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.list_(pa.float32(), 3)) + super().__init__(pa.list_(pa.float32(), 3), "my_package.Point3DType") - def __reduce__(self): - return Point3DType, () + def __arrow_ext_serialize__(self): + return b'' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + return Point3DType() def __arrow_ext_scalar_class__(self): return Point3DScalar diff --git a/python/pyarrow/tests/test_cffi.py b/python/pyarrow/tests/test_cffi.py index 55bab4359bffc..a9c17cc100cb4 100644 --- a/python/pyarrow/tests/test_cffi.py +++ b/python/pyarrow/tests/test_cffi.py @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. +import contextlib import ctypes import gc @@ -51,18 +52,33 @@ def PyCapsule_IsValid(capsule, name): return ctypes.pythonapi.PyCapsule_IsValid(ctypes.py_object(capsule), name) == 1 -class ParamExtType(pa.PyExtensionType): +@contextlib.contextmanager +def registered_extension_type(ext_type): + pa.register_extension_type(ext_type) + try: + yield + finally: + pa.unregister_extension_type(ext_type.extension_name) + + +class ParamExtType(pa.ExtensionType): def __init__(self, width): self._width = width - pa.PyExtensionType.__init__(self, pa.binary(width)) + super().__init__(pa.binary(width), + "pyarrow.tests.test_cffi.ParamExtType") @property def width(self): return self._width - def __reduce__(self): - return ParamExtType, (self.width,) + def __arrow_ext_serialize__(self): + return str(self.width).encode() + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + width = int(serialized.decode()) + return cls(width) def make_schema(): @@ -75,6 +91,12 @@ def make_extension_schema(): metadata={b'key1': b'value1'}) +def make_extension_storage_schema(): + # Should be kept in sync with make_extension_schema + return pa.schema([('ext', ParamExtType(3).storage_type)], + metadata={b'key1': b'value1'}) + + def make_batch(): return pa.record_batch([[[1], [2, 42]]], make_schema()) @@ -204,7 +226,10 @@ def test_export_import_array(): pa.Array._import_from_c(ptr_array, ptr_schema) -def check_export_import_schema(schema_factory): +def check_export_import_schema(schema_factory, expected_schema_factory=None): + if expected_schema_factory is None: + expected_schema_factory = schema_factory + c_schema = ffi.new("struct ArrowSchema*") ptr_schema = int(ffi.cast("uintptr_t", c_schema)) @@ -215,7 +240,7 @@ def check_export_import_schema(schema_factory): assert pa.total_allocated_bytes() > old_allocated # Delete and recreate C++ object from exported pointer schema_new = pa.Schema._import_from_c(ptr_schema) - assert schema_new == schema_factory() + assert schema_new == expected_schema_factory() assert pa.total_allocated_bytes() == old_allocated del schema_new assert pa.total_allocated_bytes() == old_allocated @@ -240,7 +265,13 @@ def test_export_import_schema(): @needs_cffi def test_export_import_schema_with_extension(): - check_export_import_schema(make_extension_schema) + # Extension type is unregistered => the storage type is imported + check_export_import_schema(make_extension_schema, + make_extension_storage_schema) + + # Extension type is registered => the extension type is imported + with registered_extension_type(ParamExtType(1)): + check_export_import_schema(make_extension_schema) @needs_cffi @@ -319,7 +350,8 @@ def test_export_import_batch(): @needs_cffi def test_export_import_batch_with_extension(): - check_export_import_batch(make_extension_batch) + with registered_extension_type(ParamExtType(1)): + check_export_import_batch(make_extension_batch) def _export_import_batch_reader(ptr_stream, reader_factory): diff --git a/python/pyarrow/tests/test_extension_type.py b/python/pyarrow/tests/test_extension_type.py index ce575d984e41c..a88e20eefe098 100644 --- a/python/pyarrow/tests/test_extension_type.py +++ b/python/pyarrow/tests/test_extension_type.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import contextlib import os import shutil import subprocess @@ -29,31 +30,69 @@ import pytest -class TinyIntType(pa.PyExtensionType): +@contextlib.contextmanager +def registered_extension_type(ext_type): + pa.register_extension_type(ext_type) + try: + yield + finally: + pa.unregister_extension_type(ext_type.extension_name) + + +@contextlib.contextmanager +def enabled_auto_load(): + pa.PyExtensionType.set_auto_load(True) + try: + yield + finally: + pa.PyExtensionType.set_auto_load(False) + + +class TinyIntType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.int8()) + super().__init__(pa.int8(), 'pyarrow.tests.TinyIntType') - def __reduce__(self): - return TinyIntType, () + def __arrow_ext_serialize__(self): + return b'' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + assert serialized == b'' + assert storage_type == pa.int8() + return cls() -class IntegerType(pa.PyExtensionType): +class IntegerType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.int64()) + super().__init__(pa.int64(), 'pyarrow.tests.IntegerType') - def __reduce__(self): - return IntegerType, () + def __arrow_ext_serialize__(self): + return b'' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + assert serialized == b'' + assert storage_type == pa.int64() + return cls() -class IntegerEmbeddedType(pa.PyExtensionType): +class IntegerEmbeddedType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, IntegerType()) + super().__init__(IntegerType(), 'pyarrow.tests.IntegerType') - def __reduce__(self): - return IntegerEmbeddedType, () + def __arrow_ext_serialize__(self): + # XXX pa.BaseExtensionType should expose C++ serialization method + return self.storage_type.__arrow_ext_serialize__() + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + deserialized_storage_type = storage_type.__arrow_ext_deserialize__( + serialized) + assert deserialized_storage_type == storage_type + return cls() class UuidScalarType(pa.ExtensionScalar): @@ -61,81 +100,125 @@ def as_py(self): return None if self.value is None else UUID(bytes=self.value.as_py()) -class UuidType(pa.PyExtensionType): +class UuidType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.binary(16)) - - def __reduce__(self): - return UuidType, () + super().__init__(pa.binary(16), 'pyarrow.tests.UuidType') def __arrow_ext_scalar_class__(self): return UuidScalarType + def __arrow_ext_serialize__(self): + return b'' -class UuidType2(pa.PyExtensionType): + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + return cls() + + +class UuidType2(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.binary(16)) + super().__init__(pa.binary(16), 'pyarrow.tests.UuidType2') - def __reduce__(self): - return UuidType2, () + def __arrow_ext_serialize__(self): + return b'' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + return cls() -class LabelType(pa.PyExtensionType): +class LabelType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.string()) + super().__init__(pa.string(), 'pyarrow.tests.LabelType') - def __reduce__(self): - return LabelType, () + def __arrow_ext_serialize__(self): + return b'' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + return cls() -class ParamExtType(pa.PyExtensionType): +class ParamExtType(pa.ExtensionType): def __init__(self, width): self._width = width - pa.PyExtensionType.__init__(self, pa.binary(width)) + super().__init__(pa.binary(width), 'pyarrow.tests.ParamExtType') @property def width(self): return self._width - def __reduce__(self): - return ParamExtType, (self.width,) + def __arrow_ext_serialize__(self): + return str(self._width).encode() + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + width = int(serialized.decode()) + assert storage_type == pa.binary(width) + return cls(width) -class MyStructType(pa.PyExtensionType): +class MyStructType(pa.ExtensionType): storage_type = pa.struct([('left', pa.int64()), ('right', pa.int64())]) def __init__(self): - pa.PyExtensionType.__init__(self, self.storage_type) + super().__init__(self.storage_type, 'pyarrow.tests.MyStructType') - def __reduce__(self): - return MyStructType, () + def __arrow_ext_serialize__(self): + return b'' + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + assert serialized == b'' + assert storage_type == cls.storage_type + return cls() -class MyListType(pa.PyExtensionType): + +class MyListType(pa.ExtensionType): def __init__(self, storage_type): - pa.PyExtensionType.__init__(self, storage_type) + assert isinstance(storage_type, pa.ListType) + super().__init__(storage_type, 'pyarrow.tests.MyListType') - def __reduce__(self): - return MyListType, (self.storage_type,) + def __arrow_ext_serialize__(self): + return b'' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + assert serialized == b'' + return cls(storage_type) -class AnnotatedType(pa.PyExtensionType): +class AnnotatedType(pa.ExtensionType): """ Generic extension type that can store any storage type. """ def __init__(self, storage_type, annotation): self.annotation = annotation - super().__init__(storage_type) + super().__init__(storage_type, 'pyarrow.tests.AnnotatedType') + + def __arrow_ext_serialize__(self): + return b'' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + assert serialized == b'' + return cls(storage_type) + + +class LegacyIntType(pa.PyExtensionType): + + def __init__(self): + pa.PyExtensionType.__init__(self, pa.int8()) def __reduce__(self): - return AnnotatedType, (self.storage_type, self.annotation) + return LegacyIntType, () def ipc_write_batch(batch): @@ -153,12 +236,12 @@ def ipc_read_batch(buf): def test_ext_type_basics(): ty = UuidType() - assert ty.extension_name == "arrow.py_extension_type" + assert ty.extension_name == "pyarrow.tests.UuidType" def test_ext_type_str(): ty = IntegerType() - expected = "extension>" + expected = "extension>" assert str(ty) == expected assert pa.DataType.__str__(ty) == expected @@ -223,7 +306,7 @@ def test_uuid_type_pickle(pickle_module): del ty ty = pickle_module.loads(ser) wr = weakref.ref(ty) - assert ty.extension_name == "arrow.py_extension_type" + assert ty.extension_name == "pyarrow.tests.UuidType" del ty assert wr() is None @@ -571,9 +654,9 @@ def test_cast_between_extension_types(): assert tiny_int_arr.type == TinyIntType() # Casting between extension types w/ different storage types not okay. - msg = ("Casting from 'extension>' " + msg = ("Casting from 'extension<.*?>' " "to different extension type " - "'extension>' not permitted. " + "'extension<.*?>' not permitted. " "One can first cast to the storage type, " "then to the extension type." ) @@ -660,53 +743,38 @@ def example_batch(): return pa.RecordBatch.from_arrays([arr], ["exts"]) -def check_example_batch(batch): +def check_example_batch(batch, *, expect_extension): arr = batch.column(0) - assert isinstance(arr, pa.ExtensionArray) - assert arr.type.storage_type == pa.binary(3) - assert arr.storage.to_pylist() == [b"foo", b"bar"] + if expect_extension: + assert isinstance(arr, pa.ExtensionArray) + assert arr.type.storage_type == pa.binary(3) + assert arr.storage.to_pylist() == [b"foo", b"bar"] + else: + assert arr.type == pa.binary(3) + assert arr.to_pylist() == [b"foo", b"bar"] return arr -def test_ipc(): +def test_ipc_unregistered(): batch = example_batch() buf = ipc_write_batch(batch) del batch batch = ipc_read_batch(buf) - arr = check_example_batch(batch) - assert arr.type == ParamExtType(3) + batch.validate(full=True) + check_example_batch(batch, expect_extension=False) -def test_ipc_unknown_type(): - batch = example_batch() - buf = ipc_write_batch(batch) - del batch - - orig_type = ParamExtType - try: - # Simulate the original Python type being unavailable. - # Deserialization should not fail but return a placeholder type. - del globals()['ParamExtType'] +def test_ipc_registered(): + with registered_extension_type(ParamExtType(1)): + batch = example_batch() + buf = ipc_write_batch(batch) + del batch batch = ipc_read_batch(buf) - arr = check_example_batch(batch) - assert isinstance(arr.type, pa.UnknownExtensionType) - - # Can be serialized again - buf2 = ipc_write_batch(batch) - del batch, arr - - batch = ipc_read_batch(buf2) - arr = check_example_batch(batch) - assert isinstance(arr.type, pa.UnknownExtensionType) - finally: - globals()['ParamExtType'] = orig_type - - # Deserialize again with the type restored - batch = ipc_read_batch(buf2) - arr = check_example_batch(batch) - assert arr.type == ParamExtType(3) + batch.validate(full=True) + arr = check_example_batch(batch, expect_extension=True) + assert arr.type == ParamExtType(3) class PeriodArray(pa.ExtensionArray): @@ -930,6 +998,7 @@ def test_parquet_period(tmpdir, registered_period_type): # When reading in, properly create extension type if it is registered result = pq.read_table(filename) + result.validate(full=True) assert result.schema.field("ext").type == period_type assert result.schema.field("ext").metadata == {} # Get the exact array class defined by the registered type. @@ -939,6 +1008,7 @@ def test_parquet_period(tmpdir, registered_period_type): # When the type is not registered, read in as storage type pa.unregister_extension_type(period_type.extension_name) result = pq.read_table(filename) + result.validate(full=True) assert result.schema.field("ext").type == pa.int64() # The extension metadata is present for roundtripping. assert result.schema.field("ext").metadata == { @@ -967,13 +1037,28 @@ def test_parquet_extension_with_nested_storage(tmpdir): filename = tmpdir / 'nested_extension_storage.parquet' pq.write_table(orig_table, filename) + # Unregistered table = pq.read_table(filename) - assert table.column('structs').type == mystruct_array.type - assert table.column('lists').type == mylist_array.type - assert table == orig_table - - with pytest.raises(pa.ArrowInvalid, match='without all of its fields'): - pq.ParquetFile(filename).read(columns=['structs.left']) + table.validate(full=True) + assert table.column('structs').type == struct_array.type + assert table.column('structs').combine_chunks() == struct_array + assert table.column('lists').type == list_array.type + assert table.column('lists').combine_chunks() == list_array + + # Registered + with registered_extension_type(mystruct_array.type): + with registered_extension_type(mylist_array.type): + table = pq.read_table(filename) + table.validate(full=True) + assert table.column('structs').type == mystruct_array.type + assert table.column('lists').type == mylist_array.type + assert table == orig_table + + # Cannot select a subfield of an extension type with + # a struct storage type. + with pytest.raises(pa.ArrowInvalid, + match='without all of its fields'): + pq.ParquetFile(filename).read(columns=['structs.left']) @pytest.mark.parquet @@ -995,8 +1080,14 @@ def test_parquet_nested_extension(tmpdir): pq.write_table(orig_table, filename) table = pq.read_table(filename) - assert table.column(0).type == struct_array.type - assert table == orig_table + table.validate(full=True) + assert table.column(0).type == pa.struct({'ints': pa.int64(), + 'exts': pa.int64()}) + with registered_extension_type(ext_type): + table = pq.read_table(filename) + table.validate(full=True) + assert table.column(0).type == struct_array.type + assert table == orig_table # List of extensions list_array = pa.ListArray.from_arrays([0, 1, None, 3], ext_array) @@ -1006,8 +1097,13 @@ def test_parquet_nested_extension(tmpdir): pq.write_table(orig_table, filename) table = pq.read_table(filename) - assert table.column(0).type == list_array.type - assert table == orig_table + table.validate(full=True) + assert table.column(0).type == pa.list_(pa.int64()) + with registered_extension_type(ext_type): + table = pq.read_table(filename) + table.validate(full=True) + assert table.column(0).type == list_array.type + assert table == orig_table # Large list of extensions list_array = pa.LargeListArray.from_arrays([0, 1, None, 3], ext_array) @@ -1017,8 +1113,13 @@ def test_parquet_nested_extension(tmpdir): pq.write_table(orig_table, filename) table = pq.read_table(filename) - assert table.column(0).type == list_array.type - assert table == orig_table + table.validate(full=True) + assert table.column(0).type == pa.large_list(pa.int64()) + with registered_extension_type(ext_type): + table = pq.read_table(filename) + table.validate(full=True) + assert table.column(0).type == list_array.type + assert table == orig_table @pytest.mark.parquet @@ -1040,8 +1141,12 @@ def test_parquet_extension_nested_in_extension(tmpdir): pq.write_table(orig_table, filename) table = pq.read_table(filename) - assert table.column(0).type == mylist_array.type - assert table == orig_table + assert table.column(0).type == pa.list_(pa.int64()) + with registered_extension_type(mylist_array.type): + with registered_extension_type(inner_ext_array.type): + table = pq.read_table(filename) + assert table.column(0).type == mylist_array.type + assert table == orig_table def test_to_numpy(): @@ -1370,3 +1475,25 @@ def test_tensor_type_is_picklable(pickle_module): def test_tensor_type_str(tensor_type, text): tensor_type_str = tensor_type.__str__() assert text in tensor_type_str + + +def test_legacy_int_type(): + with pytest.warns(FutureWarning, match="PyExtensionType is deprecated"): + ext_ty = LegacyIntType() + arr = pa.array([1, 2, 3], type=ext_ty.storage_type) + ext_arr = pa.ExtensionArray.from_storage(ext_ty, arr) + batch = pa.RecordBatch.from_arrays([ext_arr], names=['ext']) + buf = ipc_write_batch(batch) + + with pytest.warns( + RuntimeWarning, + match="pickle-based deserialization of pyarrow.PyExtensionType " + "subclasses is disabled by default"): + batch = ipc_read_batch(buf) + assert isinstance(batch.column(0).type, pa.UnknownExtensionType) + + with enabled_auto_load(): + with pytest.warns(FutureWarning, match="PyExtensionType is deprecated"): + batch = ipc_read_batch(buf) + assert isinstance(batch.column(0).type, LegacyIntType) + assert batch.column(0) == ext_arr diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 62a9443953a3d..10eb931592093 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -4096,13 +4096,20 @@ def test_array_protocol(): assert result.equals(expected2) -class DummyExtensionType(pa.PyExtensionType): +class DummyExtensionType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.int64()) + super().__init__(pa.int64(), + 'pyarrow.tests.test_pandas.DummyExtensionType') - def __reduce__(self): - return DummyExtensionType, () + def __arrow_ext_serialize__(self): + return b'' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + assert serialized == b'' + assert storage_type == pa.int64() + return cls() def PandasArray__arrow_array__(self, type=None): @@ -4198,13 +4205,14 @@ def test_convert_to_extension_array(monkeypatch): assert not isinstance(_get_mgr(result).blocks[0], _int.ExtensionBlock) -class MyCustomIntegerType(pa.PyExtensionType): +class MyCustomIntegerType(pa.ExtensionType): def __init__(self): - pa.PyExtensionType.__init__(self, pa.int64()) + super().__init__(pa.int64(), + 'pyarrow.tests.test_pandas.MyCustomIntegerType') - def __reduce__(self): - return MyCustomIntegerType, () + def __arrow_ext_serialize__(self): + return b'' def to_pandas_dtype(self): return pd.Int64Dtype() diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index d394b803e7fc2..a0ddf09d69423 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -1437,7 +1437,10 @@ cdef class ExtensionType(BaseExtensionType): Parameters ---------- storage_type : DataType + The underlying storage type for the extension type. extension_name : str + A unique name distinguishing this extension type. The name will be + used when deserializing IPC data. Examples -------- @@ -1671,60 +1674,22 @@ cdef class FixedShapeTensorType(BaseExtensionType): self.dim_names, self.permutation) +_py_extension_type_auto_load = False + + cdef class PyExtensionType(ExtensionType): """ Concrete base class for Python-defined extension types based on pickle for (de)serialization. + .. warning:: + This class is deprecated and its deserialization is disabled by default. + :class:`ExtensionType` is recommended instead. + Parameters ---------- storage_type : DataType The storage type for which the extension is built. - - Examples - -------- - Define a UuidType extension type subclassing PyExtensionType: - - >>> import pyarrow as pa - >>> class UuidType(pa.PyExtensionType): - ... def __init__(self): - ... pa.PyExtensionType.__init__(self, pa.binary(16)) - ... def __reduce__(self): - ... return UuidType, () - ... - - Create an instance of UuidType extension type: - - >>> uuid_type = UuidType() # doctest: +SKIP - >>> uuid_type # doctest: +SKIP - UuidType(FixedSizeBinaryType(fixed_size_binary[16])) - - Inspect the extension type: - - >>> uuid_type.extension_name # doctest: +SKIP - 'arrow.py_extension_type' - >>> uuid_type.storage_type # doctest: +SKIP - FixedSizeBinaryType(fixed_size_binary[16]) - - Wrap an array as an extension array: - - >>> import uuid - >>> storage_array = pa.array([uuid.uuid4().bytes for _ in range(4)], - ... pa.binary(16)) # doctest: +SKIP - >>> uuid_type.wrap_array(storage_array) # doctest: +SKIP - - [ - ... - ] - - Or do the same with creating an ExtensionArray: - - >>> pa.ExtensionArray.from_storage(uuid_type, - ... storage_array) # doctest: +SKIP - - [ - ... - ] """ def __cinit__(self): @@ -1733,6 +1698,12 @@ cdef class PyExtensionType(ExtensionType): "PyExtensionType") def __init__(self, DataType storage_type): + warnings.warn( + "pyarrow.PyExtensionType is deprecated " + "and will refuse deserialization by default. " + "Instead, please derive from pyarrow.ExtensionType and implement " + "your own serialization mechanism.", + FutureWarning) ExtensionType.__init__(self, storage_type, "arrow.py_extension_type") def __reduce__(self): @@ -1744,6 +1715,17 @@ cdef class PyExtensionType(ExtensionType): @classmethod def __arrow_ext_deserialize__(cls, storage_type, serialized): + if not _py_extension_type_auto_load: + warnings.warn( + "pickle-based deserialization of pyarrow.PyExtensionType subclasses " + "is disabled by default; if you only ingest " + "trusted data files, you may re-enable this using " + "`pyarrow.PyExtensionType.set_auto_load(True)`.\n" + "In the future, Python-defined extension subclasses should " + "derive from pyarrow.ExtensionType (not pyarrow.PyExtensionType) " + "and implement their own serialization mechanism.\n", + RuntimeWarning) + return UnknownExtensionType(storage_type, serialized) try: ty = pickle.loads(serialized) except Exception: @@ -1759,6 +1741,22 @@ cdef class PyExtensionType(ExtensionType): .format(ty.storage_type, storage_type)) return ty + # XXX Cython marks extension types as immutable, so cannot expose this + # as a writable class attribute. + @classmethod + def set_auto_load(cls, value): + """ + Enable or disable auto-loading of serialized PyExtensionType instances. + + Parameters + ---------- + value : bool + Whether to enable auto-loading. + """ + global _py_extension_type_auto_load + assert isinstance(value, bool) + _py_extension_type_auto_load = value + cdef class UnknownExtensionType(PyExtensionType): """ From 6dcba930bed468268288dc6f66c1580cbf5f80e0 Mon Sep 17 00:00:00 2001 From: Maximilian Muecke Date: Tue, 7 Nov 2023 00:47:50 +0100 Subject: [PATCH 11/57] GH-38602: [R] Add missing `prod` for summarize (#38601) ### Rationale for this change `prod` is currently missing for use in summarize. ### What changes are included in this PR? Added `prod` for summarize aggregation. ### Are these changes tested? Yes, included the same tests used for the other aggregation functions for summarize. ### Are there any user-facing changes? Yes, added `prod` function. * Closes: #38602 Authored-by: Maximilian Muecke Signed-off-by: Dewey Dunnington --- r/R/dplyr-funcs-doc.R | 1 + r/R/dplyr-summarize.R | 7 +++++++ r/man/acero.Rd | 1 + r/src/compute.cpp | 3 ++- r/tests/testthat/test-dplyr-summarize.R | 23 +++++++++++++++++++++++ 5 files changed, 34 insertions(+), 1 deletion(-) diff --git a/r/R/dplyr-funcs-doc.R b/r/R/dplyr-funcs-doc.R index 5099e903da670..492729df8c12a 100644 --- a/r/R/dplyr-funcs-doc.R +++ b/r/R/dplyr-funcs-doc.R @@ -167,6 +167,7 @@ #' * [`paste0()`][base::paste0()]: the `collapse` argument is not yet supported #' * [`pmax()`][base::pmax()] #' * [`pmin()`][base::pmin()] +#' * [`prod()`][base::prod()] #' * [`round()`][base::round()] #' * [`sign()`][base::sign()] #' * [`sin()`][base::sin()] diff --git a/r/R/dplyr-summarize.R b/r/R/dplyr-summarize.R index 1e9d42969d080..e2c70f9e89e42 100644 --- a/r/R/dplyr-summarize.R +++ b/r/R/dplyr-summarize.R @@ -42,6 +42,13 @@ register_bindings_aggregate <- function() { options = list(skip_nulls = na.rm, min_count = 0L) ) }) + register_binding_agg("base::prod", function(..., na.rm = FALSE) { + list( + fun = "product", + data = ensure_one_arg(list2(...), "prod"), + options = list(skip_nulls = na.rm, min_count = 0L) + ) + }) register_binding_agg("base::any", function(..., na.rm = FALSE) { list( fun = "any", diff --git a/r/man/acero.Rd b/r/man/acero.Rd index c9fb4d37a1d95..12afdc23138ac 100644 --- a/r/man/acero.Rd +++ b/r/man/acero.Rd @@ -156,6 +156,7 @@ Consider using the lubridate specialised parsing functions \code{ymd()}, \code{y \item \code{\link[base:paste]{paste0()}}: the \code{collapse} argument is not yet supported \item \code{\link[base:Extremes]{pmax()}} \item \code{\link[base:Extremes]{pmin()}} +\item \code{\link[base:prod]{prod()}} \item \code{\link[base:Round]{round()}} \item \code{\link[base:sign]{sign()}} \item \code{\link[base:Trig]{sin()}} diff --git a/r/src/compute.cpp b/r/src/compute.cpp index 578ce74d05d12..87d1326ed3419 100644 --- a/r/src/compute.cpp +++ b/r/src/compute.cpp @@ -176,7 +176,8 @@ std::shared_ptr make_compute_options( func_name == "hash_approximate_median" || func_name == "mean" || func_name == "hash_mean" || func_name == "min_max" || func_name == "hash_min_max" || func_name == "min" || func_name == "hash_min" || func_name == "max" || - func_name == "hash_max" || func_name == "sum" || func_name == "hash_sum") { + func_name == "hash_max" || func_name == "sum" || func_name == "hash_sum" || + func_name == "product" || func_name == "hash_product") { using Options = arrow::compute::ScalarAggregateOptions; auto out = std::make_shared(Options::Defaults()); if (!Rf_isNull(options["min_count"])) { diff --git a/r/tests/testthat/test-dplyr-summarize.R b/r/tests/testthat/test-dplyr-summarize.R index e2fb9841e72ad..29993711921e2 100644 --- a/r/tests/testthat/test-dplyr-summarize.R +++ b/r/tests/testthat/test-dplyr-summarize.R @@ -91,6 +91,27 @@ test_that("Group by sum on dataset", { ) }) +test_that("Group by prod on dataset", { + compare_dplyr_binding( + .input %>% + group_by(some_grouping) %>% + summarize(prod = prod(int, na.rm = TRUE)) %>% + collect(), + tbl + ) + + compare_dplyr_binding( + .input %>% + group_by(some_grouping) %>% + summarize( + prod = prod(int, na.rm = FALSE), + prod2 = base::prod(int, na.rm = TRUE) + ) %>% + collect(), + tbl + ) +}) + test_that("Group by mean on dataset", { compare_dplyr_binding( .input %>% @@ -319,6 +340,7 @@ test_that("Functions that take ... but we only accept a single arg", { # the agg_funcs directly expect_error(call_binding_agg("n_distinct"), "n_distinct() with 0 arguments", fixed = TRUE) expect_error(call_binding_agg("sum"), "sum() with 0 arguments", fixed = TRUE) + expect_error(call_binding_agg("prod"), "prod() with 0 arguments", fixed = TRUE) expect_error(call_binding_agg("any"), "any() with 0 arguments", fixed = TRUE) expect_error(call_binding_agg("all"), "all() with 0 arguments", fixed = TRUE) expect_error(call_binding_agg("min"), "min() with 0 arguments", fixed = TRUE) @@ -642,6 +664,7 @@ test_that("summarise() with !!sym()", { group_by(false) %>% summarise( sum = sum(!!sym(test_dbl_col)), + prod = prod(!!sym(test_dbl_col)), any = any(!!sym(test_lgl_col)), all = all(!!sym(test_lgl_col)), mean = mean(!!sym(test_dbl_col)), From 2e01f0ce4d05726b3d6203e216211201e4c50d00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 09:48:01 +0900 Subject: [PATCH 12/57] MINOR: [JS] Bump @types/command-line-args from 5.2.0 to 5.2.2 in /js (#38546) Bumps [@ types/command-line-args](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/command-line-args) from 5.2.0 to 5.2.2.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=@ types/command-line-args&package-manager=npm_and_yarn&previous-version=5.2.0&new-version=5.2.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/js/yarn.lock b/js/yarn.lock index b2ed3ed20600b..26a459a73a206 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1255,9 +1255,9 @@ integrity sha512-rVCCileCU5NhP9Ix1e03sIn4gd0mpjh7VNULVQAxzF+9vddk6A5QAHzp2h5kXH8pkv1Ow45fUf3QP3wOEiISvA== "@types/command-line-args@^5.2.1": - version "5.2.1" - resolved "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.1.tgz#233bd1ba687e84ecbec0388e09f9ec9ebf63c55b" - integrity sha512-U2OcmS2tj36Yceu+mRuPyUV0ILfau/h5onStcSCzqTENsq0sBiAp2TmaXu1k8fY4McLcPKSYl9FRzn2hx5bI+w== + version "5.2.2" + resolved "https://registry.yarnpkg.com/@types/command-line-args/-/command-line-args-5.2.2.tgz#ca93ae7045305a2b7c60936bfcb1ebc897b42270" + integrity sha512-9aZ7KzLDOBYyqH5J2bvB9edvsMXusX+H/aS8idAJOpWNmscZG5RqO1CVJPFa4Q0/1xKgvxcweXunFVx2l/dYFA== "@types/command-line-usage@^5.0.2": version "5.0.2" From 7622ded456497911785f997a9e0240033e57042d Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 6 Nov 2023 19:08:13 -0600 Subject: [PATCH 13/57] GH-38570: [R] Ensure that test-nix-libs is warning free (#38571) ### Rationale for this change Although we stop / fail the CI job for warnings elsewhere, we don't seem to be doing that for our buildsystem tests. Let's change that so warnings + other output doesn't creep in. - [x] Prevent `*** Failed to find latest nightly for 8.0.0.9000` from showing up when the file is sourced under test - [x] fix `Warning ('test-nixlibs.R:140:3'): select_binary() with test program 'x' is NULL so the result will be NULL` - [x] update version strings to be characters to avoid r-devel warnings ### What changes are included in this PR? Added `stop_on_warning` to the test call, and fixes to make that pass ### Are these changes tested? Yes, they are all part of the test suite ### Are there any user-facing changes? No. * Closes: #38570 Authored-by: Jonathan Keane Signed-off-by: Jacob Wujciak-Jens --- ci/scripts/r_test.sh | 2 +- r/tools/check-versions.R | 4 ++-- r/tools/nixlibs.R | 14 +++++++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index e0c2ce9efedd8..22ec551edb9fa 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -27,7 +27,7 @@ pushd ${source_dir} printenv # Run the nixlibs.R test suite, which is not included in the installed package -${R_BIN} -e 'setwd("tools"); testthat::test_dir(".")' +${R_BIN} -e 'setwd("tools"); testthat::test_dir(".", stop_on_warning = TRUE)' # Before release, we always copy the relevant parts of the cpp source into the # package. In some CI checks, we will use this version of the source: diff --git a/r/tools/check-versions.R b/r/tools/check-versions.R index 45fbd0b61a40a..3d8cbf02a14c9 100644 --- a/r/tools/check-versions.R +++ b/r/tools/check-versions.R @@ -23,8 +23,8 @@ test_mode <- exists("TESTING") check_versions <- function(r_version, cpp_version) { r_parsed <- package_version(r_version) r_dev_version <- r_parsed[1, 4] - r_is_dev <- !is.na(r_dev_version) && r_dev_version > 100 - r_is_patch <- !is.na(r_dev_version) && r_dev_version <= 100 + r_is_dev <- !is.na(r_dev_version) && r_dev_version > "100" + r_is_patch <- !is.na(r_dev_version) && r_dev_version <= "100" cpp_is_dev <- grepl("SNAPSHOT$", cpp_version) cpp_parsed <- package_version(sub("-SNAPSHOT$", "", cpp_version)) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 96f2b858cf180..63c185ce54563 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -198,7 +198,11 @@ select_binary <- function(os = tolower(Sys.info()[["sysname"]]), errs <- compile_test_program(test_program) openssl_version <- determine_binary_from_stderr(errs) arch <- ifelse(identical(os, "darwin"), paste0("-", arch, "-"), "-") - ifelse(is.null(openssl_version), NULL, paste0(os, arch, openssl_version)) + if (is.null(openssl_version)) { + NULL + } else { + paste0(os, arch, openssl_version) + } }, error = function(e) { lg("Unable to find libcurl and openssl") @@ -210,7 +214,7 @@ select_binary <- function(os = tolower(Sys.info()[["sysname"]]), lg("Building on %s %s", os, arch) binary <- NULL } - return(binary) + binary } # This tests that curl and OpenSSL are present (bc we can include their headers) @@ -268,7 +272,7 @@ get_macos_openssl_dir <- function() { openssl_root_dir <- "/usr/local" } } - return(openssl_root_dir) + openssl_root_dir } # (built with newer devtoolset but older glibc (2.17) for broader compatibility,# like manylinux2014) @@ -828,10 +832,10 @@ quietly <- !env_is("ARROW_R_DEV", "true") not_cran <- env_is("NOT_CRAN", "true") -if (is_release) { +if (is_release & !test_mode) { VERSION <- VERSION[1, 1:3] arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/") -} else { +} else if(!test_mode) { not_cran <- TRUE arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/") VERSION <- find_latest_nightly(VERSION) From f1820cb3d79a31945a9e0992fbedbc5d3b932ec2 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 7 Nov 2023 10:34:18 +0900 Subject: [PATCH 14/57] GH-38330: [C++][Azure] Use properties for input stream metadata (#38524) ### Rationale for this change We use user defined metadata for input stream metadata for now. But we should use properties returned from Azure like other remove filesystem implementations such as S3 and GCS. ### What changes are included in this PR? Convert `Azure::Storage::Blobs::Models::BlobProperties` to `KeyValueMetadata`. The following values aren't supported yet: * `BlobProperties::ObjectReplicationSourceProperties` * `BlobProperties::Metadata` If they need, we will add support for them as a follow-up task. ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes. * Closes: #38330 Lead-authored-by: Sutou Kouhei Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- cpp/src/arrow/filesystem/azurefs.cc | 150 +++++++++++++++++++++-- cpp/src/arrow/filesystem/azurefs_test.cc | 67 ++++++++-- 2 files changed, 201 insertions(+), 16 deletions(-) diff --git a/cpp/src/arrow/filesystem/azurefs.cc b/cpp/src/arrow/filesystem/azurefs.cc index 179be069b2acf..a04338d999d11 100644 --- a/cpp/src/arrow/filesystem/azurefs.cc +++ b/cpp/src/arrow/filesystem/azurefs.cc @@ -24,6 +24,7 @@ #include "arrow/filesystem/util_internal.h" #include "arrow/result.h" #include "arrow/util/checked_cast.h" +#include "arrow/util/formatting.h" #include "arrow/util/future.h" #include "arrow/util/key_value_metadata.h" #include "arrow/util/logging.h" @@ -150,13 +151,148 @@ Status ErrorToStatus(const std::string& prefix, return Status::IOError(prefix, " Azure Error: ", exception.what()); } -template -std::shared_ptr GetObjectMetadata(const ObjectResult& result) { - auto md = std::make_shared(); - for (auto prop : result) { - md->Append(prop.first, prop.second); +template +std::string FormatValue(typename TypeTraits::CType value) { + struct StringAppender { + std::string string; + Status operator()(std::string_view view) { + string.append(view.data(), view.size()); + return Status::OK(); + } + } appender; + arrow::internal::StringFormatter formatter; + ARROW_UNUSED(formatter(value, appender)); + return appender.string; +} + +std::shared_ptr PropertiesToMetadata( + const Azure::Storage::Blobs::Models::BlobProperties& properties) { + auto metadata = std::make_shared(); + // Not supported yet: + // * properties.ObjectReplicationSourceProperties + // * properties.Metadata + // + // They may have the same key defined in the following + // metadata->Append() list. If we have duplicated key in metadata, + // the first value may be only used by users because + // KeyValueMetadata::Get() returns the first found value. Note that + // users can use all values by using KeyValueMetadata::keys() and + // KeyValueMetadata::values(). + if (properties.ImmutabilityPolicy.HasValue()) { + metadata->Append("Immutability-Policy-Expires-On", + properties.ImmutabilityPolicy.Value().ExpiresOn.ToString()); + metadata->Append("Immutability-Policy-Mode", + properties.ImmutabilityPolicy.Value().PolicyMode.ToString()); + } + metadata->Append("Content-Type", properties.HttpHeaders.ContentType); + metadata->Append("Content-Encoding", properties.HttpHeaders.ContentEncoding); + metadata->Append("Content-Language", properties.HttpHeaders.ContentLanguage); + const auto& content_hash = properties.HttpHeaders.ContentHash.Value; + metadata->Append("Content-Hash", HexEncode(content_hash.data(), content_hash.size())); + metadata->Append("Content-Disposition", properties.HttpHeaders.ContentDisposition); + metadata->Append("Cache-Control", properties.HttpHeaders.CacheControl); + metadata->Append("Last-Modified", properties.LastModified.ToString()); + metadata->Append("Created-On", properties.CreatedOn.ToString()); + if (properties.ObjectReplicationDestinationPolicyId.HasValue()) { + metadata->Append("Object-Replication-Destination-Policy-Id", + properties.ObjectReplicationDestinationPolicyId.Value()); + } + metadata->Append("Blob-Type", properties.BlobType.ToString()); + if (properties.CopyCompletedOn.HasValue()) { + metadata->Append("Copy-Completed-On", properties.CopyCompletedOn.Value().ToString()); + } + if (properties.CopyStatusDescription.HasValue()) { + metadata->Append("Copy-Status-Description", properties.CopyStatusDescription.Value()); + } + if (properties.CopyId.HasValue()) { + metadata->Append("Copy-Id", properties.CopyId.Value()); + } + if (properties.CopyProgress.HasValue()) { + metadata->Append("Copy-Progress", properties.CopyProgress.Value()); + } + if (properties.CopySource.HasValue()) { + metadata->Append("Copy-Source", properties.CopySource.Value()); + } + if (properties.CopyStatus.HasValue()) { + metadata->Append("Copy-Status", properties.CopyStatus.Value().ToString()); + } + if (properties.IsIncrementalCopy.HasValue()) { + metadata->Append("Is-Incremental-Copy", + FormatValue(properties.IsIncrementalCopy.Value())); + } + if (properties.IncrementalCopyDestinationSnapshot.HasValue()) { + metadata->Append("Incremental-Copy-Destination-Snapshot", + properties.IncrementalCopyDestinationSnapshot.Value()); + } + if (properties.LeaseDuration.HasValue()) { + metadata->Append("Lease-Duration", properties.LeaseDuration.Value().ToString()); + } + if (properties.LeaseState.HasValue()) { + metadata->Append("Lease-State", properties.LeaseState.Value().ToString()); + } + if (properties.LeaseStatus.HasValue()) { + metadata->Append("Lease-Status", properties.LeaseStatus.Value().ToString()); + } + metadata->Append("Content-Length", FormatValue(properties.BlobSize)); + if (properties.ETag.HasValue()) { + metadata->Append("ETag", properties.ETag.ToString()); + } + if (properties.SequenceNumber.HasValue()) { + metadata->Append("Sequence-Number", + FormatValue(properties.SequenceNumber.Value())); + } + if (properties.CommittedBlockCount.HasValue()) { + metadata->Append("Committed-Block-Count", + FormatValue(properties.CommittedBlockCount.Value())); + } + metadata->Append("IsServerEncrypted", + FormatValue(properties.IsServerEncrypted)); + if (properties.EncryptionKeySha256.HasValue()) { + const auto& sha256 = properties.EncryptionKeySha256.Value(); + metadata->Append("Encryption-Key-Sha-256", HexEncode(sha256.data(), sha256.size())); + } + if (properties.EncryptionScope.HasValue()) { + metadata->Append("Encryption-Scope", properties.EncryptionScope.Value()); + } + if (properties.AccessTier.HasValue()) { + metadata->Append("Access-Tier", properties.AccessTier.Value().ToString()); + } + if (properties.IsAccessTierInferred.HasValue()) { + metadata->Append("Is-Access-Tier-Inferred", + FormatValue(properties.IsAccessTierInferred.Value())); + } + if (properties.ArchiveStatus.HasValue()) { + metadata->Append("Archive-Status", properties.ArchiveStatus.Value().ToString()); + } + if (properties.AccessTierChangedOn.HasValue()) { + metadata->Append("Access-Tier-Changed-On", + properties.AccessTierChangedOn.Value().ToString()); + } + if (properties.VersionId.HasValue()) { + metadata->Append("Version-Id", properties.VersionId.Value()); + } + if (properties.IsCurrentVersion.HasValue()) { + metadata->Append("Is-Current-Version", + FormatValue(properties.IsCurrentVersion.Value())); + } + if (properties.TagCount.HasValue()) { + metadata->Append("Tag-Count", FormatValue(properties.TagCount.Value())); + } + if (properties.ExpiresOn.HasValue()) { + metadata->Append("Expires-On", properties.ExpiresOn.Value().ToString()); + } + if (properties.IsSealed.HasValue()) { + metadata->Append("Is-Sealed", FormatValue(properties.IsSealed.Value())); + } + if (properties.RehydratePriority.HasValue()) { + metadata->Append("Rehydrate-Priority", + properties.RehydratePriority.Value().ToString()); + } + if (properties.LastAccessedOn.HasValue()) { + metadata->Append("Last-Accessed-On", properties.LastAccessedOn.Value().ToString()); } - return md; + metadata->Append("Has-Legal-Hold", FormatValue(properties.HasLegalHold)); + return metadata; } class ObjectInputFile final : public io::RandomAccessFile { @@ -176,7 +312,7 @@ class ObjectInputFile final : public io::RandomAccessFile { try { auto properties = blob_client_->GetProperties(); content_length_ = properties.Value.BlobSize; - metadata_ = GetObjectMetadata(properties.Value.Metadata); + metadata_ = PropertiesToMetadata(properties.Value); return Status::OK(); } catch (const Azure::Storage::StorageException& exception) { if (exception.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index 5d454bdc33f47..f57fc4d8140a0 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -51,6 +51,8 @@ #include "arrow/testing/util.h" #include "arrow/util/io_util.h" #include "arrow/util/key_value_metadata.h" +#include "arrow/util/string.h" +#include "arrow/util/value_parsing.h" namespace arrow { using internal::TemporaryDir; @@ -296,21 +298,68 @@ TEST_F(TestAzureFileSystem, OpenInputStreamTrailingSlash) { ASSERT_RAISES(IOError, fs_->OpenInputStream(PreexistingObjectPath() + '/')); } -TEST_F(TestAzureFileSystem, OpenInputStreamReadMetadata) { - const std::string object_name = "OpenInputStreamMetadataTest/simple.txt"; - - service_client_->GetBlobContainerClient(PreexistingContainerName()) - .GetBlobClient(PreexistingObjectName()) - .SetMetadata(Azure::Storage::Metadata{{"key0", "value0"}}); +namespace { +std::shared_ptr NormalizerKeyValueMetadata( + std::shared_ptr metadata) { + auto normalized = std::make_shared(); + for (int64_t i = 0; i < metadata->size(); ++i) { + auto key = metadata->key(i); + auto value = metadata->value(i); + if (key == "Content-Hash") { + std::vector output; + output.reserve(value.size() / 2); + if (ParseHexValues(value, output.data()).ok()) { + // Valid value + value = std::string(value.size(), 'F'); + } + } else if (key == "Last-Modified" || key == "Created-On" || + key == "Access-Tier-Changed-On") { + auto parser = TimestampParser::MakeISO8601(); + int64_t output; + if ((*parser)(value.data(), value.size(), TimeUnit::NANO, &output)) { + // Valid value + value = "2023-10-31T08:15:20Z"; + } + } else if (key == "ETag") { + if (internal::StartsWith(value, "\"") && internal::EndsWith(value, "\"")) { + // Valid value + value = "\"ETagValue\""; + } + } + normalized->Append(key, value); + } + return normalized; +} +}; // namespace +TEST_F(TestAzureFileSystem, OpenInputStreamReadMetadata) { std::shared_ptr stream; ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream(PreexistingObjectPath())); std::shared_ptr actual; ASSERT_OK_AND_ASSIGN(actual, stream->ReadMetadata()); - // TODO(GH-38330): This is asserting that the user defined metadata is returned but this - // is probably not the correct behaviour. - ASSERT_OK_AND_EQ("value0", actual->Get("key0")); + ASSERT_EQ( + "\n" + "-- metadata --\n" + "Content-Type: application/octet-stream\n" + "Content-Encoding: \n" + "Content-Language: \n" + "Content-Hash: FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n" + "Content-Disposition: \n" + "Cache-Control: \n" + "Last-Modified: 2023-10-31T08:15:20Z\n" + "Created-On: 2023-10-31T08:15:20Z\n" + "Blob-Type: BlockBlob\n" + "Lease-State: available\n" + "Lease-Status: unlocked\n" + "Content-Length: 447\n" + "ETag: \"ETagValue\"\n" + "IsServerEncrypted: true\n" + "Access-Tier: Hot\n" + "Is-Access-Tier-Inferred: true\n" + "Access-Tier-Changed-On: 2023-10-31T08:15:20Z\n" + "Has-Legal-Hold: false", + NormalizerKeyValueMetadata(actual)->ToString()); } TEST_F(TestAzureFileSystem, OpenInputStreamClosed) { From dbed49914eb0dab00250566b993a7b30123b963c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:00:23 +0900 Subject: [PATCH 15/57] MINOR: [C#] Bump BenchmarkDotNet.Diagnostics.Windows from 0.13.9 to 0.13.10 in /csharp (#38605) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [BenchmarkDotNet.Diagnostics.Windows](https://github.com/dotnet/BenchmarkDotNet) from 0.13.9 to 0.13.10.
Release notes

Sourced from BenchmarkDotNet.Diagnostics.Windows's releases.

0.13.10

Full changelog: https://benchmarkdotnet.org/changelog/v0.13.10.html

Highlights

Initial support of .NET 9 and minor bug fixes.

Details

In the v0.13.10 scope, 2 issues were resolved and 3 pull requests were merged. This release includes 10 commits by 4 contributors.

Resolved issues (2)

Merged pull requests (3)

Commits (10)

Contributors (4)

Thank you very much!

Commits
  • 40c4142 Prepare v0.13.10 changelog
  • ae49148 implement full .NET 9 support (#2456)
  • 0a734a9 feat: set RuntimeHostConfigurationOption on generated project (#2453)
  • e170684 Adjust 'Failed to set up high priority' message
  • 7b342f5 Add support for wasm/net9.0 (#2447)
  • e111368 [build] Bump Docfx.App: 2.71.0->2.71.1
  • af9c5c6 Exception handling in DotNetCliGenerator.GetRootDirectory, fix #2436
  • d17c6ad Support Windows 11 23H2 (10.0.22631) in OsBrandStringHelper
  • 2e96d29 Don't show AssemblyInformationalVersion metadata in BDN BrandVersion
  • c27152b Set next BenchmarkDotNet version: 0.13.10
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=BenchmarkDotNet.Diagnostics.Windows&package-manager=nuget&previous-version=0.13.9&new-version=0.13.10)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- .../test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj b/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj index 4cbab77e1fe24..df76fd4a7b45f 100644 --- a/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj +++ b/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj @@ -7,7 +7,7 @@ - + From 25c18d8cd6a299f3bb6b72966f2dca357db26399 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:00:57 +0900 Subject: [PATCH 16/57] MINOR: [JS] Bump eslint from 8.42.0 to 8.52.0 in /js (#38545) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [eslint](https://github.com/eslint/eslint) from 8.42.0 to 8.52.0.
Release notes

Sourced from eslint's releases.

v8.52.0

Features

  • 70648ee feat: report-unused-disable-directive to report unused eslint-enable (#17611) (Yosuke Ota)

Bug Fixes

  • 5de9637 fix: Ensure shared references in rule configs are separated (#17666) (Nicholas C. Zakas)
  • dcfe573 fix: add preceding semicolon in suggestions of no-object-constructor (#17649) (Francesco Trotta)

Documentation

  • 476d58a docs: Add note about invalid CLI flags when using flat config. (#17664) (Nicholas C. Zakas)
  • 660ed3a docs: Plugin flat config migration guide (#17640) (Nicholas C. Zakas)
  • a58aa20 docs: fix examples for several rules (#17645) (Milos Djermanovic)
  • 179929b docs: Remove trailing newline from the code of Playground links (#17641) (Francesco Trotta)
  • f8e5c30 docs: Update README (GitHub Actions Bot)
  • b7ef2f3 docs: Enable pretty code formatter output (#17635) (Nicholas C. Zakas)
  • 0bcb9a8 docs: Fix syntax errors in rule examples (#17633) (Francesco Trotta)
  • 61b9083 docs: Make no-continue example code work (#17643) (Zhongyuan Zhou)
  • 9fafe45 docs: upgrade to 11ty 2.0 (#17632) (Percy Ma)
  • ff8e4bf docs: Update README (GitHub Actions Bot)
  • fab249a docs: Update README (GitHub Actions Bot)
  • 392305b docs: Update no-irregular-whitespace and fix examples (#17626) (Francesco Trotta)
  • 6b8acfb docs: Add real whitespace to no-trailing-spaces examples (#17630) (Francesco Trotta)
  • 1000187 docs: Fix examples in unicode-bom (#17631) (Francesco Trotta)
  • 000290c docs: Update README (GitHub Actions Bot)

Chores

  • 6d1f0c2 chore: upgrade @​eslint/js@​8.52.0 (#17671) (Milos Djermanovic)
  • d63d4fe chore: package.json update for @​eslint/js release (ESLint Jenkins)
  • f30cefe test: fix FlatESLint tests for caching (#17658) (Milos Djermanovic)
  • ef650cb test: update tests for no-promise-executor-return (#17661) (Milos Djermanovic)

v8.51.0

Features

  • 0a9c433 feat: Add --no-warn-ignored CLI option for flat config (#17569) (Domantas Petrauskas)
  • 977e67e feat: logical-assignment-operators to report expressions with 3 operands (#17600) (Yosuke Ota)

Bug Fixes

  • f976b2f fix: make rule severity case-sensitive in flat config (#17619) (Milos Djermanovic)
  • 0edfe36 fix: Ensure crash error messages are not duplicated (#17584) (Nicholas C. Zakas)
  • dd79abc fix: eslint-disable to be able to parse quoted rule names (#17612) (Yosuke Ota)
  • d2f6801 fix: Ensure correct code path for && followed by ?? (#17618) (Nicholas C. Zakas)

Documentation

  • ee5be81 docs: default to sourceType: "module" in rule examples (#17615) (Francesco Trotta)
  • 1aa26df docs: Add more examples for multiline-ternary (#17610) (George Ashiotis)
  • 47d0b44 docs: Update README (GitHub Actions Bot)
  • dbf831e docs: use generated og image (#17601) (Percy Ma)
  • 1866da5 docs: Update README (GitHub Actions Bot)

Chores

... (truncated)

Changelog

Sourced from eslint's changelog.

v8.52.0 - October 20, 2023

  • 6d1f0c2 chore: upgrade @​eslint/js@​8.52.0 (#17671) (Milos Djermanovic)
  • d63d4fe chore: package.json update for @​eslint/js release (ESLint Jenkins)
  • 476d58a docs: Add note about invalid CLI flags when using flat config. (#17664) (Nicholas C. Zakas)
  • 5de9637 fix: Ensure shared references in rule configs are separated (#17666) (Nicholas C. Zakas)
  • f30cefe test: fix FlatESLint tests for caching (#17658) (Milos Djermanovic)
  • ef650cb test: update tests for no-promise-executor-return (#17661) (Milos Djermanovic)
  • 70648ee feat: report-unused-disable-directive to report unused eslint-enable (#17611) (Yosuke Ota)
  • dcfe573 fix: add preceding semicolon in suggestions of no-object-constructor (#17649) (Francesco Trotta)
  • 660ed3a docs: Plugin flat config migration guide (#17640) (Nicholas C. Zakas)
  • a58aa20 docs: fix examples for several rules (#17645) (Milos Djermanovic)
  • 179929b docs: Remove trailing newline from the code of Playground links (#17641) (Francesco Trotta)
  • f8e5c30 docs: Update README (GitHub Actions Bot)
  • b7ef2f3 docs: Enable pretty code formatter output (#17635) (Nicholas C. Zakas)
  • 0bcb9a8 docs: Fix syntax errors in rule examples (#17633) (Francesco Trotta)
  • 61b9083 docs: Make no-continue example code work (#17643) (Zhongyuan Zhou)
  • 9fafe45 docs: upgrade to 11ty 2.0 (#17632) (Percy Ma)
  • ff8e4bf docs: Update README (GitHub Actions Bot)
  • fab249a docs: Update README (GitHub Actions Bot)
  • 392305b docs: Update no-irregular-whitespace and fix examples (#17626) (Francesco Trotta)
  • 6b8acfb docs: Add real whitespace to no-trailing-spaces examples (#17630) (Francesco Trotta)
  • 1000187 docs: Fix examples in unicode-bom (#17631) (Francesco Trotta)
  • 000290c docs: Update README (GitHub Actions Bot)

v8.51.0 - October 6, 2023

  • 1ef39ea chore: upgrade @​eslint/js@​8.51.0 (#17624) (Milos Djermanovic)
  • f8c7403 chore: package.json update for @​eslint/js release (ESLint Jenkins)
  • f976b2f fix: make rule severity case-sensitive in flat config (#17619) (Milos Djermanovic)
  • 0edfe36 fix: Ensure crash error messages are not duplicated (#17584) (Nicholas C. Zakas)
  • ee5be81 docs: default to sourceType: "module" in rule examples (#17615) (Francesco Trotta)
  • dd79abc fix: eslint-disable to be able to parse quoted rule names (#17612) (Yosuke Ota)
  • d2f6801 fix: Ensure correct code path for && followed by ?? (#17618) (Nicholas C. Zakas)
  • 2665552 test: fix flat config linter tests to use Linter in flat config mode (#17616) (Milos Djermanovic)
  • 1aa26df docs: Add more examples for multiline-ternary (#17610) (George Ashiotis)
  • 47d0b44 docs: Update README (GitHub Actions Bot)
  • dbf831e docs: use generated og image (#17601) (Percy Ma)
  • 0a9c433 feat: Add --no-warn-ignored CLI option for flat config (#17569) (Domantas Petrauskas)
  • 1866da5 docs: Update README (GitHub Actions Bot)
  • 7b77bcc chore: Refactor CodePathState (#17510) (Nicholas C. Zakas)
  • 977e67e feat: logical-assignment-operators to report expressions with 3 operands (#17600) (Yosuke Ota)
  • bc77c9a chore: Document and refactor ForkContext (#17566) (Nicholas C. Zakas)
  • 24e1f14 chore: Refactor and document CodePath (#17558) (Nicholas C. Zakas)

v8.50.0 - September 22, 2023

  • f8a8a2d chore: upgrade @​eslint/js@​8.50.0 (#17599) (Milos Djermanovic)
  • 38ada6d chore: package.json update for @​eslint/js release (ESLint Jenkins)
  • 27d5a9e feat: add suggestions to array-callback-return (#17590) (Tanuj Kanti)

... (truncated)

Commits
  • 331cf62 8.52.0
  • 7dc28ed Build: changelog update for 8.52.0
  • 6d1f0c2 chore: upgrade @​eslint/js@​8.52.0 (#17671)
  • d63d4fe chore: package.json update for @​eslint/js release
  • 476d58a docs: Add note about invalid CLI flags when using flat config. (#17664)
  • 5de9637 fix: Ensure shared references in rule configs are separated (#17666)
  • f30cefe test: fix FlatESLint tests for caching (#17658)
  • ef650cb test: update tests for no-promise-executor-return (#17661)
  • 70648ee feat: report-unused-disable-directive to report unused eslint-enable (#17611)
  • dcfe573 fix: add preceding semicolon in suggestions of no-object-constructor (#17649)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=eslint&package-manager=npm_and_yarn&previous-version=8.42.0&new-version=8.52.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/package.json | 2 +- js/yarn.lock | 92 ++++++++++++++++++++++++++----------------------- 2 files changed, 49 insertions(+), 45 deletions(-) diff --git a/js/package.json b/js/package.json index cca333ad3d006..33bc4849903ef 100644 --- a/js/package.json +++ b/js/package.json @@ -83,7 +83,7 @@ "del-cli": "5.1.0", "esbuild": "0.19.2", "esbuild-plugin-alias": "0.2.1", - "eslint": "8.42.0", + "eslint": "8.52.0", "eslint-plugin-jest": "27.4.2", "eslint-plugin-unicorn": "47.0.0", "esm": "https://github.com/jsg2021/esm/releases/download/v3.x.x-pr883/esm-3.x.x-pr883.tgz", diff --git a/js/yarn.lock b/js/yarn.lock index 26a459a73a206..eddf380d1ffad 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -626,14 +626,14 @@ dependencies: eslint-visitor-keys "^3.3.0" -"@eslint-community/regexpp@^4.4.0": - version "4.8.1" - resolved "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.8.1.tgz#8c4bb756cc2aa7eaf13cfa5e69c83afb3260c20c" - integrity sha512-PWiOzLIUAjN/w5K17PoF4n6sKBw0gqLHPhywmYHP4t1VFQQVYeb1yWsJwnMVEMl3tUHME7X/SJPZLmtG7XBDxQ== +"@eslint-community/regexpp@^4.4.0", "@eslint-community/regexpp@^4.6.1": + version "4.10.0" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.10.0.tgz#548f6de556857c8bb73bbee70c35dc82a2e74d63" + integrity sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA== -"@eslint/eslintrc@^2.0.3": +"@eslint/eslintrc@^2.1.2": version "2.1.2" - resolved "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.2.tgz#c6936b4b328c64496692f76944e755738be62396" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.1.2.tgz#c6936b4b328c64496692f76944e755738be62396" integrity sha512-+wvgpDsrB1YqAMdEUCcnTlpfVBH7Vqn6A/NT3D8WVXFIaKMlErPIZT3oCIAVCOtarRpMtelZLqJeU3t7WY6X6g== dependencies: ajv "^6.12.4" @@ -646,10 +646,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.42.0": - version "8.42.0" - resolved "https://registry.npmjs.org/@eslint/js/-/js-8.42.0.tgz#484a1d638de2911e6f5a30c12f49c7e4a3270fb6" - integrity sha512-6SWlXpWU5AvId8Ac7zjzmIOqMOba/JWY8XZ4A7q7Gn1Vlfg/SFFIlrtHXt9nPn4op9ZPAkl91Jao+QQv3r/ukw== +"@eslint/js@8.52.0": + version "8.52.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.52.0.tgz#78fe5f117840f69dc4a353adf9b9cd926353378c" + integrity sha512-mjZVbpaeMZludF2fsWLD0Z9gCref1Tk4i9+wddjRvpUNqqcndPkBD09N/Mapey0b3jaXbLm2kICwFv2E64QinA== "@gulp-sourcemaps/identity-map@^2.0.1": version "2.0.1" @@ -670,12 +670,12 @@ normalize-path "^2.0.1" through2 "^2.0.3" -"@humanwhocodes/config-array@^0.11.10": - version "0.11.11" - resolved "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.11.tgz#88a04c570dbbc7dd943e4712429c3df09bc32844" - integrity sha512-N2brEuAadi0CcdeMXUkhbZB84eskAc8MEX1By6qEchoVywSgXPIjou4rYsl0V3Hj0ZnuGycGCjdNgockbzeWNA== +"@humanwhocodes/config-array@^0.11.13": + version "0.11.13" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz#075dc9684f40a531d9b26b0822153c1e832ee297" + integrity sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ== dependencies: - "@humanwhocodes/object-schema" "^1.2.1" + "@humanwhocodes/object-schema" "^2.0.1" debug "^4.1.1" minimatch "^3.0.5" @@ -684,10 +684,10 @@ resolved "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== -"@humanwhocodes/object-schema@^1.2.1": - version "1.2.1" - resolved "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" - integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@humanwhocodes/object-schema@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz#e5211452df060fa8522b55c7b3c0c4d1981cb044" + integrity sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw== "@isaacs/cliui@^8.0.2": version "8.0.2" @@ -1548,6 +1548,11 @@ "@typescript-eslint/types" "5.62.0" eslint-visitor-keys "^3.3.0" +"@ungap/structured-clone@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" + integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== + "@webassemblyjs/ast@1.11.6", "@webassemblyjs/ast@^1.11.5": version "1.11.6" resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.6.tgz#db046555d3c413f8966ca50a95176a0e2c642e24" @@ -1717,7 +1722,7 @@ ajv-keywords@^3.5.2: resolved "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d" integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== -ajv@^6.10.0, ajv@^6.12.4, ajv@^6.12.5: +ajv@^6.12.4, ajv@^6.12.5: version "6.12.6" resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -3030,40 +3035,41 @@ eslint-scope@5.1.1, eslint-scope@^5.1.1: esrecurse "^4.3.0" estraverse "^4.1.1" -eslint-scope@^7.2.0: +eslint-scope@^7.2.2: version "7.2.2" - resolved "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz#deb4f92563390f32006894af62a22dba1c46423f" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.2.2.tgz#deb4f92563390f32006894af62a22dba1c46423f" integrity sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg== dependencies: esrecurse "^4.3.0" estraverse "^5.2.0" -eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1: +eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4.3: version "3.4.3" resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== -eslint@8.42.0: - version "8.42.0" - resolved "https://registry.npmjs.org/eslint/-/eslint-8.42.0.tgz#7bebdc3a55f9ed7167251fe7259f75219cade291" - integrity sha512-ulg9Ms6E1WPf67PHaEY4/6E2tEn5/f7FXGzr3t9cBMugOmf1INYvuUwwh1aXQN4MfJ6a5K2iNwP3w4AColvI9A== +eslint@8.52.0: + version "8.52.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.52.0.tgz#d0cd4a1fac06427a61ef9242b9353f36ea7062fc" + integrity sha512-zh/JHnaixqHZsolRB/w9/02akBk9EPrOs9JwcTP2ek7yL5bVvXuRariiaAjjoJ5DvuwQ1WAE/HsMz+w17YgBCg== dependencies: "@eslint-community/eslint-utils" "^4.2.0" - "@eslint-community/regexpp" "^4.4.0" - "@eslint/eslintrc" "^2.0.3" - "@eslint/js" "8.42.0" - "@humanwhocodes/config-array" "^0.11.10" + "@eslint-community/regexpp" "^4.6.1" + "@eslint/eslintrc" "^2.1.2" + "@eslint/js" "8.52.0" + "@humanwhocodes/config-array" "^0.11.13" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" - ajv "^6.10.0" + "@ungap/structured-clone" "^1.2.0" + ajv "^6.12.4" chalk "^4.0.0" cross-spawn "^7.0.2" debug "^4.3.2" doctrine "^3.0.0" escape-string-regexp "^4.0.0" - eslint-scope "^7.2.0" - eslint-visitor-keys "^3.4.1" - espree "^9.5.2" + eslint-scope "^7.2.2" + eslint-visitor-keys "^3.4.3" + espree "^9.6.1" esquery "^1.4.2" esutils "^2.0.2" fast-deep-equal "^3.1.3" @@ -3073,7 +3079,6 @@ eslint@8.42.0: globals "^13.19.0" graphemer "^1.4.0" ignore "^5.2.0" - import-fresh "^3.0.0" imurmurhash "^0.1.4" is-glob "^4.0.0" is-path-inside "^3.0.3" @@ -3083,18 +3088,17 @@ eslint@8.42.0: lodash.merge "^4.6.2" minimatch "^3.1.2" natural-compare "^1.4.0" - optionator "^0.9.1" + optionator "^0.9.3" strip-ansi "^6.0.1" - strip-json-comments "^3.1.0" text-table "^0.2.0" "esm@https://github.com/jsg2021/esm/releases/download/v3.x.x-pr883/esm-3.x.x-pr883.tgz": version "3.2.25" resolved "https://github.com/jsg2021/esm/releases/download/v3.x.x-pr883/esm-3.x.x-pr883.tgz#c463cfa4e14aceea6b7cd7e669ef90de072ea60a" -espree@^9.5.2, espree@^9.6.0: +espree@^9.6.0, espree@^9.6.1: version "9.6.1" - resolved "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz#a2a17b8e434690a5432f2f8018ce71d331a48c6f" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.6.1.tgz#a2a17b8e434690a5432f2f8018ce71d331a48c6f" integrity sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ== dependencies: acorn "^8.9.0" @@ -3987,7 +3991,7 @@ ignore@^5.2.0, ignore@^5.2.4: resolved "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ== -import-fresh@^3.0.0, import-fresh@^3.2.1: +import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== @@ -5570,9 +5574,9 @@ opener@^1.5.2: resolved "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598" integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A== -optionator@^0.9.1: +optionator@^0.9.3: version "0.9.3" - resolved "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz#007397d44ed1872fdc6ed31360190f81814e2c64" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.3.tgz#007397d44ed1872fdc6ed31360190f81814e2c64" integrity sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg== dependencies: "@aashutoshrathi/word-wrap" "^1.2.3" @@ -6679,7 +6683,7 @@ strip-indent@^4.0.0: dependencies: min-indent "^1.0.1" -strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: +strip-json-comments@^3.1.1: version "3.1.1" resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== From fafd48c3196b4e6af0c09f633c6a3440419bf1f0 Mon Sep 17 00:00:00 2001 From: James Duong Date: Tue, 7 Nov 2023 08:40:38 -0800 Subject: [PATCH 17/57] GH-38576: [Java] Change JDBC driver to optionally preserve cookies and auth tokens when getting streams (#38580) ### Rationale for this change This change restores the original behavior of transmitting existing cookies and auth tokens when getting separate streams returned by getFlightInfo after adding support for multiple endpoints. These properties are now optional though. ### What changes are included in this PR? - Change the JDBC driver to add new properties "retainCookies" and "retainAuth" - These properties allow internally spawned connections for getting streams to use the cookies and bearer tokens from the original connection. - Add tests for validating defaults from ArrowFlightSqlClient.Builder ### Are these changes tested? Unit tests have been added. ### Are there any user-facing changes? Yes. There are now properties and they are documented. * Closes: #38576 Authored-by: James Duong Signed-off-by: David Li --- docs/source/java/flight_sql_jdbc_driver.rst | 11 ++ .../ClientIncomingAuthHeaderMiddleware.java | 2 +- .../driver/jdbc/ArrowFlightConnection.java | 2 + .../client/ArrowFlightSqlClientHandler.java | 117 +++++++++++--- .../ArrowFlightConnectionConfigImpl.java | 20 ++- .../driver/jdbc/ConnectionMutualTlsTest.java | 56 ++++--- .../arrow/driver/jdbc/ConnectionTest.java | 123 ++++++++------- .../jdbc/ConnectionTlsRootCertsTest.java | 68 +++++---- .../arrow/driver/jdbc/ConnectionTlsTest.java | 56 +++---- .../driver/jdbc/FlightServerTestRule.java | 5 +- .../arrow/driver/jdbc/ResultSetTest.java | 15 +- ...rrowFlightSqlClientHandlerBuilderTest.java | 143 ++++++++++++++++++ 12 files changed, 448 insertions(+), 170 deletions(-) create mode 100644 java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java diff --git a/docs/source/java/flight_sql_jdbc_driver.rst b/docs/source/java/flight_sql_jdbc_driver.rst index 34ccfea47f9e3..0ace2185983a9 100644 --- a/docs/source/java/flight_sql_jdbc_driver.rst +++ b/docs/source/java/flight_sql_jdbc_driver.rst @@ -141,6 +141,17 @@ case-sensitive. The supported parameters are: - true - When TLS is enabled, whether to use the system certificate store + * - retainCookies + - true + - Whether to use cookies from the initial connection in subsequent + internal connections when retrieving streams from separate endpoints. + + * - retainAuth + - true + - Whether to use bearer tokens obtained from the initial connection + in subsequent internal connections used for retrieving streams + from separate endpoints. + Note that URI values must be URI-encoded if they contain characters such as !, @, $, etc. diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientIncomingAuthHeaderMiddleware.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientIncomingAuthHeaderMiddleware.java index be5f3f54d326c..7bb55d145d104 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientIncomingAuthHeaderMiddleware.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/ClientIncomingAuthHeaderMiddleware.java @@ -34,7 +34,7 @@ public class ClientIncomingAuthHeaderMiddleware implements FlightClientMiddlewar */ public static class Factory implements FlightClientMiddleware.Factory { private final ClientHeaderHandler headerHandler; - private CredentialCallOption credentialCallOption; + private CredentialCallOption credentialCallOption = null; /** * Construct a factory with the given header handler. diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index fdbb9381c0a55..ad19c616ff29a 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -109,6 +109,8 @@ private static ArrowFlightSqlClientHandler createNewClientHandler( .withDisableCertificateVerification(config.getDisableCertificateVerification()) .withToken(config.getToken()) .withCallOptions(config.toCallOption()) + .withRetainCookies(config.retainCookies()) + .withRetainAuth(config.retainAuth()) .build(); } catch (final SQLException e) { try { diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 75e80d45dc669..54fd17853c00b 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -49,6 +49,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.util.VisibleForTesting; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.calcite.avatica.Meta.StatementType; @@ -425,21 +426,59 @@ public static final class Builder { private final Set options = new HashSet<>(); private String host; private int port; - private String username; - private String password; - private String trustStorePath; - private String trustStorePassword; - private String token; - private boolean useEncryption; - private boolean disableCertificateVerification; - private boolean useSystemTrustStore; - private String tlsRootCertificatesPath; - private String clientCertificatePath; - private String clientKeyPath; + + @VisibleForTesting + String username; + + @VisibleForTesting + String password; + + @VisibleForTesting + String trustStorePath; + + @VisibleForTesting + String trustStorePassword; + + @VisibleForTesting + String token; + + @VisibleForTesting + boolean useEncryption = true; + + @VisibleForTesting + boolean disableCertificateVerification; + + @VisibleForTesting + boolean useSystemTrustStore = true; + + @VisibleForTesting + String tlsRootCertificatesPath; + + @VisibleForTesting + String clientCertificatePath; + + @VisibleForTesting + String clientKeyPath; + + @VisibleForTesting private BufferAllocator allocator; - public Builder() { + @VisibleForTesting + boolean retainCookies = true; + + @VisibleForTesting + boolean retainAuth = true; + + // These two middlewares are for internal use within build() and should not be exposed by builder APIs. + // Note that these middlewares may not necessarily be registered. + @VisibleForTesting + ClientIncomingAuthHeaderMiddleware.Factory authFactory + = new ClientIncomingAuthHeaderMiddleware.Factory(new ClientBearerHeaderHandler()); + @VisibleForTesting + ClientCookieMiddleware.Factory cookieFactory = new ClientCookieMiddleware.Factory(); + + public Builder() { } /** @@ -447,7 +486,8 @@ public Builder() { * * @param original The builder to base this copy off of. */ - private Builder(Builder original) { + @VisibleForTesting + Builder(Builder original) { this.middlewareFactories.addAll(original.middlewareFactories); this.options.addAll(original.options); this.host = original.host; @@ -464,6 +504,14 @@ private Builder(Builder original) { this.clientCertificatePath = original.clientCertificatePath; this.clientKeyPath = original.clientKeyPath; this.allocator = original.allocator; + + if (original.retainCookies) { + this.cookieFactory = original.cookieFactory; + } + + if (original.retainAuth) { + this.authFactory = original.authFactory; + } } /** @@ -622,6 +670,28 @@ public Builder withBufferAllocator(final BufferAllocator allocator) { return this; } + /** + * Indicates if cookies should be re-used by connections spawned for getStreams() calls. + * @param retainCookies The flag indicating if cookies should be re-used. + * @return this builder instance. + */ + public Builder withRetainCookies(boolean retainCookies) { + this.retainCookies = retainCookies; + return this; + } + + /** + * Indicates if bearer tokens negotiated should be re-used by connections + * spawned for getStreams() calls. + * + * @param retainAuth The flag indicating if auth tokens should be re-used. + * @return this builder instance. + */ + public Builder withRetainAuth(boolean retainAuth) { + this.retainAuth = retainAuth; + return this; + } + /** * Adds the provided {@code factories} to the list of {@link #middlewareFactories} of this handler. * @@ -675,13 +745,11 @@ public ArrowFlightSqlClientHandler build() throws SQLException { // Copy middlewares so that the build method doesn't change the state of the builder fields itself. Set buildTimeMiddlewareFactories = new HashSet<>(this.middlewareFactories); FlightClient client = null; + boolean isUsingUserPasswordAuth = username != null && token == null; try { - ClientIncomingAuthHeaderMiddleware.Factory authFactory = null; // Token should take priority since some apps pass in a username/password even when a token is provided - if (username != null && token == null) { - authFactory = - new ClientIncomingAuthHeaderMiddleware.Factory(new ClientBearerHeaderHandler()); + if (isUsingUserPasswordAuth) { buildTimeMiddlewareFactories.add(authFactory); } final FlightClient.Builder clientBuilder = FlightClient.builder().allocator(allocator); @@ -722,10 +790,17 @@ public ArrowFlightSqlClientHandler build() throws SQLException { client = clientBuilder.build(); final ArrayList credentialOptions = new ArrayList<>(); - if (authFactory != null) { - credentialOptions.add( - ClientAuthenticationUtils.getAuthenticate( - client, username, password, authFactory, options.toArray(new CallOption[0]))); + if (isUsingUserPasswordAuth) { + // If the authFactory has already been used for a handshake, use the existing token. + // This can occur if the authFactory is being re-used for a new connection spawned for getStream(). + if (authFactory.getCredentialCallOption() != null) { + credentialOptions.add(authFactory.getCredentialCallOption()); + } else { + // Otherwise do the handshake and get the token if possible. + credentialOptions.add( + ClientAuthenticationUtils.getAuthenticate( + client, username, password, authFactory, options.toArray(new CallOption[0]))); + } } else if (token != null) { credentialOptions.add( ClientAuthenticationUtils.getAuthenticate( diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java index 59118e1d6f788..6237a8b58d68a 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java @@ -143,6 +143,22 @@ public int threadPoolSize() { return ArrowFlightConnectionProperty.THREAD_POOL_SIZE.getInteger(properties); } + /** + * Indicates if sub-connections created for stream retrieval + * should reuse cookies from the main connection. + */ + public boolean retainCookies() { + return ArrowFlightConnectionProperty.RETAIN_COOKIES.getBoolean(properties); + } + + /** + * Indicates if sub-connections created for stream retrieval + * should reuse bearer tokens created from the main connection. + */ + public boolean retainAuth() { + return ArrowFlightConnectionProperty.RETAIN_AUTH.getBoolean(properties); + } + /** * Gets the {@link CallOption}s from this {@link ConnectionConfig}. * @@ -191,7 +207,9 @@ public enum ArrowFlightConnectionProperty implements ConnectionProperty { CLIENT_CERTIFICATE("clientCertificate", null, Type.STRING, false), CLIENT_KEY("clientKey", null, Type.STRING, false), THREAD_POOL_SIZE("threadPoolSize", 1, Type.NUMBER, false), - TOKEN("token", null, Type.STRING, false); + TOKEN("token", null, Type.STRING, false), + RETAIN_COOKIES("retainCookies", true, Type.BOOLEAN, false), + RETAIN_AUTH("retainAuth", true, Type.BOOLEAN, false); private final String camelName; private final Object defaultValue; diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java index 783e0c41e9269..927b3e426c6ba 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java @@ -140,17 +140,21 @@ public void testGetEncryptedClientWithBadMTlsCertPath() { final UsernamePasswordCredentials credentials = new UsernamePasswordCredentials( userTest, passTest); - assertThrows(SQLException.class, () -> new ArrowFlightSqlClientHandler.Builder() - .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) - .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) - .withUsername(credentials.getUserName()) - .withPassword(credentials.getPassword()) - .withTlsRootCertificates(tlsRootCertsPath) - .withClientCertificate(badClientMTlsCertPath) - .withClientKey(clientMTlsKeyPath) - .withBufferAllocator(allocator) - .withEncryption(true) - .build()); + assertThrows(SQLException.class, () -> { + try (ArrowFlightSqlClientHandler handler = new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withUsername(credentials.getUserName()) + .withPassword(credentials.getPassword()) + .withTlsRootCertificates(tlsRootCertsPath) + .withClientCertificate(badClientMTlsCertPath) + .withClientKey(clientMTlsKeyPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()) { + Assert.fail(); + } + }); } /** @@ -162,17 +166,21 @@ public void testGetEncryptedClientWithBadMTlsKeyPath() { final UsernamePasswordCredentials credentials = new UsernamePasswordCredentials( userTest, passTest); - assertThrows(SQLException.class, () -> new ArrowFlightSqlClientHandler.Builder() - .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) - .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) - .withUsername(credentials.getUserName()) - .withPassword(credentials.getPassword()) - .withTlsRootCertificates(tlsRootCertsPath) - .withClientCertificate(clientMTlsCertPath) - .withClientKey(badClientMTlsKeyPath) - .withBufferAllocator(allocator) - .withEncryption(true) - .build()); + assertThrows(SQLException.class, () -> { + try (ArrowFlightSqlClientHandler handler = new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withUsername(credentials.getUserName()) + .withPassword(credentials.getPassword()) + .withTlsRootCertificates(tlsRootCertsPath) + .withClientCertificate(clientMTlsCertPath) + .withClientKey(badClientMTlsKeyPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()) { + Assert.fail(); + } + }); } /** @@ -222,7 +230,7 @@ public void testGetEncryptedConnectionWithValidCredentialsAndTlsRootsPath() thro final ArrowFlightJdbcDataSource dataSource = ArrowFlightJdbcDataSource.createNewDataSource(properties); try (final Connection connection = dataSource.getConnection()) { - assert connection.isValid(300); + Assert.assertTrue(connection.isValid(300)); } } @@ -245,7 +253,7 @@ public void testGetNonAuthenticatedEncryptedConnection() throws Exception { final ArrowFlightJdbcDataSource dataSource = ArrowFlightJdbcDataSource.createNewDataSource(properties); try (final Connection connection = dataSource.getConnection()) { - assert connection.isValid(300); + Assert.assertTrue(connection.isValid(300)); } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java index bec0ff1e59752..357506b3d177c 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.net.URISyntaxException; import java.sql.Connection; @@ -100,7 +101,7 @@ public void testUnencryptedConnectionShouldOpenSuccessfullyWhenProvidedValidCred try (Connection connection = DriverManager.getConnection( "jdbc:arrow-flight-sql://" + FLIGHT_SERVER_TEST_RULE.getHost() + ":" + FLIGHT_SERVER_TEST_RULE.getPort(), properties)) { - assert connection.isValid(300); + Assert.assertTrue(connection.isValid(300)); } } @@ -122,11 +123,13 @@ public void testTokenOverridesUsernameAndPasswordAuth() { properties.put(ArrowFlightConnectionProperty.TOKEN.camelName(), "token"); properties.put("useEncryption", false); - SQLException e = assertThrows(SQLException.class, () -> - DriverManager.getConnection( - "jdbc:arrow-flight-sql://" + FLIGHT_SERVER_TEST_RULE.getHost() + ":" + - FLIGHT_SERVER_TEST_RULE.getPort(), - properties)); + SQLException e = assertThrows(SQLException.class, () -> { + try (Connection conn = DriverManager.getConnection( + "jdbc:arrow-flight-sql://" + FLIGHT_SERVER_TEST_RULE.getHost() + ":" + FLIGHT_SERVER_TEST_RULE.getPort(), + properties)) { + Assert.fail(); + } + }); assertTrue(e.getMessage().contains("UNAUTHENTICATED")); } @@ -145,7 +148,9 @@ public void testUnencryptedConnectionWithEmptyHost() properties.put("password", passTest); final String invalidUrl = "jdbc:arrow-flight-sql://"; - DriverManager.getConnection(invalidUrl, properties); + try (Connection conn = DriverManager.getConnection(invalidUrl, properties)) { + Assert.fail("Expected SQLException."); + } } /** @@ -161,6 +166,7 @@ public void testGetBasicClientAuthenticatedShouldOpenConnection() new ArrowFlightSqlClientHandler.Builder() .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withEncryption(false) .withUsername(userTest) .withPassword(passTest) .withBufferAllocator(allocator) @@ -191,7 +197,9 @@ public void testUnencryptedConnectionProvidingInvalidPort() final String invalidUrl = "jdbc:arrow-flight-sql://" + FLIGHT_SERVER_TEST_RULE.getHost() + ":" + 65537; - DriverManager.getConnection(invalidUrl, properties); + try (Connection conn = DriverManager.getConnection(invalidUrl, properties)) { + fail("Expected SQLException"); + } } /** @@ -206,6 +214,7 @@ public void testGetBasicClientNoAuthShouldOpenConnection() throws Exception { new ArrowFlightSqlClientHandler.Builder() .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) .withBufferAllocator(allocator) + .withEncryption(false) .build()) { assertNotNull(client); } @@ -228,7 +237,7 @@ public void testUnencryptedConnectionShouldOpenSuccessfullyWithoutAuthentication false); try (Connection connection = DriverManager .getConnection("jdbc:arrow-flight-sql://localhost:32010", properties)) { - assert connection.isValid(300); + Assert.assertTrue(connection.isValid(300)); } } @@ -271,14 +280,14 @@ public void testTLSConnectionPropertyFalseCorrectCastUrlWithDriverManager() thro final Driver driver = new ArrowFlightJdbcDriver(); DriverManager.registerDriver(driver); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s&useEncryption=false", FLIGHT_SERVER_TEST_RULE.getPort(), userTest, - passTest)); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + passTest))) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -301,13 +310,13 @@ public void testTLSConnectionPropertyFalseCorrectCastUrlAndPropertiesUsingSetPro passTest); properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "false"); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -329,13 +338,13 @@ public void testTLSConnectionPropertyFalseCorrectCastUrlAndPropertiesUsingPutWit passTest); properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), false); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -350,14 +359,14 @@ public void testTLSConnectionPropertyFalseIntegerCorrectCastUrlWithDriverManager final Driver driver = new ArrowFlightJdbcDriver(); DriverManager.registerDriver(driver); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s&useEncryption=0", FLIGHT_SERVER_TEST_RULE.getPort(), userTest, - passTest)); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + passTest))) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -380,13 +389,13 @@ public void testTLSConnectionPropertyFalseIntegerCorrectCastUrlAndPropertiesUsin passTest); properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "0"); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -409,13 +418,13 @@ public void testTLSConnectionPropertyFalseIntegerCorrectCastUrlAndPropertiesUsin passTest); properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), 0); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -430,15 +439,15 @@ public void testThreadPoolSizeConnectionPropertyCorrectCastUrlWithDriverManager( final Driver driver = new ArrowFlightJdbcDriver(); DriverManager.registerDriver(driver); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s&threadPoolSize=1&useEncryption=%s", FLIGHT_SERVER_TEST_RULE.getPort(), userTest, passTest, - false)); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + false))) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -462,13 +471,13 @@ public void testThreadPoolSizeConnectionPropertyCorrectCastUrlAndPropertiesUsing properties.setProperty(ArrowFlightConnectionProperty.THREAD_POOL_SIZE.camelName(), "1"); properties.put("useEncryption", false); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -492,13 +501,13 @@ public void testThreadPoolSizeConnectionPropertyCorrectCastUrlAndPropertiesUsing properties.put(ArrowFlightConnectionProperty.THREAD_POOL_SIZE.camelName(), 1); properties.put("useEncryption", false); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -513,15 +522,15 @@ public void testPasswordConnectionPropertyIntegerCorrectCastUrlWithDriverManager final Driver driver = new ArrowFlightJdbcDriver(); DriverManager.registerDriver(driver); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s&useEncryption=%s", FLIGHT_SERVER_TEST_RULE.getPort(), userTest, passTest, - false)); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + false))) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -544,13 +553,13 @@ public void testPasswordConnectionPropertyIntegerCorrectCastUrlAndPropertiesUsin passTest); properties.put("useEncryption", false); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -573,12 +582,12 @@ public void testPasswordConnectionPropertyIntegerCorrectCastUrlAndPropertiesUsin passTest); properties.put("useEncryption", false); - Connection connection = DriverManager.getConnection( + try (Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsRootCertsTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsRootCertsTest.java index 333c474824e52..5579cf0cf5f54 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsRootCertsTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsRootCertsTest.java @@ -118,12 +118,16 @@ public void testGetEncryptedClientAuthenticated() throws Exception { */ @Test public void testGetEncryptedClientWithNoCertificateOnKeyStore() { - assertThrows(SQLException.class, () -> new ArrowFlightSqlClientHandler.Builder() - .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) - .withTlsRootCertificates(badTlsRootCertsPath) - .withBufferAllocator(allocator) - .withEncryption(true) - .build()); + assertThrows(SQLException.class, () -> { + try (ArrowFlightSqlClientHandler handler = new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withTlsRootCertificates(badTlsRootCertsPath) + .withBufferAllocator(allocator) + .withEncryption(true) + .build()) { + Assert.fail(); + } + }); } /** @@ -167,7 +171,7 @@ public void testGetEncryptedConnectionWithValidCredentialsAndTlsRootsPath() thro final ArrowFlightJdbcDataSource dataSource = ArrowFlightJdbcDataSource.createNewDataSource(properties); try (final Connection connection = dataSource.getConnection()) { - assert connection.isValid(300); + Assert.assertTrue(connection.isValid(300)); } } @@ -188,7 +192,7 @@ public void testGetNonAuthenticatedEncryptedConnection() throws Exception { final ArrowFlightJdbcDataSource dataSource = ArrowFlightJdbcDataSource.createNewDataSource(properties); try (final Connection connection = dataSource.getConnection()) { - assert connection.isValid(300); + Assert.assertTrue(connection.isValid(300)); } } @@ -203,7 +207,7 @@ public void testTLSConnectionPropertyTrueCorrectCastUrlWithDriverManager() throw final Driver driver = new ArrowFlightJdbcDriver(); DriverManager.registerDriver(driver); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s" + "&useEncryption=true&%s=%s", @@ -211,9 +215,9 @@ public void testTLSConnectionPropertyTrueCorrectCastUrlWithDriverManager() throw userTest, passTest, ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), - URLEncoder.encode(tlsRootCertsPath, "UTF-8"))); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + URLEncoder.encode(tlsRootCertsPath, "UTF-8")))) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -235,13 +239,13 @@ public void testTLSConnectionPropertyTrueCorrectCastUrlAndPropertiesUsingSetProp properties.setProperty(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "true"); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -263,13 +267,13 @@ public void testTLSConnectionPropertyTrueCorrectCastUrlAndPropertiesUsingPutWith properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), true); properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -284,7 +288,7 @@ public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlWithDriverManager( final Driver driver = new ArrowFlightJdbcDriver(); DriverManager.registerDriver(driver); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s" + "&useEncryption=1&useSystemTrustStore=0&%s=%s", @@ -292,9 +296,9 @@ public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlWithDriverManager( userTest, passTest, ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), - URLEncoder.encode(tlsRootCertsPath, "UTF-8"))); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + URLEncoder.encode(tlsRootCertsPath, "UTF-8")))) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -316,11 +320,11 @@ public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlAndPropertiesUsing properties.setProperty(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "1"); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format("jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -342,11 +346,11 @@ public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlAndPropertiesUsing properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), 1); properties.put(ArrowFlightConnectionProperty.TLS_ROOT_CERTS.camelName(), tlsRootCertsPath); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format("jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsTest.java index 95d591766a836..7e160f3f0c385 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTlsTest.java @@ -127,6 +127,7 @@ public void testGetEncryptedClientAuthenticated() throws Exception { new ArrowFlightSqlClientHandler.Builder() .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withSystemTrustStore(false) .withUsername(credentials.getUserName()) .withPassword(credentials.getPassword()) .withTrustStorePath(trustStorePath) @@ -153,6 +154,7 @@ public void testGetEncryptedClientWithNoCertificateOnKeyStore() throws Exception .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) .withTrustStorePath(noCertificateKeyStorePath) .withTrustStorePassword(noCertificateKeyStorePassword) + .withSystemTrustStore(false) .withBufferAllocator(allocator) .withEncryption(true) .build()) { @@ -170,6 +172,7 @@ public void testGetNonAuthenticatedEncryptedClientNoAuth() throws Exception { try (ArrowFlightSqlClientHandler client = new ArrowFlightSqlClientHandler.Builder() .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withSystemTrustStore(false) .withTrustStorePath(trustStorePath) .withTrustStorePassword(trustStorePass) .withBufferAllocator(allocator) @@ -192,6 +195,7 @@ public void testGetEncryptedClientWithKeyStoreBadPasswordAndNoAuth() throws Exce try (ArrowFlightSqlClientHandler ignored = new ArrowFlightSqlClientHandler.Builder() .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withSystemTrustStore(false) .withTrustStorePath(trustStorePath) .withTrustStorePassword(keyStoreBadPassword) .withBufferAllocator(allocator) @@ -225,7 +229,7 @@ public void testGetEncryptedConnectionWithValidCredentialsAndKeyStore() throws E final ArrowFlightJdbcDataSource dataSource = ArrowFlightJdbcDataSource.createNewDataSource(properties); try (final Connection connection = dataSource.getConnection()) { - assert connection.isValid(300); + Assert.assertTrue(connection.isValid(300)); } } @@ -276,7 +280,7 @@ public void testGetNonAuthenticatedEncryptedConnection() throws Exception { final ArrowFlightJdbcDataSource dataSource = ArrowFlightJdbcDataSource.createNewDataSource(properties); try (final Connection connection = dataSource.getConnection()) { - assert connection.isValid(300); + Assert.assertTrue(connection.isValid(300)); } } @@ -291,7 +295,7 @@ public void testTLSConnectionPropertyTrueCorrectCastUrlWithDriverManager() throw final Driver driver = new ArrowFlightJdbcDriver(); DriverManager.registerDriver(driver); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s" + "&useEncryption=true&useSystemTrustStore=false&%s=%s&%s=%s", @@ -301,9 +305,9 @@ public void testTLSConnectionPropertyTrueCorrectCastUrlWithDriverManager() throw ArrowFlightConnectionProperty.TRUST_STORE.camelName(), URLEncoder.encode(trustStorePath, "UTF-8"), ArrowFlightConnectionProperty.TRUST_STORE_PASSWORD.camelName(), - URLEncoder.encode(trustStorePass, "UTF-8"))); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + URLEncoder.encode(trustStorePass, "UTF-8")))) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -327,13 +331,13 @@ public void testTLSConnectionPropertyTrueCorrectCastUrlAndPropertiesUsingSetProp properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "true"); properties.setProperty(ArrowFlightConnectionProperty.USE_SYSTEM_TRUST_STORE.camelName(), "false"); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -357,13 +361,13 @@ public void testTLSConnectionPropertyTrueCorrectCastUrlAndPropertiesUsingPutWith properties.put(ArrowFlightConnectionProperty.TRUST_STORE.camelName(), trustStorePath); properties.put(ArrowFlightConnectionProperty.TRUST_STORE_PASSWORD.camelName(), trustStorePass); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -378,7 +382,7 @@ public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlWithDriverManager( final Driver driver = new ArrowFlightJdbcDriver(); DriverManager.registerDriver(driver); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format( "jdbc:arrow-flight-sql://localhost:%s?user=%s&password=%s" + "&useEncryption=1&useSystemTrustStore=0&%s=%s&%s=%s", @@ -388,9 +392,9 @@ public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlWithDriverManager( ArrowFlightConnectionProperty.TRUST_STORE.camelName(), URLEncoder.encode(trustStorePath, "UTF-8"), ArrowFlightConnectionProperty.TRUST_STORE_PASSWORD.camelName(), - URLEncoder.encode(trustStorePass, "UTF-8"))); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + URLEncoder.encode(trustStorePass, "UTF-8")))) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -414,11 +418,11 @@ public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlAndPropertiesUsing properties.setProperty(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), "1"); properties.setProperty(ArrowFlightConnectionProperty.USE_SYSTEM_TRUST_STORE.camelName(), "0"); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format("jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } /** @@ -442,11 +446,11 @@ public void testTLSConnectionPropertyTrueIntegerCorrectCastUrlAndPropertiesUsing properties.put(ArrowFlightConnectionProperty.TRUST_STORE.camelName(), trustStorePath); properties.put(ArrowFlightConnectionProperty.TRUST_STORE_PASSWORD.camelName(), trustStorePass); - final Connection connection = DriverManager.getConnection( + try (final Connection connection = DriverManager.getConnection( String.format("jdbc:arrow-flight-sql://localhost:%s", FLIGHT_SERVER_TEST_RULE.getPort()), - properties); - Assert.assertTrue(connection.isValid(0)); - connection.close(); + properties)) { + Assert.assertTrue(connection.isValid(0)); + } } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestRule.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestRule.java index df7cbea56ee2f..39eb0a29866f1 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestRule.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestRule.java @@ -55,6 +55,9 @@ * and interact with it. */ public class FlightServerTestRule implements TestRule, AutoCloseable { + public static final String DEFAULT_USER = "flight-test-user"; + public static final String DEFAULT_PASSWORD = "flight-test-password"; + private static final Logger LOGGER = LoggerFactory.getLogger(FlightServerTestRule.class); private final Properties properties; @@ -92,7 +95,7 @@ private FlightServerTestRule(final Properties properties, public static FlightServerTestRule createStandardTestRule(final FlightSqlProducer producer) { UserPasswordAuthentication authentication = new UserPasswordAuthentication.Builder() - .user("flight-test-user", "flight-test-password") + .user(DEFAULT_USER, DEFAULT_PASSWORD) .build(); return new Builder() diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java index 5b9e269fb3cc8..52910812fb4fb 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java @@ -237,16 +237,17 @@ public void testShouldCloseStatementWhenIsCloseOnCompletion() throws Exception { */ @Test public void testShouldCloseStatementWhenIsCloseOnCompletionWithMaxRowsLimit() throws Exception { - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.LEGACY_REGULAR_SQL_CMD); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.LEGACY_REGULAR_SQL_CMD)) { - final long maxRowsLimit = 3; - statement.setLargeMaxRows(maxRowsLimit); - statement.closeOnCompletion(); + final long maxRowsLimit = 3; + statement.setLargeMaxRows(maxRowsLimit); + statement.closeOnCompletion(); - resultSetNextUntilDone(resultSet); + resultSetNextUntilDone(resultSet); - collector.checkThat(statement.isClosed(), is(true)); + collector.checkThat(statement.isClosed(), is(true)); + } } /** diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java new file mode 100644 index 0000000000000..6565a85ddf99f --- /dev/null +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.driver.jdbc.client; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import org.apache.arrow.driver.jdbc.FlightServerTestRule; +import org.apache.arrow.driver.jdbc.utils.CoreMockedSqlProducers; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +/** + * Test the behavior of ArrowFlightSqlClientHandler.Builder + */ +public class ArrowFlightSqlClientHandlerBuilderTest { + @ClassRule + public static final FlightServerTestRule FLIGHT_SERVER_TEST_RULE = FlightServerTestRule + .createStandardTestRule(CoreMockedSqlProducers.getLegacyProducer()); + + private static BufferAllocator allocator; + + @BeforeClass + public static void setup() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @AfterClass + public static void tearDown() { + allocator.close(); + } + + @Test + public void testRetainCookiesOnAuthOff() throws Exception { + // Arrange + final ArrowFlightSqlClientHandler.Builder rootBuilder = new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withBufferAllocator(allocator) + .withUsername(FlightServerTestRule.DEFAULT_USER) + .withPassword(FlightServerTestRule.DEFAULT_PASSWORD) + .withEncryption(false) + .withRetainCookies(true) + .withRetainAuth(false); + + try (ArrowFlightSqlClientHandler rootHandler = rootBuilder.build()) { + // Act + final ArrowFlightSqlClientHandler.Builder testBuilder = new ArrowFlightSqlClientHandler.Builder(rootBuilder); + + // Assert + assertSame(rootBuilder.cookieFactory, testBuilder.cookieFactory); + assertNotSame(rootBuilder.authFactory, testBuilder.authFactory); + } + } + + @Test + public void testRetainCookiesOffAuthOff() throws Exception { + // Arrange + final ArrowFlightSqlClientHandler.Builder rootBuilder = new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withBufferAllocator(allocator) + .withUsername(FlightServerTestRule.DEFAULT_USER) + .withPassword(FlightServerTestRule.DEFAULT_PASSWORD) + .withEncryption(false) + .withRetainCookies(false) + .withRetainAuth(false); + + try (ArrowFlightSqlClientHandler rootHandler = rootBuilder.build()) { + // Act + final ArrowFlightSqlClientHandler.Builder testBuilder = new ArrowFlightSqlClientHandler.Builder(rootBuilder); + + // Assert + assertNotSame(rootBuilder.cookieFactory, testBuilder.cookieFactory); + assertNotSame(rootBuilder.authFactory, testBuilder.authFactory); + } + } + + @Test + public void testRetainCookiesOnAuthOn() throws Exception { + // Arrange + final ArrowFlightSqlClientHandler.Builder rootBuilder = new ArrowFlightSqlClientHandler.Builder() + .withHost(FLIGHT_SERVER_TEST_RULE.getHost()) + .withPort(FLIGHT_SERVER_TEST_RULE.getPort()) + .withBufferAllocator(allocator) + .withUsername(FlightServerTestRule.DEFAULT_USER) + .withPassword(FlightServerTestRule.DEFAULT_PASSWORD) + .withEncryption(false) + .withRetainCookies(true) + .withRetainAuth(true); + + try (ArrowFlightSqlClientHandler rootHandler = rootBuilder.build()) { + // Act + final ArrowFlightSqlClientHandler.Builder testBuilder = new ArrowFlightSqlClientHandler.Builder(rootBuilder); + + // Assert + assertSame(rootBuilder.cookieFactory, testBuilder.cookieFactory); + assertSame(rootBuilder.authFactory, testBuilder.authFactory); + } + } + + @Test + public void testDefaults() { + final ArrowFlightSqlClientHandler.Builder builder = new ArrowFlightSqlClientHandler.Builder(); + + // Validate all non-mandatory fields against defaults in ArrowFlightConnectionProperty. + assertNull(builder.username); + assertNull(builder.password); + assertTrue(builder.useEncryption); + assertFalse(builder.disableCertificateVerification); + assertNull(builder.trustStorePath); + assertNull(builder.trustStorePassword); + assertTrue(builder.useSystemTrustStore); + assertNull(builder.token); + assertTrue(builder.retainAuth); + assertTrue(builder.retainCookies); + assertNull(builder.tlsRootCertificatesPath); + assertNull(builder.clientCertificatePath); + assertNull(builder.clientKeyPath); + } +} From c4db009f8e3fa8698c58d31a7d3db9d6344da7d5 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Wed, 8 Nov 2023 04:59:04 +0900 Subject: [PATCH 18/57] MINOR: [Docs][C++] Fix a typo (#38625) ### Rationale for this change Noticed a typo on the web doc and proposing a fix ### Are these changes tested? No. But the PR is changing only text. ### Are there any user-facing changes? Yes. But only documentation typo Authored-by: Hyunseok Seo Signed-off-by: Sutou Kouhei --- docs/source/cpp/datatypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/cpp/datatypes.rst b/docs/source/cpp/datatypes.rst index 1d2133cbdf3d6..922fef1498b9c 100644 --- a/docs/source/cpp/datatypes.rst +++ b/docs/source/cpp/datatypes.rst @@ -157,7 +157,7 @@ Visitor Pattern --------------- In order to process :class:`arrow::DataType`, :class:`arrow::Scalar`, or -:class:`arrow::Array`, you may need to write write logic that specializes based +:class:`arrow::Array`, you may need to write logic that specializes based on the particular Arrow type. In these cases, use the `visitor pattern `_. Arrow provides the template functions: From bbb610e61a00bd80aff5cc7b2d10842bed34e0a4 Mon Sep 17 00:00:00 2001 From: Yue Date: Wed, 8 Nov 2023 08:56:03 +0800 Subject: [PATCH 19/57] GH-37753: [C++][Gandiva] Add external function registry support (#38116) # Rationale for this change This PR tries to enhance Gandiva by supporting external function registry, so that developers can author third party functions without modifying Gandiva's core codebase. See https://github.com/apache/arrow/issues/37753 for more details. In this PR, the external function needs to be compiled into LLVM IR for integration. # What changes are included in this PR? Two new APIs are added to `FunctionRegistry`: ```C++ /// \brief register a set of functions into the function registry from a given bitcode /// file arrow::Status Register(const std::vector& funcs, const std::string& bitcode_path); /// \brief register a set of functions into the function registry from a given bitcode /// buffer arrow::Status Register(const std::vector& funcs, std::shared_ptr bitcode_buffer); ``` Developers can use these two APIs to register external functions. Typically, developers will register a set of function metadatas (`funcs`) for all functions in a LLVM bitcode file, by giving either the path to the LLVM bitcode file or an `arrow::Buffer` containing the LLVM bitcode buffer. The overall flow looks like this: ![image](https://github.com/apache/arrow/assets/27754/b2b346fe-931f-4253-b198-4c388c57a56b) # Are these changes tested? Some unit tests are added to verify this enhancement # Are there any user-facing changes? Some new ways to interfacing the library are added in this PR: * The `Configuration` class now supports accepting a customized function registry, which developers can register their own external functions and uses it as the function registry * The `FunctionRegistry` class has two new APIs mentioned above * The `FunctionRegistry` class, after instantiation, now it doesn't have any built-in function registered in it. And we switch to use a new function `GANDIVA_EXPORT std::shared_ptr default_function_registry();` to retrieve the default function registry, which contains all the Gandiva built-in functions. * Some library depending on Gandiva C++ library, such as Gandiva's Ruby binding's `Gandiva::FunctionRegistry` class behavior is changed accordingly # Notes * Performance * the code generation time grows with the number of externally added function bitcodes (the more functions are added, the slower the codegen will be), even if the externally function is not used in the given expression at all. But this is not a new issue, and it applies to built-in functions as well (the more built-in functions are there, the slower the codegen will be). In my limited testing, this is because `llvm::Linker::linkModule` takes non trivial of time, which happens to every IR loaded, and the `RemoveUnusedFunctions` happens after that, which doesn't help to reduce the time of `linkModule`. We may have to selectively load only necessary IR (primarily selectively doing `linkModule` for these IR), but more metadata may be needed to tell which functions can be found in which IR. This could be a separated PR for improving it, please advice if any one has any idea on improving it. Thanks. * Integration with other programming languages via LLVM IR/bitcode * So far I only added an external C++ function in the codebase for unit testing purpose. Rust based function is possible but I gave it a try and found another issue (Rust has std lib which needs to be processed in different approach), I will do some exploration for other languages such as zig later. * Non pre-compiled functions, may require some different approach to get the function pointer, and we may discuss and work on it in a separated PR later. Another issue https://github.com/apache/arrow/issues/38589 was logged for this. * The discussion thread in dev mail list, https://lists.apache.org/thread/lm4sbw61w9cl7fsmo7tz3gvkq0ox6rod * I submitted another PR previously (https://github.com/apache/arrow/pull/37787) which introduced JSON based function registry, and after discussion, I will close that PR and use this PR instead * Closes: #37753 Lead-authored-by: Yue Ni Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- c_glib/arrow-glib/version.h.in | 23 ++++ c_glib/doc/gandiva-glib/gandiva-glib-docs.xml | 4 + c_glib/gandiva-glib/function-registry.cpp | 118 +++++++++++++++--- c_glib/gandiva-glib/function-registry.h | 2 + c_glib/gandiva-glib/function-registry.hpp | 30 +++++ c_glib/test/gandiva/test-function-registry.rb | 2 +- c_glib/test/gandiva/test-native-function.rb | 2 +- cpp/cmake_modules/GandivaAddBitcode.cmake | 75 +++++++++++ cpp/cmake_modules/ThirdpartyToolchain.cmake | 21 ++-- cpp/src/gandiva/CMakeLists.txt | 6 +- cpp/src/gandiva/GandivaConfig.cmake.in | 1 + cpp/src/gandiva/configuration.cc | 5 +- cpp/src/gandiva/configuration.h | 26 +++- cpp/src/gandiva/engine.cc | 69 +++++++--- cpp/src/gandiva/engine.h | 4 + cpp/src/gandiva/expr_decomposer_test.cc | 17 ++- cpp/src/gandiva/expr_validator.cc | 2 +- cpp/src/gandiva/expr_validator.h | 9 +- cpp/src/gandiva/expression_registry.cc | 6 +- cpp/src/gandiva/expression_registry.h | 6 +- cpp/src/gandiva/expression_registry_test.cc | 4 +- cpp/src/gandiva/filter.cc | 3 +- cpp/src/gandiva/function_registry.cc | 111 +++++++++++----- cpp/src/gandiva/function_registry.h | 34 ++++- cpp/src/gandiva/function_registry_test.cc | 47 +++++-- cpp/src/gandiva/llvm_generator.cc | 13 +- cpp/src/gandiva/llvm_generator.h | 8 +- cpp/src/gandiva/llvm_generator_test.cc | 19 ++- cpp/src/gandiva/native_function.h | 12 +- cpp/src/gandiva/precompiled/CMakeLists.txt | 67 ++-------- cpp/src/gandiva/projector.cc | 3 +- cpp/src/gandiva/tests/CMakeLists.txt | 47 ++++--- cpp/src/gandiva/tests/date_time_test.cc | 26 ++-- .../tests/external_functions/CMakeLists.txt | 50 ++++++++ .../external_functions/multiply_by_two.cc | 20 +++ .../external_functions/multiply_by_two.h | 24 ++++ cpp/src/gandiva/tests/filter_test.cc | 6 +- cpp/src/gandiva/tests/huge_table_test.cc | 5 +- .../tests/projector_build_validation_test.cc | 24 ++-- cpp/src/gandiva/tests/projector_test.cc | 26 ++++ cpp/src/gandiva/tests/test_util.cc | 52 ++++++++ cpp/src/gandiva/tests/test_util.h | 11 +- cpp/src/gandiva/tree_expr_test.cc | 16 +-- 43 files changed, 809 insertions(+), 247 deletions(-) create mode 100644 c_glib/gandiva-glib/function-registry.hpp create mode 100644 cpp/cmake_modules/GandivaAddBitcode.cmake create mode 100644 cpp/src/gandiva/tests/external_functions/CMakeLists.txt create mode 100644 cpp/src/gandiva/tests/external_functions/multiply_by_two.cc create mode 100644 cpp/src/gandiva/tests/external_functions/multiply_by_two.h create mode 100644 cpp/src/gandiva/tests/test_util.cc diff --git a/c_glib/arrow-glib/version.h.in b/c_glib/arrow-glib/version.h.in index 60c02936193bc..abb8ba08708de 100644 --- a/c_glib/arrow-glib/version.h.in +++ b/c_glib/arrow-glib/version.h.in @@ -110,6 +110,15 @@ # define GARROW_UNAVAILABLE(major, minor) G_UNAVAILABLE(major, minor) #endif +/** + * GARROW_VERSION_15_0: + * + * You can use this macro value for compile time API version check. + * + * Since: 15.0.0 + */ +#define GARROW_VERSION_15_0 G_ENCODE_VERSION(15, 0) + /** * GARROW_VERSION_14_0: * @@ -346,6 +355,20 @@ #define GARROW_AVAILABLE_IN_ALL +#if GARROW_VERSION_MIN_REQUIRED >= GARROW_VERSION_15_0 +# define GARROW_DEPRECATED_IN_15_0 GARROW_DEPRECATED +# define GARROW_DEPRECATED_IN_15_0_FOR(function) GARROW_DEPRECATED_FOR(function) +#else +# define GARROW_DEPRECATED_IN_15_0 +# define GARROW_DEPRECATED_IN_15_0_FOR(function) +#endif + +#if GARROW_VERSION_MAX_ALLOWED < GARROW_VERSION_15_0 +# define GARROW_AVAILABLE_IN_15_0 GARROW_UNAVAILABLE(15, 0) +#else +# define GARROW_AVAILABLE_IN_15_0 +#endif + #if GARROW_VERSION_MIN_REQUIRED >= GARROW_VERSION_14_0 # define GARROW_DEPRECATED_IN_14_0 GARROW_DEPRECATED # define GARROW_DEPRECATED_IN_14_0_FOR(function) GARROW_DEPRECATED_FOR(function) diff --git a/c_glib/doc/gandiva-glib/gandiva-glib-docs.xml b/c_glib/doc/gandiva-glib/gandiva-glib-docs.xml index 182bbfb527eb2..a5c32f11337e8 100644 --- a/c_glib/doc/gandiva-glib/gandiva-glib-docs.xml +++ b/c_glib/doc/gandiva-glib/gandiva-glib-docs.xml @@ -100,6 +100,10 @@ Index of deprecated API + + Index of new symbols in 15.0.0 + + Index of new symbols in 4.0.0 diff --git a/c_glib/gandiva-glib/function-registry.cpp b/c_glib/gandiva-glib/function-registry.cpp index a95019bd62c2b..f47262986db82 100644 --- a/c_glib/gandiva-glib/function-registry.cpp +++ b/c_glib/gandiva-glib/function-registry.cpp @@ -18,8 +18,8 @@ */ #include -#include +#include #include #include @@ -34,18 +34,86 @@ G_BEGIN_DECLS * Since: 0.14.0 */ -G_DEFINE_TYPE(GGandivaFunctionRegistry, - ggandiva_function_registry, - G_TYPE_OBJECT) +struct GGandivaFunctionRegistryPrivate { + std::shared_ptr function_registry; +}; + +enum { + PROP_FUNCTION_REGISTRY = 1, +}; + +G_DEFINE_TYPE_WITH_PRIVATE(GGandivaFunctionRegistry, + ggandiva_function_registry, + G_TYPE_OBJECT) + +#define GGANDIVA_FUNCTION_REGISTRY_GET_PRIVATE(object) \ + static_cast( \ + ggandiva_function_registry_get_instance_private( \ + GGANDIVA_FUNCTION_REGISTRY(object))) + +static void +ggandiva_function_registry_finalize(GObject *object) +{ + auto priv = GGANDIVA_FUNCTION_REGISTRY_GET_PRIVATE(object); + priv->function_registry.~shared_ptr(); + G_OBJECT_CLASS(ggandiva_function_registry_parent_class)->finalize(object); +} + +static void +ggandiva_function_registry_set_property(GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + auto priv = GGANDIVA_FUNCTION_REGISTRY_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_FUNCTION_REGISTRY: + priv->function_registry = + *static_cast *>( + g_value_get_pointer(value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} static void ggandiva_function_registry_init(GGandivaFunctionRegistry *object) { + auto priv = GGANDIVA_FUNCTION_REGISTRY_GET_PRIVATE(object); + new(&priv->function_registry) std::shared_ptr; } static void ggandiva_function_registry_class_init(GGandivaFunctionRegistryClass *klass) { + auto gobject_class = G_OBJECT_CLASS(klass); + gobject_class->finalize = ggandiva_function_registry_finalize; + gobject_class->set_property = ggandiva_function_registry_set_property; + + GParamSpec *spec; + spec = g_param_spec_pointer("function-registry", + "Function registry", + "The raw std::shared_ptr *", + static_cast(G_PARAM_WRITABLE | + G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_FUNCTION_REGISTRY, spec); +} + +/** + * ggandiva_function_registry_default: + * + * Returns: (transfer full): The process-wide default function registry. + * + * Since: 15.0.0 + */ +GGandivaFunctionRegistry * +ggandiva_function_registry_default(void) +{ + auto gandiva_function_registry = gandiva::default_function_registry(); + return ggandiva_function_registry_new_raw(&gandiva_function_registry); } /** @@ -58,7 +126,8 @@ ggandiva_function_registry_class_init(GGandivaFunctionRegistryClass *klass) GGandivaFunctionRegistry * ggandiva_function_registry_new(void) { - return GGANDIVA_FUNCTION_REGISTRY(g_object_new(GGANDIVA_TYPE_FUNCTION_REGISTRY, NULL)); + auto gandiva_function_registry = std::make_shared(); + return ggandiva_function_registry_new_raw(&gandiva_function_registry); } /** @@ -75,15 +144,16 @@ GGandivaNativeFunction * ggandiva_function_registry_lookup(GGandivaFunctionRegistry *function_registry, GGandivaFunctionSignature *function_signature) { - gandiva::FunctionRegistry gandiva_function_registry; + auto gandiva_function_registry = + ggandiva_function_registry_get_raw(function_registry); auto gandiva_function_signature = ggandiva_function_signature_get_raw(function_signature); auto gandiva_native_function = - gandiva_function_registry.LookupSignature(*gandiva_function_signature); + gandiva_function_registry->LookupSignature(*gandiva_function_signature); if (gandiva_native_function) { return ggandiva_native_function_new_raw(gandiva_native_function); } else { - return NULL; + return nullptr; } } @@ -99,18 +169,32 @@ ggandiva_function_registry_lookup(GGandivaFunctionRegistry *function_registry, GList * ggandiva_function_registry_get_native_functions(GGandivaFunctionRegistry *function_registry) { - gandiva::FunctionRegistry gandiva_function_registry; - + auto gandiva_function_registry = + ggandiva_function_registry_get_raw(function_registry); GList *native_functions = nullptr; - for (auto gandiva_native_function = gandiva_function_registry.begin(); - gandiva_native_function != gandiva_function_registry.end(); - ++gandiva_native_function) { - auto native_function = ggandiva_native_function_new_raw(gandiva_native_function); + for (const auto &gandiva_native_function : *gandiva_function_registry) { + auto native_function = ggandiva_native_function_new_raw(&gandiva_native_function); native_functions = g_list_prepend(native_functions, native_function); } - native_functions = g_list_reverse(native_functions); - - return native_functions; + return g_list_reverse(native_functions); } G_END_DECLS + +GGandivaFunctionRegistry * +ggandiva_function_registry_new_raw( + std::shared_ptr *gandiva_function_registry) +{ + return GGANDIVA_FUNCTION_REGISTRY( + g_object_new(GGANDIVA_TYPE_FUNCTION_REGISTRY, + "function-registry", gandiva_function_registry, + nullptr)); +} + +std::shared_ptr +ggandiva_function_registry_get_raw(GGandivaFunctionRegistry *function_registry) +{ + auto priv = GGANDIVA_FUNCTION_REGISTRY_GET_PRIVATE(function_registry); + return priv->function_registry; +} + diff --git a/c_glib/gandiva-glib/function-registry.h b/c_glib/gandiva-glib/function-registry.h index 1a0d767d45354..8ff6027cf1734 100644 --- a/c_glib/gandiva-glib/function-registry.h +++ b/c_glib/gandiva-glib/function-registry.h @@ -35,6 +35,8 @@ struct _GGandivaFunctionRegistryClass GObjectClass parent_class; }; +GARROW_AVAILABLE_IN_15_0 +GGandivaFunctionRegistry *ggandiva_function_registry_default(void); GGandivaFunctionRegistry *ggandiva_function_registry_new(void); GGandivaNativeFunction * ggandiva_function_registry_lookup(GGandivaFunctionRegistry *function_registry, diff --git a/c_glib/gandiva-glib/function-registry.hpp b/c_glib/gandiva-glib/function-registry.hpp new file mode 100644 index 0000000000000..0430fc57dead2 --- /dev/null +++ b/c_glib/gandiva-glib/function-registry.hpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include + +GGandivaFunctionRegistry * +ggandiva_function_registry_new_raw( + std::shared_ptr *gandiva_function_registry); +std::shared_ptr +ggandiva_function_registry_get_raw(GGandivaFunctionRegistry *function_registry); diff --git a/c_glib/test/gandiva/test-function-registry.rb b/c_glib/test/gandiva/test-function-registry.rb index 25bac6673105e..d0f959a1c5f5f 100644 --- a/c_glib/test/gandiva/test-function-registry.rb +++ b/c_glib/test/gandiva/test-function-registry.rb @@ -20,7 +20,7 @@ class TestGandivaFunctionRegistry < Test::Unit::TestCase def setup omit("Gandiva is required") unless defined?(::Gandiva) - @registry = Gandiva::FunctionRegistry.new + @registry = Gandiva::FunctionRegistry.default end sub_test_case("lookup") do diff --git a/c_glib/test/gandiva/test-native-function.rb b/c_glib/test/gandiva/test-native-function.rb index 7888f96b678b7..630a1f7c32d2a 100644 --- a/c_glib/test/gandiva/test-native-function.rb +++ b/c_glib/test/gandiva/test-native-function.rb @@ -20,7 +20,7 @@ class TestGandivaNativeFunction < Test::Unit::TestCase def setup omit("Gandiva is required") unless defined?(::Gandiva) - @registry = Gandiva::FunctionRegistry.new + @registry = Gandiva::FunctionRegistry.default @not = lookup("not", [boolean_data_type], boolean_data_type) @isnull = lookup("isnull", [int8_data_type], boolean_data_type) end diff --git a/cpp/cmake_modules/GandivaAddBitcode.cmake b/cpp/cmake_modules/GandivaAddBitcode.cmake new file mode 100644 index 0000000000000..98847f8a186fe --- /dev/null +++ b/cpp/cmake_modules/GandivaAddBitcode.cmake @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Create bitcode for the given source file. +function(gandiva_add_bitcode SOURCE) + set(CLANG_OPTIONS -std=c++17) + if(MSVC) + # "19.20" means that it's compatible with Visual Studio 16 2019. + # We can update this to "19.30" when we dropped support for Visual + # Studio 16 2019. + # + # See https://cmake.org/cmake/help/latest/variable/MSVC_VERSION.html + # for MSVC_VERSION and Visual Studio version. + set(FMS_COMPATIBILITY 19.20) + list(APPEND CLANG_OPTIONS -fms-compatibility + -fms-compatibility-version=${FMS_COMPATIBILITY}) + endif() + + get_filename_component(SOURCE_BASE ${SOURCE} NAME_WE) + get_filename_component(ABSOLUTE_SOURCE ${SOURCE} ABSOLUTE) + set(BC_FILE ${CMAKE_CURRENT_BINARY_DIR}/${SOURCE_BASE}.bc) + set(PRECOMPILE_COMMAND) + if(CMAKE_OSX_SYSROOT) + list(APPEND + PRECOMPILE_COMMAND + ${CMAKE_COMMAND} + -E + env + SDKROOT=${CMAKE_OSX_SYSROOT}) + endif() + list(APPEND + PRECOMPILE_COMMAND + ${CLANG_EXECUTABLE} + ${CLANG_OPTIONS} + -DGANDIVA_IR + -DNDEBUG # DCHECK macros not implemented in precompiled code + -DARROW_STATIC # Do not set __declspec(dllimport) on MSVC on Arrow symbols + -DGANDIVA_STATIC # Do not set __declspec(dllimport) on MSVC on Gandiva symbols + -fno-use-cxa-atexit # Workaround for unresolved __dso_handle + -emit-llvm + -O3 + -c + ${ABSOLUTE_SOURCE} + -o + ${BC_FILE} + ${ARROW_GANDIVA_PC_CXX_FLAGS}) + if(ARROW_BINARY_DIR) + list(APPEND PRECOMPILE_COMMAND -I${ARROW_BINARY_DIR}/src) + endif() + if(ARROW_SOURCE_DIR) + list(APPEND PRECOMPILE_COMMAND -I${ARROW_SOURCE_DIR}/src) + endif() + if(NOT ARROW_USE_NATIVE_INT128) + foreach(boost_include_dir ${Boost_INCLUDE_DIRS}) + list(APPEND PRECOMPILE_COMMAND -I${boost_include_dir}) + endforeach() + endif() + add_custom_command(OUTPUT ${BC_FILE} + COMMAND ${PRECOMPILE_COMMAND} + DEPENDS ${SOURCE_FILE}) +endfunction() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 5de8ff9b1cb11..52632d554aafb 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -230,18 +230,21 @@ macro(build_dependency DEPENDENCY_NAME) endif() endmacro() -# Find modules are needed by the consumer in case of a static build, or if the -# linkage is PUBLIC or INTERFACE. -macro(provide_find_module PACKAGE_NAME ARROW_CMAKE_PACKAGE_NAME) - set(module_ "${CMAKE_SOURCE_DIR}/cmake_modules/Find${PACKAGE_NAME}.cmake") - if(EXISTS "${module_}") - message(STATUS "Providing CMake module for ${PACKAGE_NAME} as part of ${ARROW_CMAKE_PACKAGE_NAME} CMake package" +function(provide_cmake_module MODULE_NAME ARROW_CMAKE_PACKAGE_NAME) + set(module "${CMAKE_SOURCE_DIR}/cmake_modules/${MODULE_NAME}.cmake") + if(EXISTS "${module}") + message(STATUS "Providing CMake module for ${MODULE_NAME} as part of ${ARROW_CMAKE_PACKAGE_NAME} CMake package" ) - install(FILES "${module_}" + install(FILES "${module}" DESTINATION "${ARROW_CMAKE_DIR}/${ARROW_CMAKE_PACKAGE_NAME}") endif() - unset(module_) -endmacro() +endfunction() + +# Find modules are needed by the consumer in case of a static build, or if the +# linkage is PUBLIC or INTERFACE. +function(provide_find_module PACKAGE_NAME ARROW_CMAKE_PACKAGE_NAME) + provide_cmake_module("Find${PACKAGE_NAME}" ${ARROW_CMAKE_PACKAGE_NAME}) +endfunction() macro(resolve_dependency DEPENDENCY_NAME) set(options) diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 6b6743bc8e52f..3448d516768bb 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -25,11 +25,14 @@ add_custom_target(gandiva-benchmarks) add_dependencies(gandiva-all gandiva gandiva-tests gandiva-benchmarks) +include(GandivaAddBitcode) + find_package(LLVMAlt REQUIRED) provide_find_module(LLVMAlt "Gandiva") if(ARROW_WITH_ZSTD AND "${zstd_SOURCE}" STREQUAL "SYSTEM") provide_find_module(zstdAlt "Gandiva") endif() +provide_cmake_module(GandivaAddBitcode "Gandiva") # Set the path where the bitcode file generated, see precompiled/CMakeLists.txt set(GANDIVA_PRECOMPILED_BC_PATH "${CMAKE_CURRENT_BINARY_DIR}/irhelpers.bc") @@ -249,7 +252,8 @@ add_gandiva_test(internals-test random_generator_holder_test.cc hash_utils_test.cc gdv_function_stubs_test.cc - interval_holder_test.cc) + interval_holder_test.cc + tests/test_util.cc) add_subdirectory(precompiled) add_subdirectory(tests) diff --git a/cpp/src/gandiva/GandivaConfig.cmake.in b/cpp/src/gandiva/GandivaConfig.cmake.in index f02e29f25bb3a..68579debd183b 100644 --- a/cpp/src/gandiva/GandivaConfig.cmake.in +++ b/cpp/src/gandiva/GandivaConfig.cmake.in @@ -49,6 +49,7 @@ else() endif() include("${CMAKE_CURRENT_LIST_DIR}/GandivaTargets.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/GandivaAddBitcode.cmake") arrow_keep_backward_compatibility(Gandiva gandiva) diff --git a/cpp/src/gandiva/configuration.cc b/cpp/src/gandiva/configuration.cc index 1e26c5c70d4ec..b79f4118e07f2 100644 --- a/cpp/src/gandiva/configuration.cc +++ b/cpp/src/gandiva/configuration.cc @@ -29,11 +29,14 @@ std::size_t Configuration::Hash() const { size_t result = kHashSeed; arrow::internal::hash_combine(result, static_cast(optimize_)); arrow::internal::hash_combine(result, static_cast(target_host_cpu_)); + arrow::internal::hash_combine( + result, reinterpret_cast(function_registry_.get())); return result; } bool Configuration::operator==(const Configuration& other) const { - return optimize_ == other.optimize_ && target_host_cpu_ == other.target_host_cpu_; + return optimize_ == other.optimize_ && target_host_cpu_ == other.target_host_cpu_ && + function_registry_ == other.function_registry_; } bool Configuration::operator!=(const Configuration& other) const { diff --git a/cpp/src/gandiva/configuration.h b/cpp/src/gandiva/configuration.h index 9cd301524d03d..f43a2b190731f 100644 --- a/cpp/src/gandiva/configuration.h +++ b/cpp/src/gandiva/configuration.h @@ -21,6 +21,7 @@ #include #include "arrow/status.h" +#include "gandiva/function_registry.h" #include "gandiva/visibility.h" namespace gandiva { @@ -34,8 +35,14 @@ class GANDIVA_EXPORT Configuration { public: friend class ConfigurationBuilder; - Configuration() : optimize_(true), target_host_cpu_(true) {} - explicit Configuration(bool optimize) : optimize_(optimize), target_host_cpu_(true) {} + explicit Configuration(bool optimize, + std::shared_ptr function_registry = + gandiva::default_function_registry()) + : optimize_(optimize), + target_host_cpu_(true), + function_registry_(function_registry) {} + + Configuration() : Configuration(true) {} std::size_t Hash() const; bool operator==(const Configuration& other) const; @@ -43,13 +50,21 @@ class GANDIVA_EXPORT Configuration { bool optimize() const { return optimize_; } bool target_host_cpu() const { return target_host_cpu_; } + std::shared_ptr function_registry() const { + return function_registry_; + } void set_optimize(bool optimize) { optimize_ = optimize; } void target_host_cpu(bool target_host_cpu) { target_host_cpu_ = target_host_cpu; } + void set_function_registry(std::shared_ptr function_registry) { + function_registry_ = std::move(function_registry); + } private: bool optimize_; /* optimise the generated llvm IR */ bool target_host_cpu_; /* set the mcpu flag to host cpu while compiling llvm ir */ + std::shared_ptr + function_registry_; /* function registry that may contain external functions */ }; /// \brief configuration builder for gandiva @@ -68,6 +83,13 @@ class GANDIVA_EXPORT ConfigurationBuilder { return configuration; } + std::shared_ptr build( + std::shared_ptr function_registry) { + std::shared_ptr configuration( + new Configuration(true, std::move(function_registry))); + return configuration; + } + static std::shared_ptr DefaultConfiguration() { return default_configuration_; } diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index 8ebe927437567..5ae1d76876148 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -141,7 +141,8 @@ Engine::Engine(const std::shared_ptr& conf, module_(module), types_(*context_), optimize_(conf->optimize()), - cached_(cached) {} + cached_(cached), + function_registry_(conf->function_registry()) {} Status Engine::Init() { std::call_once(register_exported_funcs_flag, gandiva::RegisterExportedFuncs); @@ -155,6 +156,7 @@ Status Engine::LoadFunctionIRs() { if (!functions_loaded_) { ARROW_RETURN_NOT_OK(LoadPreCompiledIR()); ARROW_RETURN_NOT_OK(DecimalIR::AddFunctions(this)); + ARROW_RETURN_NOT_OK(LoadExternalPreCompiledIR()); functions_loaded_ = true; } return Status::OK(); @@ -236,7 +238,38 @@ static void SetDataLayout(llvm::Module* module) { module->setDataLayout(machine->createDataLayout()); } -// end of the mofified method from MLIR +// end of the modified method from MLIR + +template +static arrow::Result AsArrowResult(llvm::Expected& expected) { + if (!expected) { + std::string str; + llvm::raw_string_ostream stream(str); + stream << expected.takeError(); + return Status::CodeGenError(stream.str()); + } + return std::move(expected.get()); +} + +static arrow::Status VerifyAndLinkModule( + llvm::Module* dest_module, + llvm::Expected> src_module_or_error) { + ARROW_ASSIGN_OR_RAISE(auto src_ir_module, AsArrowResult(src_module_or_error)); + + // set dataLayout + SetDataLayout(src_ir_module.get()); + + std::string error_info; + llvm::raw_string_ostream error_stream(error_info); + ARROW_RETURN_IF( + llvm::verifyModule(*src_ir_module, &error_stream), + Status::CodeGenError("verify of IR Module failed: " + error_stream.str())); + + ARROW_RETURN_IF(llvm::Linker::linkModules(*dest_module, std::move(src_ir_module)), + Status::CodeGenError("failed to link IR Modules")); + + return Status::OK(); +} // Handling for pre-compiled IR libraries. Status Engine::LoadPreCompiledIR() { @@ -256,23 +289,25 @@ Status Engine::LoadPreCompiledIR() { /// Parse the IR module. llvm::Expected> module_or_error = llvm::getOwningLazyBitcodeModule(std::move(buffer), *context()); - if (!module_or_error) { - // NOTE: llvm::handleAllErrors() fails linking with RTTI-disabled LLVM builds - // (ARROW-5148) - std::string str; - llvm::raw_string_ostream stream(str); - stream << module_or_error.takeError(); - return Status::CodeGenError(stream.str()); - } - std::unique_ptr ir_module = std::move(module_or_error.get()); + // NOTE: llvm::handleAllErrors() fails linking with RTTI-disabled LLVM builds + // (ARROW-5148) + ARROW_RETURN_NOT_OK(VerifyAndLinkModule(module_, std::move(module_or_error))); + return Status::OK(); +} - // set dataLayout - SetDataLayout(ir_module.get()); +static llvm::MemoryBufferRef AsLLVMMemoryBuffer(const arrow::Buffer& arrow_buffer) { + auto data = reinterpret_cast(arrow_buffer.data()); + auto size = arrow_buffer.size(); + return llvm::MemoryBufferRef(llvm::StringRef(data, size), "external_bitcode"); +} - ARROW_RETURN_IF(llvm::verifyModule(*ir_module, &llvm::errs()), - Status::CodeGenError("verify of IR Module failed")); - ARROW_RETURN_IF(llvm::Linker::linkModules(*module_, std::move(ir_module)), - Status::CodeGenError("failed to link IR Modules")); +Status Engine::LoadExternalPreCompiledIR() { + auto const& buffers = function_registry_->GetBitcodeBuffers(); + for (auto const& buffer : buffers) { + auto llvm_memory_buffer_ref = AsLLVMMemoryBuffer(*buffer); + auto module_or_error = llvm::parseBitcodeFile(llvm_memory_buffer_ref, *context()); + ARROW_RETURN_NOT_OK(VerifyAndLinkModule(module_, std::move(module_or_error))); + } return Status::OK(); } diff --git a/cpp/src/gandiva/engine.h b/cpp/src/gandiva/engine.h index a4d6a5fd1a758..566977dc4adad 100644 --- a/cpp/src/gandiva/engine.h +++ b/cpp/src/gandiva/engine.h @@ -93,6 +93,9 @@ class GANDIVA_EXPORT Engine { /// the main module. Status LoadPreCompiledIR(); + // load external pre-compiled bitcodes into module + Status LoadExternalPreCompiledIR(); + // Create and add mappings for cpp functions that can be accessed from LLVM. void AddGlobalMappings(); @@ -111,6 +114,7 @@ class GANDIVA_EXPORT Engine { bool module_finalized_ = false; bool cached_; bool functions_loaded_ = false; + std::shared_ptr function_registry_; }; } // namespace gandiva diff --git a/cpp/src/gandiva/expr_decomposer_test.cc b/cpp/src/gandiva/expr_decomposer_test.cc index 638ceebcb19fd..7681d9e646297 100644 --- a/cpp/src/gandiva/expr_decomposer_test.cc +++ b/cpp/src/gandiva/expr_decomposer_test.cc @@ -24,7 +24,6 @@ #include "gandiva/function_registry.h" #include "gandiva/gandiva_aliases.h" #include "gandiva/node.h" -#include "gandiva/tree_expr_builder.h" namespace gandiva { @@ -32,12 +31,12 @@ using arrow::int32; class TestExprDecomposer : public ::testing::Test { protected: - FunctionRegistry registry_; + std::shared_ptr registry_ = default_function_registry(); }; TEST_F(TestExprDecomposer, TestStackSimple) { Annotator annotator; - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); // if (a) _ // else _ @@ -58,7 +57,7 @@ TEST_F(TestExprDecomposer, TestStackSimple) { TEST_F(TestExprDecomposer, TestNested) { Annotator annotator; - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); // if (a) _ // else _ @@ -97,7 +96,7 @@ TEST_F(TestExprDecomposer, TestNested) { TEST_F(TestExprDecomposer, TestInternalIf) { Annotator annotator; - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); // if (a) _ // if (b) _ @@ -136,7 +135,7 @@ TEST_F(TestExprDecomposer, TestInternalIf) { TEST_F(TestExprDecomposer, TestParallelIf) { Annotator annotator; - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); // if (a) _ // else _ @@ -174,7 +173,7 @@ TEST_F(TestExprDecomposer, TestParallelIf) { TEST_F(TestExprDecomposer, TestIfInCondition) { Annotator annotator; - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); // if (if _ else _) : a // - @@ -245,7 +244,7 @@ TEST_F(TestExprDecomposer, TestIfInCondition) { TEST_F(TestExprDecomposer, TestFunctionBetweenNestedIf) { Annotator annotator; - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); // if (a) _ // else @@ -286,7 +285,7 @@ TEST_F(TestExprDecomposer, TestFunctionBetweenNestedIf) { TEST_F(TestExprDecomposer, TestComplexIfCondition) { Annotator annotator; - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); // if (if _ // else diff --git a/cpp/src/gandiva/expr_validator.cc b/cpp/src/gandiva/expr_validator.cc index 35a13494523d0..8a6f86e6f0419 100644 --- a/cpp/src/gandiva/expr_validator.cc +++ b/cpp/src/gandiva/expr_validator.cc @@ -93,7 +93,7 @@ Status ExprValidator::Visit(const FunctionNode& node) { const auto& desc = node.descriptor(); FunctionSignature signature(desc->name(), desc->params(), desc->return_type()); - const NativeFunction* native_function = registry_.LookupSignature(signature); + const NativeFunction* native_function = registry_->LookupSignature(signature); ARROW_RETURN_IF(native_function == nullptr, Status::ExpressionValidationError("Function ", signature.ToString(), " not supported yet. ")); diff --git a/cpp/src/gandiva/expr_validator.h b/cpp/src/gandiva/expr_validator.h index 7f6d7fd131fbe..8a423fc93b02b 100644 --- a/cpp/src/gandiva/expr_validator.h +++ b/cpp/src/gandiva/expr_validator.h @@ -37,8 +37,9 @@ class FunctionRegistry; /// data types, signatures and return types class ExprValidator : public NodeVisitor { public: - explicit ExprValidator(LLVMTypes* types, SchemaPtr schema) - : types_(types), schema_(schema) { + explicit ExprValidator(LLVMTypes* types, SchemaPtr schema, + std::shared_ptr registry) + : types_(types), schema_(schema), registry_(std::move(registry)) { for (auto& field : schema_->fields()) { field_map_[field->name()] = field; } @@ -65,12 +66,12 @@ class ExprValidator : public NodeVisitor { Status Visit(const InExpressionNode& node) override; Status Visit(const InExpressionNode& node) override; - FunctionRegistry registry_; - LLVMTypes* types_; SchemaPtr schema_; + std::shared_ptr registry_; + using FieldMap = std::unordered_map; FieldMap field_map_; }; diff --git a/cpp/src/gandiva/expression_registry.cc b/cpp/src/gandiva/expression_registry.cc index 9bff97f5ad269..dd964a7cb8a7a 100644 --- a/cpp/src/gandiva/expression_registry.cc +++ b/cpp/src/gandiva/expression_registry.cc @@ -22,9 +22,9 @@ namespace gandiva { -ExpressionRegistry::ExpressionRegistry() { - function_registry_.reset(new FunctionRegistry()); -} +ExpressionRegistry::ExpressionRegistry( + std::shared_ptr function_registry) + : function_registry_{function_registry} {} ExpressionRegistry::~ExpressionRegistry() {} diff --git a/cpp/src/gandiva/expression_registry.h b/cpp/src/gandiva/expression_registry.h index 609a2dbbe21f9..156a6392564f9 100644 --- a/cpp/src/gandiva/expression_registry.h +++ b/cpp/src/gandiva/expression_registry.h @@ -21,6 +21,7 @@ #include #include "gandiva/arrow.h" +#include "gandiva/function_registry.h" #include "gandiva/function_signature.h" #include "gandiva/gandiva_aliases.h" #include "gandiva/visibility.h" @@ -37,7 +38,8 @@ class GANDIVA_EXPORT ExpressionRegistry { public: using native_func_iterator_type = const NativeFunction*; using func_sig_iterator_type = const FunctionSignature*; - ExpressionRegistry(); + explicit ExpressionRegistry(std::shared_ptr function_registry = + gandiva::default_function_registry()); ~ExpressionRegistry(); static DataTypeVector supported_types() { return supported_types_; } class GANDIVA_EXPORT FunctionSignatureIterator { @@ -62,7 +64,7 @@ class GANDIVA_EXPORT ExpressionRegistry { private: static DataTypeVector supported_types_; - std::unique_ptr function_registry_; + std::shared_ptr function_registry_; }; /// \brief Get the list of all function signatures. diff --git a/cpp/src/gandiva/expression_registry_test.cc b/cpp/src/gandiva/expression_registry_test.cc index c254ff4f3aa5e..cd784192c194e 100644 --- a/cpp/src/gandiva/expression_registry_test.cc +++ b/cpp/src/gandiva/expression_registry_test.cc @@ -31,7 +31,7 @@ typedef int64_t (*add_vector_func_t)(int64_t* elements, int nelements); class TestExpressionRegistry : public ::testing::Test { protected: - FunctionRegistry registry_; + std::shared_ptr registry_ = default_function_registry(); }; // Verify all functions in registry are exported. @@ -42,7 +42,7 @@ TEST_F(TestExpressionRegistry, VerifySupportedFunctions) { iter != expr_registry.function_signature_end(); iter++) { functions.push_back((*iter)); } - for (auto& iter : registry_) { + for (auto& iter : *registry_) { for (auto& func_iter : iter.signatures()) { auto element = std::find(functions.begin(), functions.end(), func_iter); EXPECT_NE(element, functions.end()) << "function signature " << func_iter.ToString() diff --git a/cpp/src/gandiva/filter.cc b/cpp/src/gandiva/filter.cc index 78917467a0f56..416d97b5dbd1d 100644 --- a/cpp/src/gandiva/filter.cc +++ b/cpp/src/gandiva/filter.cc @@ -71,7 +71,8 @@ Status Filter::Make(SchemaPtr schema, ConditionPtr condition, if (!is_cached) { // Run the validation on the expression. // Return if the expression is invalid since we will not be able to process further. - ExprValidator expr_validator(llvm_gen->types(), schema); + ExprValidator expr_validator(llvm_gen->types(), schema, + configuration->function_registry()); ARROW_RETURN_NOT_OK(expr_validator.Validate(condition)); } diff --git a/cpp/src/gandiva/function_registry.cc b/cpp/src/gandiva/function_registry.cc index 67b7b404b325c..5d676dfa8df74 100644 --- a/cpp/src/gandiva/function_registry.cc +++ b/cpp/src/gandiva/function_registry.cc @@ -16,6 +16,13 @@ // under the License. #include "gandiva/function_registry.h" + +#include +#include + +#include + +#include "arrow/util/logging.h" #include "gandiva/function_registry_arithmetic.h" #include "gandiva/function_registry_datetime.h" #include "gandiva/function_registry_hash.h" @@ -23,12 +30,26 @@ #include "gandiva/function_registry_string.h" #include "gandiva/function_registry_timestamp_arithmetic.h" -#include -#include -#include - namespace gandiva { +static constexpr uint32_t kMaxFunctionSignatures = 2048; + +// encapsulates an llvm memory buffer in an arrow buffer +// this is needed because we don't expose the llvm memory buffer to the outside world in +// the header file +class LLVMMemoryArrowBuffer : public arrow::Buffer { + public: + explicit LLVMMemoryArrowBuffer(std::unique_ptr llvm_buffer) + : arrow::Buffer(reinterpret_cast(llvm_buffer->getBufferStart()), + static_cast(llvm_buffer->getBufferSize())), + llvm_buffer_(std::move(llvm_buffer)) {} + + private: + std::unique_ptr llvm_buffer_; +}; + +FunctionRegistry::FunctionRegistry() { pc_registry_.reserve(kMaxFunctionSignatures); } + FunctionRegistry::iterator FunctionRegistry::begin() const { return &(*pc_registry_.begin()); } @@ -41,42 +62,74 @@ FunctionRegistry::iterator FunctionRegistry::back() const { return &(pc_registry_.back()); } -std::vector FunctionRegistry::pc_registry_; +const NativeFunction* FunctionRegistry::LookupSignature( + const FunctionSignature& signature) const { + auto got = pc_registry_map_.find(&signature); + return got == pc_registry_map_.end() ? nullptr : got->second; +} -SignatureMap FunctionRegistry::pc_registry_map_ = InitPCMap(); +Status FunctionRegistry::Add(NativeFunction func) { + if (pc_registry_.size() == kMaxFunctionSignatures) { + return Status::CapacityError("Exceeded max function signatures limit of ", + kMaxFunctionSignatures); + } + pc_registry_.emplace_back(std::move(func)); + auto const& last_func = pc_registry_.back(); + for (auto const& func_signature : last_func.signatures()) { + pc_registry_map_.emplace(&func_signature, &last_func); + } + return arrow::Status::OK(); +} -SignatureMap FunctionRegistry::InitPCMap() { - SignatureMap map; +arrow::Result> GetBufferFromFile( + const std::string& bitcode_file_path) { + auto buffer_or_error = llvm::MemoryBuffer::getFile(bitcode_file_path); - auto v1 = GetArithmeticFunctionRegistry(); - pc_registry_.insert(std::end(pc_registry_), v1.begin(), v1.end()); - auto v2 = GetDateTimeFunctionRegistry(); - pc_registry_.insert(std::end(pc_registry_), v2.begin(), v2.end()); + ARROW_RETURN_IF(!buffer_or_error, + Status::IOError("Could not load module from bitcode file: ", + bitcode_file_path + + " Error: " + buffer_or_error.getError().message())); - auto v3 = GetHashFunctionRegistry(); - pc_registry_.insert(std::end(pc_registry_), v3.begin(), v3.end()); + return std::move(buffer_or_error.get()); +} - auto v4 = GetMathOpsFunctionRegistry(); - pc_registry_.insert(std::end(pc_registry_), v4.begin(), v4.end()); +Status FunctionRegistry::Register(const std::vector& funcs, + const std::string& bitcode_path) { + ARROW_ASSIGN_OR_RAISE(auto llvm_buffer, GetBufferFromFile(bitcode_path)); + auto buffer = std::make_shared(std::move(llvm_buffer)); + return Register(funcs, std::move(buffer)); +} - auto v5 = GetStringFunctionRegistry(); - pc_registry_.insert(std::end(pc_registry_), v5.begin(), v5.end()); +arrow::Status FunctionRegistry::Register(const std::vector& funcs, + std::shared_ptr bitcode_buffer) { + bitcode_memory_buffers_.emplace_back(std::move(bitcode_buffer)); + for (const auto& func : funcs) { + ARROW_RETURN_NOT_OK(FunctionRegistry::Add(func)); + } + return Status::OK(); +} - auto v6 = GetDateTimeArithmeticFunctionRegistry(); - pc_registry_.insert(std::end(pc_registry_), v6.begin(), v6.end()); - for (auto& elem : pc_registry_) { - for (auto& func_signature : elem.signatures()) { - map.insert(std::make_pair(&(func_signature), &elem)); +const std::vector>& FunctionRegistry::GetBitcodeBuffers() + const { + return bitcode_memory_buffers_; +} + +arrow::Result> MakeDefaultFunctionRegistry() { + auto registry = std::make_shared(); + for (auto const& funcs : + {GetArithmeticFunctionRegistry(), GetDateTimeFunctionRegistry(), + GetHashFunctionRegistry(), GetMathOpsFunctionRegistry(), + GetStringFunctionRegistry(), GetDateTimeArithmeticFunctionRegistry()}) { + for (auto const& func_signature : funcs) { + ARROW_RETURN_NOT_OK(registry->Add(func_signature)); } } - - return map; + return std::move(registry); } -const NativeFunction* FunctionRegistry::LookupSignature( - const FunctionSignature& signature) const { - auto got = pc_registry_map_.find(&signature); - return got == pc_registry_map_.end() ? nullptr : got->second; +std::shared_ptr default_function_registry() { + static auto default_registry = *MakeDefaultFunctionRegistry(); + return default_registry; } } // namespace gandiva diff --git a/cpp/src/gandiva/function_registry.h b/cpp/src/gandiva/function_registry.h index d9256326019c6..01984961dc90f 100644 --- a/cpp/src/gandiva/function_registry.h +++ b/cpp/src/gandiva/function_registry.h @@ -17,7 +17,12 @@ #pragma once +#include +#include #include + +#include "arrow/buffer.h" +#include "arrow/status.h" #include "gandiva/function_registry_common.h" #include "gandiva/gandiva_aliases.h" #include "gandiva/native_function.h" @@ -30,18 +35,41 @@ class GANDIVA_EXPORT FunctionRegistry { public: using iterator = const NativeFunction*; + FunctionRegistry(); + FunctionRegistry(const FunctionRegistry&) = delete; + FunctionRegistry& operator=(const FunctionRegistry&) = delete; + /// Lookup a pre-compiled function by its signature. const NativeFunction* LookupSignature(const FunctionSignature& signature) const; + /// \brief register a set of functions into the function registry from a given bitcode + /// file + arrow::Status Register(const std::vector& funcs, + const std::string& bitcode_path); + + /// \brief register a set of functions into the function registry from a given bitcode + /// buffer + arrow::Status Register(const std::vector& funcs, + std::shared_ptr bitcode_buffer); + + /// \brief get a list of bitcode memory buffers saved in the registry + const std::vector>& GetBitcodeBuffers() const; + iterator begin() const; iterator end() const; iterator back() const; + friend arrow::Result> MakeDefaultFunctionRegistry(); + private: - static SignatureMap InitPCMap(); + std::vector pc_registry_; + SignatureMap pc_registry_map_; + std::vector> bitcode_memory_buffers_; - static std::vector pc_registry_; - static SignatureMap pc_registry_map_; + Status Add(NativeFunction func); }; +/// \brief get the default function registry +GANDIVA_EXPORT std::shared_ptr default_function_registry(); + } // namespace gandiva diff --git a/cpp/src/gandiva/function_registry_test.cc b/cpp/src/gandiva/function_registry_test.cc index e3c1e85f79cba..bbe72c0ee970c 100644 --- a/cpp/src/gandiva/function_registry_test.cc +++ b/cpp/src/gandiva/function_registry_test.cc @@ -23,17 +23,26 @@ #include #include +#include "gandiva/tests/test_util.h" + namespace gandiva { class TestFunctionRegistry : public ::testing::Test { protected: - FunctionRegistry registry_; + std::shared_ptr registry_ = gandiva::default_function_registry(); + + static std::unique_ptr MakeFunctionRegistryWithExternalFunction() { + auto registry = std::make_unique(); + ARROW_EXPECT_OK( + registry->Register({GetTestExternalFunction()}, GetTestFunctionLLVMIRPath())); + return registry; + } }; TEST_F(TestFunctionRegistry, TestFound) { FunctionSignature add_i32_i32("add", {arrow::int32(), arrow::int32()}, arrow::int32()); - const NativeFunction* function = registry_.LookupSignature(add_i32_i32); + const NativeFunction* function = registry_->LookupSignature(add_i32_i32); EXPECT_NE(function, nullptr); EXPECT_THAT(function->signatures(), testing::Contains(add_i32_i32)); EXPECT_EQ(function->pc_name(), "add_int32_int32"); @@ -42,11 +51,32 @@ TEST_F(TestFunctionRegistry, TestFound) { TEST_F(TestFunctionRegistry, TestNotFound) { FunctionSignature addX_i32_i32("addX", {arrow::int32(), arrow::int32()}, arrow::int32()); - EXPECT_EQ(registry_.LookupSignature(addX_i32_i32), nullptr); + EXPECT_EQ(registry_->LookupSignature(addX_i32_i32), nullptr); FunctionSignature add_i32_i32_ret64("add", {arrow::int32(), arrow::int32()}, arrow::int64()); - EXPECT_EQ(registry_.LookupSignature(add_i32_i32_ret64), nullptr); + EXPECT_EQ(registry_->LookupSignature(add_i32_i32_ret64), nullptr); +} + +TEST_F(TestFunctionRegistry, TestCustomFunctionRegistry) { + auto registry = MakeFunctionRegistryWithExternalFunction(); + + auto multiply_by_two_func = GetTestExternalFunction(); + auto multiply_by_two_int32_ret64 = multiply_by_two_func.signatures().front(); + EXPECT_NE(registry->LookupSignature(multiply_by_two_int32_ret64), nullptr); + + FunctionSignature add_i32_i32_ret64("add", {arrow::int32(), arrow::int32()}, + arrow::int64()); + EXPECT_EQ(registry->LookupSignature(add_i32_i32_ret64), nullptr); +} + +TEST_F(TestFunctionRegistry, TestGetBitcodeMemoryBuffersDefaultFunctionRegistry) { + EXPECT_EQ(registry_->GetBitcodeBuffers().size(), 0); +} + +TEST_F(TestFunctionRegistry, TestGetBitcodeMemoryBuffersCustomFunctionRegistry) { + auto registry = MakeFunctionRegistryWithExternalFunction(); + EXPECT_EQ(registry->GetBitcodeBuffers().size(), 1); } // one nativefunction object per precompiled function @@ -55,10 +85,9 @@ TEST_F(TestFunctionRegistry, TestNoDuplicates) { std::unordered_set native_func_duplicates; std::unordered_set func_sigs; std::unordered_set func_sig_duplicates; - for (auto native_func_it = registry_.begin(); native_func_it != registry_.end(); - ++native_func_it) { - auto& first_sig = native_func_it->signatures().front(); - auto pc_func_sig = FunctionSignature(native_func_it->pc_name(), + for (const auto& native_func_it : *registry_) { + auto& first_sig = native_func_it.signatures().front(); + auto pc_func_sig = FunctionSignature(native_func_it.pc_name(), first_sig.param_types(), first_sig.ret_type()) .ToString(); if (pc_func_sigs.count(pc_func_sig) == 0) { @@ -67,7 +96,7 @@ TEST_F(TestFunctionRegistry, TestNoDuplicates) { native_func_duplicates.insert(pc_func_sig); } - for (auto& sig : native_func_it->signatures()) { + for (auto& sig : native_func_it.signatures()) { auto sig_str = sig.ToString(); if (func_sigs.count(sig_str) == 0) { func_sigs.insert(sig_str); diff --git a/cpp/src/gandiva/llvm_generator.cc b/cpp/src/gandiva/llvm_generator.cc index fa1d97be301a8..41cbe0ffe3a3a 100644 --- a/cpp/src/gandiva/llvm_generator.cc +++ b/cpp/src/gandiva/llvm_generator.cc @@ -36,11 +36,16 @@ namespace gandiva { AddTrace(__VA_ARGS__); \ } -LLVMGenerator::LLVMGenerator(bool cached) : cached_(cached), enable_ir_traces_(false) {} +LLVMGenerator::LLVMGenerator(bool cached, + std::shared_ptr function_registry) + : cached_(cached), + function_registry_(std::move(function_registry)), + enable_ir_traces_(false) {} -Status LLVMGenerator::Make(std::shared_ptr config, bool cached, +Status LLVMGenerator::Make(const std::shared_ptr& config, bool cached, std::unique_ptr* llvm_generator) { - std::unique_ptr llvmgen_obj(new LLVMGenerator(cached)); + std::unique_ptr llvmgen_obj( + new LLVMGenerator(cached, config->function_registry())); ARROW_RETURN_NOT_OK(Engine::Make(config, cached, &(llvmgen_obj->engine_))); *llvm_generator = std::move(llvmgen_obj); @@ -64,7 +69,7 @@ void LLVMGenerator::SetLLVMObjectCache(GandivaObjectCache& object_cache) { Status LLVMGenerator::Add(const ExpressionPtr expr, const FieldDescriptorPtr output) { int idx = static_cast(compiled_exprs_.size()); // decompose the expression to separate out value and validities. - ExprDecomposer decomposer(function_registry_, annotator_); + ExprDecomposer decomposer(*function_registry_, annotator_); ValueValidityPairPtr value_validity; ARROW_RETURN_NOT_OK(decomposer.Decompose(*expr->root(), &value_validity)); // Generate the IR function for the decomposed expression. diff --git a/cpp/src/gandiva/llvm_generator.h b/cpp/src/gandiva/llvm_generator.h index 04f9b854b1d29..1921e2565338b 100644 --- a/cpp/src/gandiva/llvm_generator.h +++ b/cpp/src/gandiva/llvm_generator.h @@ -47,7 +47,7 @@ class FunctionHolder; class GANDIVA_EXPORT LLVMGenerator { public: /// \brief Factory method to initialize the generator. - static Status Make(std::shared_ptr config, bool cached, + static Status Make(const std::shared_ptr& config, bool cached, std::unique_ptr* llvm_generator); /// \brief Get the cache to be used for LLVM ObjectCache. @@ -82,11 +82,13 @@ class GANDIVA_EXPORT LLVMGenerator { std::string DumpIR() { return engine_->DumpIR(); } private: - explicit LLVMGenerator(bool cached); + explicit LLVMGenerator(bool cached, + std::shared_ptr function_registry); FRIEND_TEST(TestLLVMGenerator, VerifyPCFunctions); FRIEND_TEST(TestLLVMGenerator, TestAdd); FRIEND_TEST(TestLLVMGenerator, TestNullInternal); + FRIEND_TEST(TestLLVMGenerator, VerifyExtendedPCFunctions); llvm::LLVMContext* context() { return engine_->context(); } llvm::IRBuilder<>* ir_builder() { return engine_->ir_builder(); } @@ -250,7 +252,7 @@ class GANDIVA_EXPORT LLVMGenerator { std::unique_ptr engine_; std::vector> compiled_exprs_; bool cached_; - FunctionRegistry function_registry_; + std::shared_ptr function_registry_; Annotator annotator_; SelectionVector::Mode selection_vector_mode_; diff --git a/cpp/src/gandiva/llvm_generator_test.cc b/cpp/src/gandiva/llvm_generator_test.cc index 028893b0b4594..671ce91e870f6 100644 --- a/cpp/src/gandiva/llvm_generator_test.cc +++ b/cpp/src/gandiva/llvm_generator_test.cc @@ -35,7 +35,7 @@ typedef int64_t (*add_vector_func_t)(int64_t* elements, int nelements); class TestLLVMGenerator : public ::testing::Test { protected: - FunctionRegistry registry_; + std::shared_ptr registry_ = default_function_registry(); }; // Verify that a valid pc function exists for every function in the registry. @@ -45,7 +45,7 @@ TEST_F(TestLLVMGenerator, VerifyPCFunctions) { llvm::Module* module = generator->module(); ASSERT_OK(generator->engine_->LoadFunctionIRs()); - for (auto& iter : registry_) { + for (auto& iter : *registry_) { EXPECT_NE(module->getFunction(iter.pc_name()), nullptr); } } @@ -73,7 +73,7 @@ TEST_F(TestLLVMGenerator, TestAdd) { FunctionSignature signature(func_desc->name(), func_desc->params(), func_desc->return_type()); const NativeFunction* native_func = - generator->function_registry_.LookupSignature(signature); + generator->function_registry_->LookupSignature(signature); std::vector pairs{pair0, pair1}; auto func_dex = std::make_shared( @@ -115,4 +115,17 @@ TEST_F(TestLLVMGenerator, TestAdd) { EXPECT_EQ(out_bitmap, 0ULL); } +TEST_F(TestLLVMGenerator, VerifyExtendedPCFunctions) { + auto external_registry = std::make_shared(); + auto config_with_func_registry = + TestConfigurationWithFunctionRegistry(std::move(external_registry)); + + std::unique_ptr generator; + ASSERT_OK(LLVMGenerator::Make(config_with_func_registry, false, &generator)); + + auto module = generator->module(); + ASSERT_OK(generator->engine_->LoadFunctionIRs()); + EXPECT_NE(module->getFunction("multiply_by_two_int32"), nullptr); +} + } // namespace gandiva diff --git a/cpp/src/gandiva/native_function.h b/cpp/src/gandiva/native_function.h index 1268a25674a9d..c20de3dbdd54d 100644 --- a/cpp/src/gandiva/native_function.h +++ b/cpp/src/gandiva/native_function.h @@ -54,16 +54,16 @@ class GANDIVA_EXPORT NativeFunction { bool CanReturnErrors() const { return (flags_ & kCanReturnErrors) != 0; } NativeFunction(const std::string& base_name, const std::vector& aliases, - const DataTypeVector& param_types, DataTypePtr ret_type, - const ResultNullableType& result_nullable_type, - const std::string& pc_name, int32_t flags = 0) + const DataTypeVector& param_types, const DataTypePtr& ret_type, + const ResultNullableType& result_nullable_type, std::string pc_name, + int32_t flags = 0) : signatures_(), flags_(flags), result_nullable_type_(result_nullable_type), - pc_name_(pc_name) { - signatures_.push_back(FunctionSignature(base_name, param_types, ret_type)); + pc_name_(std::move(pc_name)) { + signatures_.emplace_back(base_name, param_types, ret_type); for (auto& func_name : aliases) { - signatures_.push_back(FunctionSignature(func_name, param_types, ret_type)); + signatures_.emplace_back(func_name, param_types, ret_type); } } diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index 3e41640861123..e62a8e3d4a375 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -project(gandiva) - set(PRECOMPILED_SRCS arithmetic_ops.cc bitmap.cc @@ -29,69 +27,18 @@ set(PRECOMPILED_SRCS time.cc timestamp_arithmetic.cc ../../arrow/util/basic_decimal.cc) - -set(PLATFORM_CLANG_OPTIONS -std=c++17) -if(MSVC) - # "19.20" means that it's compatible with Visual Studio 16 2019. - # We can update this to "19.30" when we dropped support for Visual - # Studio 16 2019. - # - # See https://cmake.org/cmake/help/latest/variable/MSVC_VERSION.html - # for MSVC_VERSION and Visual Studio version. - set(FMS_COMPATIBILITY 19.20) - list(APPEND PLATFORM_CLANG_OPTIONS -fms-compatibility - -fms-compatibility-version=${FMS_COMPATIBILITY}) -endif() - -# Create bitcode for each of the source files. -foreach(SRC_FILE ${PRECOMPILED_SRCS}) - get_filename_component(SRC_BASE ${SRC_FILE} NAME_WE) - get_filename_component(ABSOLUTE_SRC ${SRC_FILE} ABSOLUTE) - set(BC_FILE ${CMAKE_CURRENT_BINARY_DIR}/${SRC_BASE}.bc) - set(PRECOMPILE_COMMAND) - if(CMAKE_OSX_SYSROOT) - list(APPEND - PRECOMPILE_COMMAND - ${CMAKE_COMMAND} - -E - env - SDKROOT=${CMAKE_OSX_SYSROOT}) - endif() - list(APPEND - PRECOMPILE_COMMAND - ${CLANG_EXECUTABLE} - ${PLATFORM_CLANG_OPTIONS} - -DGANDIVA_IR - -DNDEBUG # DCHECK macros not implemented in precompiled code - -DARROW_STATIC # Do not set __declspec(dllimport) on MSVC on Arrow symbols - -DGANDIVA_STATIC # Do not set __declspec(dllimport) on MSVC on Gandiva symbols - -fno-use-cxa-atexit # Workaround for unresolved __dso_handle - -emit-llvm - -O3 - -c - ${ABSOLUTE_SRC} - -o - ${BC_FILE} - ${ARROW_GANDIVA_PC_CXX_FLAGS} - -I${CMAKE_SOURCE_DIR}/src - -I${ARROW_BINARY_DIR}/src) - - if(NOT ARROW_USE_NATIVE_INT128) - foreach(boost_include_dir ${Boost_INCLUDE_DIRS}) - list(APPEND PRECOMPILE_COMMAND -I${boost_include_dir}) - endforeach() - endif() - add_custom_command(OUTPUT ${BC_FILE} - COMMAND ${PRECOMPILE_COMMAND} - DEPENDS ${SRC_FILE}) - list(APPEND BC_FILES ${BC_FILE}) +set(GANDIVA_PRECOMPILED_BC_FILES) +foreach(SOURCE ${PRECOMPILED_SRCS}) + gandiva_add_bitcode(${SOURCE}) + get_filename_component(SOURCE_BASE ${SOURCE} NAME_WE) + list(APPEND GANDIVA_PRECOMPILED_BC_FILES ${CMAKE_CURRENT_BINARY_DIR}/${SOURCE_BASE}.bc) endforeach() # link all of the bitcode files into a single bitcode file. add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_BC_PATH} COMMAND ${LLVM_LINK_EXECUTABLE} -o ${GANDIVA_PRECOMPILED_BC_PATH} - ${BC_FILES} - DEPENDS ${BC_FILES}) + ${GANDIVA_PRECOMPILED_BC_FILES} + DEPENDS ${GANDIVA_PRECOMPILED_BC_FILES}) # turn the bitcode file into a C++ static data variable. add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_CC_PATH} diff --git a/cpp/src/gandiva/projector.cc b/cpp/src/gandiva/projector.cc index 7024a3bc208af..e717e825dfc71 100644 --- a/cpp/src/gandiva/projector.cc +++ b/cpp/src/gandiva/projector.cc @@ -87,7 +87,8 @@ Status Projector::Make(SchemaPtr schema, const ExpressionVector& exprs, // Return if any of the expression is invalid since // we will not be able to process further. if (!is_cached) { - ExprValidator expr_validator(llvm_gen->types(), schema); + ExprValidator expr_validator(llvm_gen->types(), schema, + configuration->function_registry()); for (auto& expr : exprs) { ARROW_RETURN_NOT_OK(expr_validator.Validate(expr)); } diff --git a/cpp/src/gandiva/tests/CMakeLists.txt b/cpp/src/gandiva/tests/CMakeLists.txt index 5fa2da16c632f..68138f50d813d 100644 --- a/cpp/src/gandiva/tests/CMakeLists.txt +++ b/cpp/src/gandiva/tests/CMakeLists.txt @@ -15,28 +15,41 @@ # specific language governing permissions and limitations # under the License. -add_gandiva_test(filter_test) -add_gandiva_test(projector_test) -add_gandiva_test(projector_build_validation_test) -add_gandiva_test(if_expr_test) -add_gandiva_test(literal_test) -add_gandiva_test(boolean_expr_test) -add_gandiva_test(binary_test) -add_gandiva_test(date_time_test) -add_gandiva_test(to_string_test) -add_gandiva_test(utf8_test) -add_gandiva_test(hash_test) -add_gandiva_test(in_expr_test) -add_gandiva_test(null_validity_test) -add_gandiva_test(decimal_test) -add_gandiva_test(decimal_single_test) -add_gandiva_test(filter_project_test) +add_gandiva_test(projector-test + SOURCES + binary_test.cc + boolean_expr_test.cc + date_time_test.cc + decimal_single_test.cc + decimal_test.cc + filter_project_test.cc + filter_test.cc + hash_test.cc + huge_table_test.cc + if_expr_test.cc + in_expr_test.cc + literal_test.cc + null_validity_test.cc + projector_build_validation_test.cc + projector_test.cc + test_util.cc + to_string_test.cc + utf8_test.cc) if(ARROW_BUILD_STATIC) - add_gandiva_test(projector_test_static SOURCES projector_test.cc USE_STATIC_LINKING) + add_gandiva_test(projector_test_static + SOURCES + projector_test.cc + test_util.cc + USE_STATIC_LINKING) add_arrow_benchmark(micro_benchmarks + SOURCES + micro_benchmarks.cc + test_util.cc PREFIX "gandiva" EXTRA_LINK_LIBS gandiva_static) endif() + +add_subdirectory(external_functions) diff --git a/cpp/src/gandiva/tests/date_time_test.cc b/cpp/src/gandiva/tests/date_time_test.cc index ce1c3d05f6638..6208f1ecba9b5 100644 --- a/cpp/src/gandiva/tests/date_time_test.cc +++ b/cpp/src/gandiva/tests/date_time_test.cc @@ -36,7 +36,7 @@ using arrow::int32; using arrow::int64; using arrow::timestamp; -class TestProjector : public ::testing::Test { +class DateTimeTestProjector : public ::testing::Test { public: void SetUp() { pool_ = arrow::default_memory_pool(); } @@ -111,7 +111,7 @@ int32_t DaysSince(time_t base_line, int32_t yy, int32_t mm, int32_t dd, int32_t return static_cast(((ts - base_line) * 1000 + millis) / MILLIS_IN_DAY); } -TEST_F(TestProjector, TestIsNull) { +TEST_F(DateTimeTestProjector, TestIsNull) { auto d0 = field("d0", date64()); auto t0 = field("t0", time32(arrow::TimeUnit::MILLI)); auto schema = arrow::schema({d0, t0}); @@ -155,7 +155,7 @@ TEST_F(TestProjector, TestIsNull) { EXPECT_ARROW_ARRAY_EQUALS(exp_isnotnull, outputs.at(1)); } -TEST_F(TestProjector, TestDate32IsNull) { +TEST_F(DateTimeTestProjector, TestDate32IsNull) { auto d0 = field("d0", date32()); auto schema = arrow::schema({d0}); @@ -191,7 +191,7 @@ TEST_F(TestProjector, TestDate32IsNull) { EXPECT_ARROW_ARRAY_EQUALS(exp_isnull, outputs.at(0)); } -TEST_F(TestProjector, TestDateTime) { +TEST_F(DateTimeTestProjector, TestDateTime) { auto field0 = field("f0", date64()); auto field1 = field("f1", date32()); auto field2 = field("f2", timestamp(arrow::TimeUnit::MILLI)); @@ -292,7 +292,7 @@ TEST_F(TestProjector, TestDateTime) { EXPECT_ARROW_ARRAY_EQUALS(exp_dd_from_ts, outputs.at(5)); } -TEST_F(TestProjector, TestTime) { +TEST_F(DateTimeTestProjector, TestTime) { auto field0 = field("f0", time32(arrow::TimeUnit::MILLI)); auto schema = arrow::schema({field0}); @@ -339,7 +339,7 @@ TEST_F(TestProjector, TestTime) { EXPECT_ARROW_ARRAY_EQUALS(exp_hour, outputs.at(1)); } -TEST_F(TestProjector, TestTimestampDiff) { +TEST_F(DateTimeTestProjector, TestTimestampDiff) { auto f0 = field("f0", timestamp(arrow::TimeUnit::MILLI)); auto f1 = field("f1", timestamp(arrow::TimeUnit::MILLI)); auto schema = arrow::schema({f0, f1}); @@ -439,7 +439,7 @@ TEST_F(TestProjector, TestTimestampDiff) { } } -TEST_F(TestProjector, TestTimestampDiffMonth) { +TEST_F(DateTimeTestProjector, TestTimestampDiffMonth) { auto f0 = field("f0", timestamp(arrow::TimeUnit::MILLI)); auto f1 = field("f1", timestamp(arrow::TimeUnit::MILLI)); auto schema = arrow::schema({f0, f1}); @@ -497,7 +497,7 @@ TEST_F(TestProjector, TestTimestampDiffMonth) { } } -TEST_F(TestProjector, TestMonthsBetween) { +TEST_F(DateTimeTestProjector, TestMonthsBetween) { auto f0 = field("f0", arrow::date64()); auto f1 = field("f1", arrow::date64()); auto schema = arrow::schema({f0, f1}); @@ -550,7 +550,7 @@ TEST_F(TestProjector, TestMonthsBetween) { EXPECT_ARROW_ARRAY_EQUALS(exp_output, outputs.at(0)); } -TEST_F(TestProjector, TestCastTimestampFromInt64) { +TEST_F(DateTimeTestProjector, TestCastTimestampFromInt64) { auto f0 = field("f0", arrow::int64()); auto schema = arrow::schema({f0}); @@ -600,7 +600,7 @@ TEST_F(TestProjector, TestCastTimestampFromInt64) { EXPECT_ARROW_ARRAY_EQUALS(exp_output, outputs.at(0)); } -TEST_F(TestProjector, TestLastDay) { +TEST_F(DateTimeTestProjector, TestLastDay) { auto f0 = field("f0", arrow::date64()); auto schema = arrow::schema({f0}); @@ -650,7 +650,7 @@ TEST_F(TestProjector, TestLastDay) { EXPECT_ARROW_ARRAY_EQUALS(exp_output, outputs.at(0)); } -TEST_F(TestProjector, TestToTimestampFromInt) { +TEST_F(DateTimeTestProjector, TestToTimestampFromInt) { auto f0 = field("f0", arrow::int32()); auto f1 = field("f1", arrow::int64()); auto f2 = field("f2", arrow::float32()); @@ -721,7 +721,7 @@ TEST_F(TestProjector, TestToTimestampFromInt) { EXPECT_ARROW_ARRAY_EQUALS(exp_output1, outputs.at(3)); } -TEST_F(TestProjector, TestToUtcTimestamp) { +TEST_F(DateTimeTestProjector, TestToUtcTimestamp) { auto f0 = field("f0", timestamp(arrow::TimeUnit::MILLI)); auto f1 = field("f1", arrow::utf8()); @@ -775,7 +775,7 @@ TEST_F(TestProjector, TestToUtcTimestamp) { EXPECT_ARROW_ARRAY_EQUALS(exp_output, outputs.at(0)); } -TEST_F(TestProjector, TestFromUtcTimestamp) { +TEST_F(DateTimeTestProjector, TestFromUtcTimestamp) { auto f0 = field("f0", timestamp(arrow::TimeUnit::MILLI)); auto f1 = field("f1", arrow::utf8()); diff --git a/cpp/src/gandiva/tests/external_functions/CMakeLists.txt b/cpp/src/gandiva/tests/external_functions/CMakeLists.txt new file mode 100644 index 0000000000000..c309549e874e3 --- /dev/null +++ b/cpp/src/gandiva/tests/external_functions/CMakeLists.txt @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(NO_TESTS) + return() +endif() +# +## copy the testing data into the build directory +add_custom_target(extension-tests-data + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}) + +set(TEST_PRECOMPILED_SOURCES multiply_by_two.cc) +set(TEST_PRECOMPILED_BC_FILES) +foreach(SOURCE ${TEST_PRECOMPILED_SOURCES}) + gandiva_add_bitcode(${SOURCE}) + get_filename_component(SOURCE_BASE ${SOURCE} NAME_WE) + list(APPEND TEST_PRECOMPILED_BC_FILES ${CMAKE_CURRENT_BINARY_DIR}/${SOURCE_BASE}.bc) +endforeach() +add_custom_target(extension-tests ALL DEPENDS extension-tests-data + ${TEST_PRECOMPILED_BC_FILES}) +# +## set the GANDIVA_EXTENSION_TEST_DIR macro so that the tests can pass regardless where they are run from +## corresponding extension test data files and bitcode will be copied/generated +set(TARGETS_DEPENDING_ON_TEST_BITCODE_FILES gandiva-internals-test gandiva-projector-test + gandiva-projector-test-static) +foreach(TARGET ${TARGETS_DEPENDING_ON_TEST_BITCODE_FILES}) + if(TARGET ${TARGET}) + add_dependencies(${TARGET} extension-tests) + target_compile_definitions(${TARGET} + PRIVATE -DGANDIVA_EXTENSION_TEST_DIR="${CMAKE_CURRENT_BINARY_DIR}" + ) + endif() +endforeach() + +add_dependencies(gandiva-tests extension-tests) diff --git a/cpp/src/gandiva/tests/external_functions/multiply_by_two.cc b/cpp/src/gandiva/tests/external_functions/multiply_by_two.cc new file mode 100644 index 0000000000000..cc7e2b0f8267f --- /dev/null +++ b/cpp/src/gandiva/tests/external_functions/multiply_by_two.cc @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "multiply_by_two.h" // NOLINT + +int64_t multiply_by_two_int32(int32_t value) { return value * 2; } diff --git a/cpp/src/gandiva/tests/external_functions/multiply_by_two.h b/cpp/src/gandiva/tests/external_functions/multiply_by_two.h new file mode 100644 index 0000000000000..b8aec5185457b --- /dev/null +++ b/cpp/src/gandiva/tests/external_functions/multiply_by_two.h @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +extern "C" { +int64_t multiply_by_two_int32(int32_t value); +} diff --git a/cpp/src/gandiva/tests/filter_test.cc b/cpp/src/gandiva/tests/filter_test.cc index effd31cc27aa0..749000aa0cf27 100644 --- a/cpp/src/gandiva/tests/filter_test.cc +++ b/cpp/src/gandiva/tests/filter_test.cc @@ -42,8 +42,8 @@ class TestFilter : public ::testing::Test { TEST_F(TestFilter, TestFilterCache) { // schema for input fields - auto field0 = field("f0", int32()); - auto field1 = field("f1", int32()); + auto field0 = field("f0_filter_cache", int32()); + auto field1 = field("f1_filter_cache", int32()); auto schema = arrow::schema({field0, field1}); // Build condition f0 + f1 < 10 @@ -69,7 +69,7 @@ TEST_F(TestFilter, TestFilterCache) { EXPECT_TRUE(cached_filter->GetBuiltFromCache()); // schema is different should return a new filter. - auto field2 = field("f2", int32()); + auto field2 = field("f2_filter_cache", int32()); auto different_schema = arrow::schema({field0, field1, field2}); std::shared_ptr should_be_new_filter; status = diff --git a/cpp/src/gandiva/tests/huge_table_test.cc b/cpp/src/gandiva/tests/huge_table_test.cc index 46f814b472d84..34c8512f1b0a9 100644 --- a/cpp/src/gandiva/tests/huge_table_test.cc +++ b/cpp/src/gandiva/tests/huge_table_test.cc @@ -139,8 +139,11 @@ TEST_F(LARGE_MEMORY_TEST(TestHugeFilter), TestSimpleHugeFilter) { auto status = Filter::Make(schema, condition, TestConfiguration(), &filter); EXPECT_TRUE(status.ok()); + auto array1 = MakeArrowArray(arr1, validity); + auto array2 = MakeArrowArray(arr2, validity); + // prepare input record batch - auto in_batch = arrow::RecordBatch::Make(schema, num_records, {arr1, arr2}); + auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array1, array2}); std::shared_ptr selection_vector; status = SelectionVector::MakeInt64(num_records, pool_, &selection_vector); diff --git a/cpp/src/gandiva/tests/projector_build_validation_test.cc b/cpp/src/gandiva/tests/projector_build_validation_test.cc index 5b86844f940bf..1ed4c77a074ab 100644 --- a/cpp/src/gandiva/tests/projector_build_validation_test.cc +++ b/cpp/src/gandiva/tests/projector_build_validation_test.cc @@ -27,7 +27,7 @@ using arrow::boolean; using arrow::float32; using arrow::int32; -class TestProjector : public ::testing::Test { +class ValidationTestProjector : public ::testing::Test { public: void SetUp() { pool_ = arrow::default_memory_pool(); } @@ -35,7 +35,7 @@ class TestProjector : public ::testing::Test { arrow::MemoryPool* pool_; }; -TEST_F(TestProjector, TestNonexistentFunction) { +TEST_F(ValidationTestProjector, TestNonexistentFunction) { // schema for input fields auto field0 = field("f0", float32()); auto field1 = field("f2", float32()); @@ -57,7 +57,7 @@ TEST_F(TestProjector, TestNonexistentFunction) { EXPECT_TRUE(status.message().find(expected_error) != std::string::npos); } -TEST_F(TestProjector, TestNotMatchingDataType) { +TEST_F(ValidationTestProjector, TestNotMatchingDataType) { // schema for input fields auto field0 = field("f0", float32()); auto schema = arrow::schema({field0}); @@ -78,7 +78,7 @@ TEST_F(TestProjector, TestNotMatchingDataType) { EXPECT_TRUE(status.message().find(expected_error) != std::string::npos); } -TEST_F(TestProjector, TestNotSupportedDataType) { +TEST_F(ValidationTestProjector, TestNotSupportedDataType) { // schema for input fields auto field0 = field("f0", list(int32())); auto schema = arrow::schema({field0}); @@ -98,7 +98,7 @@ TEST_F(TestProjector, TestNotSupportedDataType) { EXPECT_TRUE(status.message().find(expected_error) != std::string::npos); } -TEST_F(TestProjector, TestIncorrectSchemaMissingField) { +TEST_F(ValidationTestProjector, TestIncorrectSchemaMissingField) { // schema for input fields auto field0 = field("f0", float32()); auto field1 = field("f2", float32()); @@ -119,7 +119,7 @@ TEST_F(TestProjector, TestIncorrectSchemaMissingField) { EXPECT_TRUE(status.message().find(expected_error) != std::string::npos); } -TEST_F(TestProjector, TestIncorrectSchemaTypeNotMatching) { +TEST_F(ValidationTestProjector, TestIncorrectSchemaTypeNotMatching) { // schema for input fields auto field0 = field("f0", float32()); auto field1 = field("f2", float32()); @@ -142,7 +142,7 @@ TEST_F(TestProjector, TestIncorrectSchemaTypeNotMatching) { EXPECT_TRUE(status.message().find(expected_error) != std::string::npos); } -TEST_F(TestProjector, TestIfNotSupportedFunction) { +TEST_F(ValidationTestProjector, TestIfNotSupportedFunction) { // schema for input fields auto fielda = field("a", int32()); auto fieldb = field("b", int32()); @@ -170,7 +170,7 @@ TEST_F(TestProjector, TestIfNotSupportedFunction) { EXPECT_TRUE(status.IsExpressionValidationError()); } -TEST_F(TestProjector, TestIfNotMatchingReturnType) { +TEST_F(ValidationTestProjector, TestIfNotMatchingReturnType) { // schema for input fields auto fielda = field("a", int32()); auto fieldb = field("b", int32()); @@ -193,7 +193,7 @@ TEST_F(TestProjector, TestIfNotMatchingReturnType) { EXPECT_TRUE(status.IsExpressionValidationError()); } -TEST_F(TestProjector, TestElseNotMatchingReturnType) { +TEST_F(ValidationTestProjector, TestElseNotMatchingReturnType) { // schema for input fields auto fielda = field("a", int32()); auto fieldb = field("b", int32()); @@ -218,7 +218,7 @@ TEST_F(TestProjector, TestElseNotMatchingReturnType) { EXPECT_TRUE(status.IsExpressionValidationError()); } -TEST_F(TestProjector, TestElseNotSupportedType) { +TEST_F(ValidationTestProjector, TestElseNotSupportedType) { // schema for input fields auto fielda = field("a", int32()); auto fieldb = field("b", int32()); @@ -244,7 +244,7 @@ TEST_F(TestProjector, TestElseNotSupportedType) { EXPECT_EQ(status.code(), StatusCode::ExpressionValidationError); } -TEST_F(TestProjector, TestAndMinChildren) { +TEST_F(ValidationTestProjector, TestAndMinChildren) { // schema for input fields auto fielda = field("a", boolean()); auto schema = arrow::schema({fielda}); @@ -263,7 +263,7 @@ TEST_F(TestProjector, TestAndMinChildren) { EXPECT_TRUE(status.IsExpressionValidationError()); } -TEST_F(TestProjector, TestAndBooleanArgType) { +TEST_F(ValidationTestProjector, TestAndBooleanArgType) { // schema for input fields auto fielda = field("a", boolean()); auto fieldb = field("b", int32()); diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 462fae64393fd..38566fb408ab5 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -26,6 +26,7 @@ #include #include "arrow/memory_pool.h" +#include "gandiva/function_registry.h" #include "gandiva/literal_holder.h" #include "gandiva/node.h" #include "gandiva/tests/test_util.h" @@ -3582,4 +3583,29 @@ TEST_F(TestProjector, TestSqrtFloat64) { EXPECT_ARROW_ARRAY_EQUALS(out, outs.at(0)); } +TEST_F(TestProjector, TestExtendedFunctions) { + auto in_field = field("in", arrow::int32()); + auto schema = arrow::schema({in_field}); + auto out_field = field("out", arrow::int64()); + // the multiply_by_two function is only available in the external function's IR bitcode + auto multiply = + TreeExprBuilder::MakeExpression("multiply_by_two", {in_field}, out_field); + + std::shared_ptr projector; + auto external_registry = std::make_shared(); + auto config_with_func_registry = + TestConfigurationWithFunctionRegistry(std::move(external_registry)); + ARROW_EXPECT_OK( + Projector::Make(schema, {multiply}, config_with_func_registry, &projector)); + + int num_records = 4; + auto array = MakeArrowArrayInt32({1, 2, 3, 4}, {true, true, true, true}); + auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array}); + auto out = MakeArrowArrayInt64({2, 4, 6, 8}, {true, true, true, true}); + + arrow::ArrayVector outs; + ARROW_EXPECT_OK(projector->Evaluate(*in_batch, pool_, &outs)); + EXPECT_ARROW_ARRAY_EQUALS(out, outs.at(0)); +} + } // namespace gandiva diff --git a/cpp/src/gandiva/tests/test_util.cc b/cpp/src/gandiva/tests/test_util.cc new file mode 100644 index 0000000000000..42f67d3824a21 --- /dev/null +++ b/cpp/src/gandiva/tests/test_util.cc @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gandiva/tests/test_util.h" + +#include + +namespace gandiva { +std::shared_ptr TestConfiguration() { + auto builder = ConfigurationBuilder(); + return builder.DefaultConfiguration(); +} + +#ifndef GANDIVA_EXTENSION_TEST_DIR +#define GANDIVA_EXTENSION_TEST_DIR "." +#endif + +std::string GetTestFunctionLLVMIRPath() { + std::filesystem::path base(GANDIVA_EXTENSION_TEST_DIR); + std::filesystem::path ir_file = base / "multiply_by_two.bc"; + return ir_file.string(); +} + +NativeFunction GetTestExternalFunction() { + NativeFunction multiply_by_two_func( + "multiply_by_two", {}, {arrow::int32()}, arrow::int64(), + ResultNullableType::kResultNullIfNull, "multiply_by_two_int32"); + return multiply_by_two_func; +} + +std::shared_ptr TestConfigurationWithFunctionRegistry( + std::shared_ptr registry) { + ARROW_EXPECT_OK( + registry->Register({GetTestExternalFunction()}, GetTestFunctionLLVMIRPath())); + auto external_func_config = ConfigurationBuilder().build(std::move(registry)); + return external_func_config; +} +} // namespace gandiva diff --git a/cpp/src/gandiva/tests/test_util.h b/cpp/src/gandiva/tests/test_util.h index 99df90769e0ad..e431e53096c2c 100644 --- a/cpp/src/gandiva/tests/test_util.h +++ b/cpp/src/gandiva/tests/test_util.h @@ -96,9 +96,12 @@ static inline ArrayPtr MakeArrowTypeArray(const std::shared_ptr EXPECT_TRUE((a)->Equals(b)) << "expected type: " << (a)->ToString() \ << " actual type: " << (b)->ToString() -static inline std::shared_ptr TestConfiguration() { - auto builder = ConfigurationBuilder(); - return builder.DefaultConfiguration(); -} +std::shared_ptr TestConfiguration(); + +std::shared_ptr TestConfigurationWithFunctionRegistry( + std::shared_ptr registry); + +std::string GetTestFunctionLLVMIRPath(); +NativeFunction GetTestExternalFunction(); } // namespace gandiva diff --git a/cpp/src/gandiva/tree_expr_test.cc b/cpp/src/gandiva/tree_expr_test.cc index e70cf12898124..86a826f29367f 100644 --- a/cpp/src/gandiva/tree_expr_test.cc +++ b/cpp/src/gandiva/tree_expr_test.cc @@ -45,7 +45,7 @@ class TestExprTree : public ::testing::Test { FieldPtr i1_; // int32 FieldPtr b0_; // bool - FunctionRegistry registry_; + std::shared_ptr registry_ = gandiva::default_function_registry(); }; TEST_F(TestExprTree, TestField) { @@ -57,7 +57,7 @@ TEST_F(TestExprTree, TestField) { auto n1 = TreeExprBuilder::MakeField(b0_); EXPECT_EQ(n1->return_type(), boolean()); - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); ValueValidityPairPtr pair; auto status = decomposer.Decompose(*n1, &pair); DCHECK_EQ(status.ok(), true) << status.message(); @@ -88,7 +88,7 @@ TEST_F(TestExprTree, TestBinary) { EXPECT_EQ(add->return_type(), int32()); EXPECT_TRUE(sign == FunctionSignature("add", {int32(), int32()}, int32())); - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); ValueValidityPairPtr pair; auto status = decomposer.Decompose(*n, &pair); DCHECK_EQ(status.ok(), true) << status.message(); @@ -97,7 +97,7 @@ TEST_F(TestExprTree, TestBinary) { auto null_if_null = std::dynamic_pointer_cast(value); FunctionSignature signature("add", {int32(), int32()}, int32()); - const NativeFunction* fn = registry_.LookupSignature(signature); + const NativeFunction* fn = registry_->LookupSignature(signature); EXPECT_EQ(null_if_null->native_function(), fn); } @@ -114,7 +114,7 @@ TEST_F(TestExprTree, TestUnary) { EXPECT_EQ(unaryFn->return_type(), boolean()); EXPECT_TRUE(sign == FunctionSignature("isnumeric", {int32()}, boolean())); - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); ValueValidityPairPtr pair; auto status = decomposer.Decompose(*n, &pair); DCHECK_EQ(status.ok(), true) << status.message(); @@ -123,7 +123,7 @@ TEST_F(TestExprTree, TestUnary) { auto never_null = std::dynamic_pointer_cast(value); FunctionSignature signature("isnumeric", {int32()}, boolean()); - const NativeFunction* fn = registry_.LookupSignature(signature); + const NativeFunction* fn = registry_->LookupSignature(signature); EXPECT_EQ(never_null->native_function(), fn); } @@ -143,7 +143,7 @@ TEST_F(TestExprTree, TestExpression) { func_desc->return_type()); EXPECT_TRUE(sign == FunctionSignature("add", {int32(), int32()}, int32())); - ExprDecomposer decomposer(registry_, annotator); + ExprDecomposer decomposer(*registry_, annotator); ValueValidityPairPtr pair; auto status = decomposer.Decompose(*root_node, &pair); DCHECK_EQ(status.ok(), true) << status.message(); @@ -152,7 +152,7 @@ TEST_F(TestExprTree, TestExpression) { auto null_if_null = std::dynamic_pointer_cast(value); FunctionSignature signature("add", {int32(), int32()}, int32()); - const NativeFunction* fn = registry_.LookupSignature(signature); + const NativeFunction* fn = registry_->LookupSignature(signature); EXPECT_EQ(null_if_null->native_function(), fn); } From e62ec62e40b04b0bfce76d58369845d3aa96a419 Mon Sep 17 00:00:00 2001 From: Eero Lihavainen Date: Wed, 8 Nov 2023 08:32:32 +0200 Subject: [PATCH 20/57] GH-38591: [Parquet][C++] Remove redundant open calls in `ParquetFileFormat::GetReaderAsync` (#38621) ### Rationale for this change There were duplicate method calls causing extra I/O operations, apparently unintentional from https://github.com/apache/arrow/commit/0793432ad0ef5cb598b7b1e61071cd4991bd1b8b. ### What changes are included in this PR? Remove the extra method calls. ### Are these changes tested? ### Are there any user-facing changes? * Closes: #38591 Authored-by: Eero Lihavainen Signed-off-by: mwish --- cpp/src/arrow/dataset/file_parquet.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cpp/src/arrow/dataset/file_parquet.cc b/cpp/src/arrow/dataset/file_parquet.cc index d486f194f38a3..65ad70181f28a 100644 --- a/cpp/src/arrow/dataset/file_parquet.cc +++ b/cpp/src/arrow/dataset/file_parquet.cc @@ -504,11 +504,6 @@ Future> ParquetFileFormat::GetReader default_fragment_scan_options)); auto properties = MakeReaderProperties(*this, parquet_scan_options.get(), source.path(), source.filesystem(), options->pool); - ARROW_ASSIGN_OR_RAISE(auto input, source.Open()); - // TODO(ARROW-12259): workaround since we have Future<(move-only type)> - auto reader_fut = parquet::ParquetFileReader::OpenAsync( - std::move(input), std::move(properties), metadata); - auto path = source.path(); auto self = checked_pointer_cast(shared_from_this()); return source.OpenAsync().Then( From 3d96bab9916e24e18255a2c2cde306a023108c5c Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Wed, 8 Nov 2023 12:52:18 -0400 Subject: [PATCH 21/57] MINOR: [R] Fix .lintr config file for updated lintr dependency (#38639) ### Rationale for this change The lint CI job is failing because an update to a dependency of lintr now does more strict DCF checking on the .lintr file ### What changes are included in this PR? The 'license' field was wrapped in quotes so that .lintr passes validation ### Are these changes tested? Yes, by existing CI job ### Are there any user-facing changes? No Authored-by: Dewey Dunnington Signed-off-by: Antoine Pitrou --- r/.lintr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/.lintr b/r/.lintr index 085ff45123411..d920f77e9bbd3 100644 --- a/r/.lintr +++ b/r/.lintr @@ -1,4 +1,4 @@ -license: # Licensed to the Apache Software Foundation (ASF) under one +license: '# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file @@ -13,7 +13,7 @@ license: # Licensed to the Apache Software Foundation (ASF) under one # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations - # under the License. + # under the License.' linters: linters_with_defaults( indentation_linter = NULL, line_length_linter = line_length_linter(120), From db19a358cdf2b550b741fc7574dea38a6e32261b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Giraldo?= Date: Wed, 8 Nov 2023 13:40:57 -0700 Subject: [PATCH 22/57] GH-38627: [Java][FlightRPC] Handle null parameter values (#38628) ### Rationale for this change We want to make sure we correctly handle binding `null` values to JDBC parameters. We also want better exceptions when handling parameter binding. ### What changes are included in this PR? - Handle adding null values to parameters if it's a nullable vector, else throw `UnsupportedOperationException` - For unsupported parameter types or casts, throw `UnsupportedOperationException` instead of `RuntimeException` * Closes: #38627 Authored-by: Diego Fernandez Signed-off-by: David Li --- .../jdbc/converter/AvaticaParameterConverter.java | 7 ++++++- .../impl/TimestampAvaticaParameterConverter.java | 1 - .../driver/jdbc/utils/AvaticaParameterBinder.java | 12 +++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/AvaticaParameterConverter.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/AvaticaParameterConverter.java index c01e688f37396..db34a8c9e1582 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/AvaticaParameterConverter.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/AvaticaParameterConverter.java @@ -33,10 +33,15 @@ public interface AvaticaParameterConverter { * * @param vector FieldVector that the parameter should be bound to. * @param typedValue TypedValue to bind as a parameter. - * @param index Vector index that the TypedValue should be bound to. + * @param index Vector index (0-indexed) that the TypedValue should be bound to. * @return Whether the value was set successfully. */ boolean bindParameter(FieldVector vector, TypedValue typedValue, int index); + /** + * Create an AvaticaParameter from the given Field. + * + * @param field Arrow Field to generate an AvaticaParameter from. + */ AvaticaParameter createParameter(Field field); } diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/TimestampAvaticaParameterConverter.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/TimestampAvaticaParameterConverter.java index 3c1940b75cfa7..eb3316a1bdbed 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/TimestampAvaticaParameterConverter.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/TimestampAvaticaParameterConverter.java @@ -41,7 +41,6 @@ public TimestampAvaticaParameterConverter(ArrowType.Timestamp type) { @Override public boolean bindParameter(FieldVector vector, TypedValue typedValue, int index) { - // FIXME: how should we handle TZ? Do we need to convert the value to the TZ on the vector? long value = (long) typedValue.toLocal(); if (vector instanceof TimeStampSecVector) { ((TimeStampSecVector) vector).setSafe(index, value); diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java index c5be4697db7c5..9e805fc79bcf8 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java @@ -102,12 +102,18 @@ public void bind(List typedValues, int index) { */ private void bind(FieldVector vector, TypedValue typedValue, int index) { try { - if (!vector.getField().getType().accept(new BinderVisitor(vector, typedValue, index))) { - throw new RuntimeException( + if (typedValue.value == null) { + if (vector.getField().isNullable()) { + vector.setNull(index); + } else { + throw new UnsupportedOperationException("Can't set null on non-nullable parameter"); + } + } else if (!vector.getField().getType().accept(new BinderVisitor(vector, typedValue, index))) { + throw new UnsupportedOperationException( String.format("Binding to vector type %s is not yet supported", vector.getClass())); } } catch (ClassCastException e) { - throw new RuntimeException( + throw new UnsupportedOperationException( String.format("Binding value of type %s is not yet supported for expected Arrow type %s", typedValue.type, vector.getField().getType())); } From 75a04030996261333e843a93f4981156372fa7f8 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Thu, 9 Nov 2023 04:11:22 +0000 Subject: [PATCH 23/57] GH-38335: [C++] Implement `GetFileInfo` for a single file in Azure filesystem (#38505) ### Rationale for this change `GetFileInfo` is an important part of an Arrow filesystem implementation. ### What changes are included in this PR? - Start `azurefs_internal` similar to GCS and S3 filesystems. - Implement `HierarchicalNamespaceDetector`. - This does not use the obvious and simple implementation. It uses a more complicated option inspired by `hadoop-azure` that avoids requiring the significantly elevated permissions needed for `blob_service_client->GetAccountInfo()`. - This can't be detected an initialisation time of the filesystem because it requires a `container_name`. Its packed into its only class so that the result can be cached. - Implement `GetFileInfo` for single paths. - Supports hierarchical or flat namespace accounts and takes advantage of hierarchical namespace where possible to avoid unnecessary extra calls to blob storage. The performance difference is actually noticeable just from running the `GetFileInfoObjectWithNestedStructure` test against real flat and hierarchical accounts. Its about 3 seconds with hierarchical namespace or 5 seconds with a flat namespace. - Update tests with TODO(GH-38335) to now use this implementation of `GetFileInfo` to replace the temporary direct Azure SDK usage. - Rename the main test fixture and introduce new ones for connecting to real blob storage. If details of real blob storage is not provided then the real blob storage tests will be skipped. ### Are these changes tested? Yes. There are new Azurite based tests for everything that can be tested with Azurite. There are also some tests that are designed to test against a real blob storage account. This is because [Azurite cannot emulate a hierarchical namespace account](https://github.com/Azure/Azurite/issues/553). Additionally some of the behaviour used to detect a hierarchical namespace account is different on Azurite compared to a real flat namespace account. These tests will be automatically skipped unless environment variables are provided with details for connecting to the relevant real storage accounts. Initially I based the tests on the GCS filesystem but I added a few extras where I thought it was appropriate. ### Are there any user-facing changes? Yes. `GetFileInfo` is now supported on the Azure filesystem. * Closes: #38335 Lead-authored-by: Thomas Newton Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- cpp/src/arrow/CMakeLists.txt | 4 +- cpp/src/arrow/filesystem/azurefs.cc | 158 ++++++++-- cpp/src/arrow/filesystem/azurefs_internal.cc | 88 ++++++ cpp/src/arrow/filesystem/azurefs_internal.h | 42 +++ cpp/src/arrow/filesystem/azurefs_test.cc | 288 ++++++++++++++----- cpp/src/arrow/filesystem/path_util.cc | 11 +- cpp/src/arrow/filesystem/path_util.h | 3 + 7 files changed, 497 insertions(+), 97 deletions(-) create mode 100644 cpp/src/arrow/filesystem/azurefs_internal.cc create mode 100644 cpp/src/arrow/filesystem/azurefs_internal.h diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 9a6117011535e..101b089ba837f 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -502,8 +502,8 @@ if(ARROW_FILESYSTEM) filesystem/util_internal.cc) if(ARROW_AZURE) - list(APPEND ARROW_SRCS filesystem/azurefs.cc) - set_source_files_properties(filesystem/azurefs.cc + list(APPEND ARROW_SRCS filesystem/azurefs.cc filesystem/azurefs_internal.cc) + set_source_files_properties(filesystem/azurefs.cc filesystem/azurefs_internal.cc PROPERTIES SKIP_PRECOMPILE_HEADERS ON SKIP_UNITY_BUILD_INCLUSION ON) endif() diff --git a/cpp/src/arrow/filesystem/azurefs.cc b/cpp/src/arrow/filesystem/azurefs.cc index a04338d999d11..6359183d90bb4 100644 --- a/cpp/src/arrow/filesystem/azurefs.cc +++ b/cpp/src/arrow/filesystem/azurefs.cc @@ -16,8 +16,10 @@ // under the License. #include "arrow/filesystem/azurefs.h" +#include "arrow/filesystem/azurefs_internal.h" #include +#include #include "arrow/buffer.h" #include "arrow/filesystem/path_util.h" @@ -59,6 +61,7 @@ Status AzureOptions::ConfigureAccountKeyCredentials(const std::string& account_n credentials_kind = AzureCredentialsKind::StorageCredentials; return Status::OK(); } + namespace { // An AzureFileSystem represents a single Azure storage account. AzurePath describes a @@ -79,18 +82,17 @@ struct AzurePath { "Expected an Azure object path of the form 'container/path...', got a URI: '", s, "'"); } - const auto src = internal::RemoveTrailingSlash(s); - auto first_sep = src.find_first_of(internal::kSep); + auto first_sep = s.find_first_of(internal::kSep); if (first_sep == 0) { return Status::Invalid("Path cannot start with a separator ('", s, "')"); } if (first_sep == std::string::npos) { - return AzurePath{std::string(src), std::string(src), "", {}}; + return AzurePath{std::string(s), std::string(s), "", {}}; } AzurePath path; - path.full_path = std::string(src); - path.container = std::string(src.substr(0, first_sep)); - path.path_to_file = std::string(src.substr(first_sep + 1)); + path.full_path = std::string(s); + path.container = std::string(s.substr(0, first_sep)); + path.path_to_file = std::string(s.substr(first_sep + 1)); path.path_to_file_parts = internal::SplitAbstractPath(path.path_to_file); RETURN_NOT_OK(Validate(path)); return path; @@ -146,11 +148,6 @@ Status ValidateFilePath(const AzurePath& path) { return Status::OK(); } -Status ErrorToStatus(const std::string& prefix, - const Azure::Storage::StorageException& exception) { - return Status::IOError(prefix, " Azure Error: ", exception.what()); -} - template std::string FormatValue(typename TypeTraits::CType value) { struct StringAppender { @@ -316,11 +313,13 @@ class ObjectInputFile final : public io::RandomAccessFile { return Status::OK(); } catch (const Azure::Storage::StorageException& exception) { if (exception.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { - // Could be either container or blob not found. return PathNotFound(path_); } - return ErrorToStatus( - "When fetching properties for '" + blob_client_->GetUrl() + "': ", exception); + return internal::ExceptionToStatus( + "GetProperties failed for '" + blob_client_->GetUrl() + + "' with an unexpected Azure error. Can not initialise an ObjectInputFile " + "without knowing the file size.", + exception); } } @@ -397,10 +396,12 @@ class ObjectInputFile final : public io::RandomAccessFile { ->DownloadTo(reinterpret_cast(out), nbytes, download_options) .Value.ContentRange.Length.Value(); } catch (const Azure::Storage::StorageException& exception) { - return ErrorToStatus("When reading from '" + blob_client_->GetUrl() + - "' at position " + std::to_string(position) + " for " + - std::to_string(nbytes) + " bytes: ", - exception); + return internal::ExceptionToStatus("DownloadTo from '" + blob_client_->GetUrl() + + "' at position " + std::to_string(position) + + " for " + std::to_string(nbytes) + + " bytes failed with an Azure error. ReadAt " + "failed to read the required byte range.", + exception); } } @@ -444,7 +445,6 @@ class ObjectInputFile final : public io::RandomAccessFile { int64_t content_length_ = kNoSize; std::shared_ptr metadata_; }; - } // namespace // ----------------------------------------------------------------------- @@ -453,27 +453,136 @@ class ObjectInputFile final : public io::RandomAccessFile { class AzureFileSystem::Impl { public: io::IOContext io_context_; - std::shared_ptr service_client_; + std::unique_ptr + datalake_service_client_; + std::unique_ptr blob_service_client_; AzureOptions options_; + internal::HierarchicalNamespaceDetector hierarchical_namespace_; explicit Impl(AzureOptions options, io::IOContext io_context) : io_context_(io_context), options_(std::move(options)) {} Status Init() { - service_client_ = std::make_shared( + blob_service_client_ = std::make_unique( options_.account_blob_url, options_.storage_credentials_provider); + datalake_service_client_ = + std::make_unique( + options_.account_dfs_url, options_.storage_credentials_provider); + RETURN_NOT_OK(hierarchical_namespace_.Init(datalake_service_client_.get())); return Status::OK(); } const AzureOptions& options() const { return options_; } + public: + Result GetFileInfo(const AzurePath& path) { + FileInfo info; + info.set_path(path.full_path); + + if (path.container.empty()) { + DCHECK(path.path_to_file.empty()); // The path is invalid if the container is empty + // but not path_to_file. + // path must refer to the root of the Azure storage account. This is a directory, + // and there isn't any extra metadata to fetch. + info.set_type(FileType::Directory); + return info; + } + if (path.path_to_file.empty()) { + // path refers to a container. This is a directory if it exists. + auto container_client = + blob_service_client_->GetBlobContainerClient(path.container); + try { + auto properties = container_client.GetProperties(); + info.set_type(FileType::Directory); + info.set_mtime( + std::chrono::system_clock::time_point(properties.Value.LastModified)); + return info; + } catch (const Azure::Storage::StorageException& exception) { + if (exception.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + info.set_type(FileType::NotFound); + return info; + } + return internal::ExceptionToStatus( + "GetProperties for '" + container_client.GetUrl() + + "' failed with an unexpected Azure error. GetFileInfo is unable to " + "determine whether the container exists.", + exception); + } + } + auto file_client = datalake_service_client_->GetFileSystemClient(path.container) + .GetFileClient(path.path_to_file); + try { + auto properties = file_client.GetProperties(); + if (properties.Value.IsDirectory) { + info.set_type(FileType::Directory); + } else if (internal::HasTrailingSlash(path.path_to_file)) { + // For a path with a trailing slash a hierarchical namespace may return a blob + // with that trailing slash removed. For consistency with flat namespace and + // other filesystems we chose to return NotFound. + info.set_type(FileType::NotFound); + return info; + } else { + info.set_type(FileType::File); + info.set_size(properties.Value.FileSize); + } + info.set_mtime( + std::chrono::system_clock::time_point(properties.Value.LastModified)); + return info; + } catch (const Azure::Storage::StorageException& exception) { + if (exception.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + ARROW_ASSIGN_OR_RAISE(auto hierarchical_namespace_enabled, + hierarchical_namespace_.Enabled(path.container)); + if (hierarchical_namespace_enabled) { + // If the hierarchical namespace is enabled, then the storage account will have + // explicit directories. Neither a file nor a directory was found. + info.set_type(FileType::NotFound); + return info; + } + // On flat namespace accounts there are no real directories. Directories are only + // implied by using `/` in the blob name. + Azure::Storage::Blobs::ListBlobsOptions list_blob_options; + + // If listing the prefix `path.path_to_file` with trailing slash returns at least + // one result then `path` refers to an implied directory. + auto prefix = internal::EnsureTrailingSlash(path.path_to_file); + list_blob_options.Prefix = prefix; + // We only need to know if there is at least one result, so minimise page size + // for efficiency. + list_blob_options.PageSizeHint = 1; + + try { + auto paged_list_result = + blob_service_client_->GetBlobContainerClient(path.container) + .ListBlobs(list_blob_options); + if (paged_list_result.Blobs.size() > 0) { + info.set_type(FileType::Directory); + } else { + info.set_type(FileType::NotFound); + } + return info; + } catch (const Azure::Storage::StorageException& exception) { + return internal::ExceptionToStatus( + "ListBlobs for '" + prefix + + "' failed with an unexpected Azure error. GetFileInfo is unable to " + "determine whether the path should be considered an implied directory.", + exception); + } + } + return internal::ExceptionToStatus( + "GetProperties for '" + file_client.GetUrl() + + "' failed with an unexpected " + "Azure error. GetFileInfo is unable to determine whether the path exists.", + exception); + } + } + Result> OpenInputFile(const std::string& s, AzureFileSystem* fs) { ARROW_RETURN_NOT_OK(internal::AssertNoTrailingSlash(s)); ARROW_ASSIGN_OR_RAISE(auto path, AzurePath::FromString(s)); RETURN_NOT_OK(ValidateFilePath(path)); auto blob_client = std::make_shared( - service_client_->GetBlobContainerClient(path.container) + blob_service_client_->GetBlobContainerClient(path.container) .GetBlobClient(path.path_to_file)); auto ptr = @@ -494,7 +603,7 @@ class AzureFileSystem::Impl { ARROW_ASSIGN_OR_RAISE(auto path, AzurePath::FromString(info.path())); RETURN_NOT_OK(ValidateFilePath(path)); auto blob_client = std::make_shared( - service_client_->GetBlobContainerClient(path.container) + blob_service_client_->GetBlobContainerClient(path.container) .GetBlobClient(path.path_to_file)); auto ptr = std::make_shared(blob_client, fs->io_context(), @@ -518,7 +627,8 @@ bool AzureFileSystem::Equals(const FileSystem& other) const { } Result AzureFileSystem::GetFileInfo(const std::string& path) { - return Status::NotImplemented("The Azure FileSystem is not fully implemented"); + ARROW_ASSIGN_OR_RAISE(auto p, AzurePath::FromString(path)); + return impl_->GetFileInfo(p); } Result AzureFileSystem::GetFileInfo(const FileSelector& select) { diff --git a/cpp/src/arrow/filesystem/azurefs_internal.cc b/cpp/src/arrow/filesystem/azurefs_internal.cc new file mode 100644 index 0000000000000..3e545d670cb04 --- /dev/null +++ b/cpp/src/arrow/filesystem/azurefs_internal.cc @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/filesystem/azurefs_internal.h" + +#include + +#include "arrow/result.h" + +namespace arrow::fs::internal { + +Status ExceptionToStatus(const std::string& prefix, + const Azure::Storage::StorageException& exception) { + return Status::IOError(prefix, " Azure Error: ", exception.what()); +} + +Status HierarchicalNamespaceDetector::Init( + Azure::Storage::Files::DataLake::DataLakeServiceClient* datalake_service_client) { + datalake_service_client_ = datalake_service_client; + return Status::OK(); +} + +Result HierarchicalNamespaceDetector::Enabled(const std::string& container_name) { + // Hierarchical namespace can't easily be changed after the storage account is created + // and its common across all containers in the storage account. Do nothing until we've + // checked for a cached result. + if (enabled_.has_value()) { + return enabled_.value(); + } + + // This approach is inspired by hadoop-azure + // https://github.com/apache/hadoop/blob/7c6af6a5f626d18d68b656d085cc23e4c1f7a1ef/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java#L356. + // Unfortunately `blob_service_client->GetAccountInfo()` requires significantly + // elevated permissions. + // https://learn.microsoft.com/en-us/rest/api/storageservices/get-blob-service-properties?tabs=azure-ad#authorization + auto filesystem_client = datalake_service_client_->GetFileSystemClient(container_name); + auto directory_client = filesystem_client.GetDirectoryClient("/"); + try { + directory_client.GetAccessControlList(); + enabled_ = true; + } catch (const Azure::Storage::StorageException& exception) { + // GetAccessControlList will fail on storage accounts without hierarchical + // namespace enabled. + + if (exception.StatusCode == Azure::Core::Http::HttpStatusCode::BadRequest || + exception.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict) { + // Flat namespace storage accounts with soft delete enabled return + // Conflict - This endpoint does not support BlobStorageEvents or SoftDelete + // otherwise it returns: BadRequest - This operation is only supported on a + // hierarchical namespace account. + enabled_ = false; + } else if (exception.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + // Azurite returns NotFound. + try { + filesystem_client.GetProperties(); + enabled_ = false; + } catch (const Azure::Storage::StorageException& exception) { + return ExceptionToStatus("Failed to confirm '" + filesystem_client.GetUrl() + + "' is an accessible container. Therefore the " + "hierarchical namespace check was invalid.", + exception); + } + } else { + return ExceptionToStatus( + "GetAccessControlList for '" + directory_client.GetUrl() + + "' failed with an unexpected Azure error, while checking " + "whether the storage account has hierarchical namespace enabled.", + exception); + } + } + return enabled_.value(); +} + +} // namespace arrow::fs::internal diff --git a/cpp/src/arrow/filesystem/azurefs_internal.h b/cpp/src/arrow/filesystem/azurefs_internal.h new file mode 100644 index 0000000000000..c3da96239a18f --- /dev/null +++ b/cpp/src/arrow/filesystem/azurefs_internal.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "arrow/result.h" + +namespace arrow::fs::internal { + +Status ExceptionToStatus(const std::string& prefix, + const Azure::Storage::StorageException& exception); + +class HierarchicalNamespaceDetector { + public: + Status Init( + Azure::Storage::Files::DataLake::DataLakeServiceClient* datalake_service_client); + Result Enabled(const std::string& container_name); + + private: + Azure::Storage::Files::DataLake::DataLakeServiceClient* datalake_service_client_; + std::optional enabled_; +}; + +} // namespace arrow::fs::internal diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index f57fc4d8140a0..c08a4b50b77a8 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -34,6 +34,7 @@ #include #include "arrow/filesystem/azurefs.h" +#include "arrow/filesystem/azurefs_internal.h" #include #include @@ -46,7 +47,10 @@ #include #include #include +#include +#include "arrow/filesystem/test_util.h" +#include "arrow/result.h" #include "arrow/testing/gtest_util.h" #include "arrow/testing/util.h" #include "arrow/util/io_util.h" @@ -139,34 +143,37 @@ TEST(AzureFileSystem, OptionsCompare) { EXPECT_TRUE(options.Equals(options)); } -class TestAzureFileSystem : public ::testing::Test { +class AzureFileSystemTest : public ::testing::Test { public: std::shared_ptr fs_; - std::shared_ptr service_client_; + std::unique_ptr blob_service_client_; + std::unique_ptr + datalake_service_client_; + AzureOptions options_; std::mt19937_64 generator_; std::string container_name_; + bool suite_skipped_ = false; - TestAzureFileSystem() : generator_(std::random_device()()) {} + AzureFileSystemTest() : generator_(std::random_device()()) {} - AzureOptions MakeOptions() { - const std::string& account_name = GetAzuriteEnv()->account_name(); - const std::string& account_key = GetAzuriteEnv()->account_key(); - AzureOptions options; - options.backend = AzureBackend::Azurite; - ARROW_EXPECT_OK(options.ConfigureAccountKeyCredentials(account_name, account_key)); - return options; - } + virtual Result MakeOptions() = 0; void SetUp() override { - ASSERT_THAT(GetAzuriteEnv(), NotNull()); - ASSERT_OK(GetAzuriteEnv()->status()); - - container_name_ = RandomChars(32); auto options = MakeOptions(); - service_client_ = std::make_shared( - options.account_blob_url, options.storage_credentials_provider); - ASSERT_OK_AND_ASSIGN(fs_, AzureFileSystem::Make(options)); - auto container_client = service_client_->GetBlobContainerClient(container_name_); + if (options.ok()) { + options_ = *options; + } else { + suite_skipped_ = true; + GTEST_SKIP() << options.status().message(); + } + container_name_ = RandomChars(32); + blob_service_client_ = std::make_unique( + options_.account_blob_url, options_.storage_credentials_provider); + datalake_service_client_ = + std::make_unique( + options_.account_dfs_url, options_.storage_credentials_provider); + ASSERT_OK_AND_ASSIGN(fs_, AzureFileSystem::Make(options_)); + auto container_client = blob_service_client_->GetBlobContainerClient(container_name_); container_client.CreateIfNotExists(); auto blob_client = container_client.GetBlockBlobClient(PreexistingObjectName()); @@ -175,10 +182,13 @@ class TestAzureFileSystem : public ::testing::Test { } void TearDown() override { - auto containers = service_client_->ListBlobContainers(); - for (auto container : containers.BlobContainers) { - auto container_client = service_client_->GetBlobContainerClient(container.Name); - container_client.DeleteIfExists(); + if (!suite_skipped_) { + auto containers = blob_service_client_->ListBlobContainers(); + for (auto container : containers.BlobContainers) { + auto container_client = + blob_service_client_->GetBlobContainerClient(container.Name); + container_client.DeleteIfExists(); + } } } @@ -218,15 +228,175 @@ class TestAzureFileSystem : public ::testing::Test { void UploadLines(const std::vector& lines, const char* path_to_file, int total_size) { // TODO(GH-38333): Switch to using Azure filesystem to write once its implemented. - auto blob_client = service_client_->GetBlobContainerClient(PreexistingContainerName()) - .GetBlockBlobClient(path_to_file); + auto blob_client = + blob_service_client_->GetBlobContainerClient(PreexistingContainerName()) + .GetBlockBlobClient(path_to_file); std::string all_lines = std::accumulate(lines.begin(), lines.end(), std::string("")); blob_client.UploadFrom(reinterpret_cast(all_lines.data()), total_size); } + + void RunGetFileInfoObjectWithNestedStructureTest(); + void RunGetFileInfoObjectTest(); +}; + +class AzuriteFileSystemTest : public AzureFileSystemTest { + Result MakeOptions() { + EXPECT_THAT(GetAzuriteEnv(), NotNull()); + ARROW_EXPECT_OK(GetAzuriteEnv()->status()); + AzureOptions options; + options.backend = AzureBackend::Azurite; + ARROW_EXPECT_OK(options.ConfigureAccountKeyCredentials( + GetAzuriteEnv()->account_name(), GetAzuriteEnv()->account_key())); + return options; + } +}; + +class AzureFlatNamespaceFileSystemTest : public AzureFileSystemTest { + Result MakeOptions() override { + AzureOptions options; + const auto account_key = std::getenv("AZURE_FLAT_NAMESPACE_ACCOUNT_KEY"); + const auto account_name = std::getenv("AZURE_FLAT_NAMESPACE_ACCOUNT_NAME"); + if (account_key && account_name) { + RETURN_NOT_OK(options.ConfigureAccountKeyCredentials(account_name, account_key)); + return options; + } + return Status::Cancelled( + "Connection details not provided for a real flat namespace " + "account."); + } }; -TEST_F(TestAzureFileSystem, OpenInputStreamString) { +class AzureHierarchicalNamespaceFileSystemTest : public AzureFileSystemTest { + Result MakeOptions() override { + AzureOptions options; + const auto account_key = std::getenv("AZURE_HIERARCHICAL_NAMESPACE_ACCOUNT_KEY"); + const auto account_name = std::getenv("AZURE_HIERARCHICAL_NAMESPACE_ACCOUNT_NAME"); + if (account_key && account_name) { + RETURN_NOT_OK(options.ConfigureAccountKeyCredentials(account_name, account_key)); + return options; + } + return Status::Cancelled( + "Connection details not provided for a real hierarchical namespace " + "account."); + } +}; + +TEST_F(AzureFlatNamespaceFileSystemTest, DetectHierarchicalNamespace) { + auto hierarchical_namespace = internal::HierarchicalNamespaceDetector(); + ASSERT_OK(hierarchical_namespace.Init(datalake_service_client_.get())); + ASSERT_OK_AND_EQ(false, hierarchical_namespace.Enabled(PreexistingContainerName())); +} + +TEST_F(AzureHierarchicalNamespaceFileSystemTest, DetectHierarchicalNamespace) { + auto hierarchical_namespace = internal::HierarchicalNamespaceDetector(); + ASSERT_OK(hierarchical_namespace.Init(datalake_service_client_.get())); + ASSERT_OK_AND_EQ(true, hierarchical_namespace.Enabled(PreexistingContainerName())); +} + +TEST_F(AzuriteFileSystemTest, DetectHierarchicalNamespace) { + auto hierarchical_namespace = internal::HierarchicalNamespaceDetector(); + ASSERT_OK(hierarchical_namespace.Init(datalake_service_client_.get())); + ASSERT_OK_AND_EQ(false, hierarchical_namespace.Enabled(PreexistingContainerName())); +} + +TEST_F(AzuriteFileSystemTest, DetectHierarchicalNamespaceFailsWithMissingContainer) { + auto hierarchical_namespace = internal::HierarchicalNamespaceDetector(); + ASSERT_OK(hierarchical_namespace.Init(datalake_service_client_.get())); + ASSERT_NOT_OK(hierarchical_namespace.Enabled("non-existent-container")); +} + +TEST_F(AzuriteFileSystemTest, GetFileInfoAccount) { + AssertFileInfo(fs_.get(), "", FileType::Directory); + + // URI + ASSERT_RAISES(Invalid, fs_->GetFileInfo("abfs://")); +} + +TEST_F(AzuriteFileSystemTest, GetFileInfoContainer) { + AssertFileInfo(fs_.get(), PreexistingContainerName(), FileType::Directory); + + AssertFileInfo(fs_.get(), "non-existent-container", FileType::NotFound); + + // URI + ASSERT_RAISES(Invalid, fs_->GetFileInfo("abfs://" + PreexistingContainerName())); +} + +void AzureFileSystemTest::RunGetFileInfoObjectWithNestedStructureTest() { + // Adds detailed tests to handle cases of different edge cases + // with directory naming conventions (e.g. with and without slashes). + constexpr auto kObjectName = "test-object-dir/some_other_dir/another_dir/foo"; + // TODO(GH-38333): Switch to using Azure filesystem to write once its implemented. + blob_service_client_->GetBlobContainerClient(PreexistingContainerName()) + .GetBlockBlobClient(kObjectName) + .UploadFrom(reinterpret_cast(kLoremIpsum), strlen(kLoremIpsum)); + + // 0 is immediately after "/" lexicographically, ensure that this doesn't + // cause unexpected issues. + // TODO(GH-38333): Switch to using Azure filesystem to write once its implemented. + blob_service_client_->GetBlobContainerClient(PreexistingContainerName()) + .GetBlockBlobClient("test-object-dir/some_other_dir0") + .UploadFrom(reinterpret_cast(kLoremIpsum), strlen(kLoremIpsum)); + + blob_service_client_->GetBlobContainerClient(PreexistingContainerName()) + .GetBlockBlobClient(std::string(kObjectName) + "0") + .UploadFrom(reinterpret_cast(kLoremIpsum), strlen(kLoremIpsum)); + + AssertFileInfo(fs_.get(), PreexistingContainerPath() + kObjectName, FileType::File); + AssertFileInfo(fs_.get(), PreexistingContainerPath() + kObjectName + "/", + FileType::NotFound); + AssertFileInfo(fs_.get(), PreexistingContainerPath() + "test-object-dir", + FileType::Directory); + AssertFileInfo(fs_.get(), PreexistingContainerPath() + "test-object-dir/", + FileType::Directory); + AssertFileInfo(fs_.get(), PreexistingContainerPath() + "test-object-dir/some_other_dir", + FileType::Directory); + AssertFileInfo(fs_.get(), + PreexistingContainerPath() + "test-object-dir/some_other_dir/", + FileType::Directory); + + AssertFileInfo(fs_.get(), PreexistingContainerPath() + "test-object-di", + FileType::NotFound); + AssertFileInfo(fs_.get(), PreexistingContainerPath() + "test-object-dir/some_other_di", + FileType::NotFound); +} + +TEST_F(AzuriteFileSystemTest, GetFileInfoObjectWithNestedStructure) { + RunGetFileInfoObjectWithNestedStructureTest(); +} + +TEST_F(AzureHierarchicalNamespaceFileSystemTest, GetFileInfoObjectWithNestedStructure) { + RunGetFileInfoObjectWithNestedStructureTest(); + datalake_service_client_->GetFileSystemClient(PreexistingContainerName()) + .GetDirectoryClient("test-empty-object-dir") + .Create(); + + AssertFileInfo(fs_.get(), PreexistingContainerPath() + "test-empty-object-dir", + FileType::Directory); +} + +void AzureFileSystemTest::RunGetFileInfoObjectTest() { + auto object_properties = + blob_service_client_->GetBlobContainerClient(PreexistingContainerName()) + .GetBlobClient(PreexistingObjectName()) + .GetProperties() + .Value; + + AssertFileInfo(fs_.get(), PreexistingObjectPath(), FileType::File, + std::chrono::system_clock::time_point(object_properties.LastModified), + static_cast(object_properties.BlobSize)); + + // URI + ASSERT_RAISES(Invalid, fs_->GetFileInfo("abfs://" + PreexistingObjectName())); +} + +TEST_F(AzuriteFileSystemTest, GetFileInfoObject) { RunGetFileInfoObjectTest(); } + +TEST_F(AzureHierarchicalNamespaceFileSystemTest, GetFileInfoObject) { + RunGetFileInfoObjectTest(); +} + +TEST_F(AzuriteFileSystemTest, OpenInputStreamString) { std::shared_ptr stream; ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream(PreexistingObjectPath())); @@ -234,7 +404,7 @@ TEST_F(TestAzureFileSystem, OpenInputStreamString) { EXPECT_EQ(buffer->ToString(), kLoremIpsum); } -TEST_F(TestAzureFileSystem, OpenInputStreamStringBuffers) { +TEST_F(AzuriteFileSystemTest, OpenInputStreamStringBuffers) { std::shared_ptr stream; ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream(PreexistingObjectPath())); @@ -248,10 +418,8 @@ TEST_F(TestAzureFileSystem, OpenInputStreamStringBuffers) { EXPECT_EQ(contents, kLoremIpsum); } -TEST_F(TestAzureFileSystem, OpenInputStreamInfo) { - // TODO(GH-38335): When implemented use ASSERT_OK_AND_ASSIGN(info, - // fs->GetFileInfo(PreexistingObjectPath())); - arrow::fs::FileInfo info(PreexistingObjectPath(), FileType::File); +TEST_F(AzuriteFileSystemTest, OpenInputStreamInfo) { + ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(PreexistingObjectPath())); std::shared_ptr stream; ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream(info)); @@ -260,10 +428,10 @@ TEST_F(TestAzureFileSystem, OpenInputStreamInfo) { EXPECT_EQ(buffer->ToString(), kLoremIpsum); } -TEST_F(TestAzureFileSystem, OpenInputStreamEmpty) { +TEST_F(AzuriteFileSystemTest, OpenInputStreamEmpty) { const auto path_to_file = "empty-object.txt"; const auto path = PreexistingContainerPath() + path_to_file; - service_client_->GetBlobContainerClient(PreexistingContainerName()) + blob_service_client_->GetBlobContainerClient(PreexistingContainerName()) .GetBlockBlobClient(path_to_file) .UploadFrom(nullptr, 0); @@ -274,27 +442,23 @@ TEST_F(TestAzureFileSystem, OpenInputStreamEmpty) { EXPECT_EQ(size, 0); } -TEST_F(TestAzureFileSystem, OpenInputStreamNotFound) { +TEST_F(AzuriteFileSystemTest, OpenInputStreamNotFound) { ASSERT_RAISES(IOError, fs_->OpenInputStream(NotFoundObjectPath())); } -TEST_F(TestAzureFileSystem, OpenInputStreamInfoInvalid) { - // TODO(GH-38335): When implemented use ASSERT_OK_AND_ASSIGN(info, - // fs->GetFileInfo(PreexistingBucketPath())); - arrow::fs::FileInfo info(PreexistingContainerPath(), FileType::Directory); +TEST_F(AzuriteFileSystemTest, OpenInputStreamInfoInvalid) { + ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(PreexistingContainerPath())); ASSERT_RAISES(IOError, fs_->OpenInputStream(info)); - // TODO(GH-38335): When implemented use ASSERT_OK_AND_ASSIGN(info, - // fs->GetFileInfo(NotFoundObjectPath())); - arrow::fs::FileInfo info2(PreexistingContainerPath(), FileType::NotFound); + ASSERT_OK_AND_ASSIGN(auto info2, fs_->GetFileInfo(NotFoundObjectPath())); ASSERT_RAISES(IOError, fs_->OpenInputStream(info2)); } -TEST_F(TestAzureFileSystem, OpenInputStreamUri) { +TEST_F(AzuriteFileSystemTest, OpenInputStreamUri) { ASSERT_RAISES(Invalid, fs_->OpenInputStream("abfss://" + PreexistingObjectPath())); } -TEST_F(TestAzureFileSystem, OpenInputStreamTrailingSlash) { +TEST_F(AzuriteFileSystemTest, OpenInputStreamTrailingSlash) { ASSERT_RAISES(IOError, fs_->OpenInputStream(PreexistingObjectPath() + '/')); } @@ -321,7 +485,8 @@ std::shared_ptr NormalizerKeyValueMetadata( value = "2023-10-31T08:15:20Z"; } } else if (key == "ETag") { - if (internal::StartsWith(value, "\"") && internal::EndsWith(value, "\"")) { + if (arrow::internal::StartsWith(value, "\"") && + arrow::internal::EndsWith(value, "\"")) { // Valid value value = "\"ETagValue\""; } @@ -332,7 +497,7 @@ std::shared_ptr NormalizerKeyValueMetadata( } }; // namespace -TEST_F(TestAzureFileSystem, OpenInputStreamReadMetadata) { +TEST_F(AzuriteFileSystemTest, OpenInputStreamReadMetadata) { std::shared_ptr stream; ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream(PreexistingObjectPath())); @@ -362,7 +527,7 @@ TEST_F(TestAzureFileSystem, OpenInputStreamReadMetadata) { NormalizerKeyValueMetadata(actual)->ToString()); } -TEST_F(TestAzureFileSystem, OpenInputStreamClosed) { +TEST_F(AzuriteFileSystemTest, OpenInputStreamClosed) { ASSERT_OK_AND_ASSIGN(auto stream, fs_->OpenInputStream(PreexistingObjectPath())); ASSERT_OK(stream->Close()); std::array buffer{}; @@ -371,7 +536,7 @@ TEST_F(TestAzureFileSystem, OpenInputStreamClosed) { ASSERT_RAISES(Invalid, stream->Tell()); } -TEST_F(TestAzureFileSystem, OpenInputFileMixedReadVsReadAt) { +TEST_F(AzuriteFileSystemTest, OpenInputFileMixedReadVsReadAt) { // Create a file large enough to make the random access tests non-trivial. auto constexpr kLineWidth = 100; auto constexpr kLineCount = 4096; @@ -417,7 +582,7 @@ TEST_F(TestAzureFileSystem, OpenInputFileMixedReadVsReadAt) { } } -TEST_F(TestAzureFileSystem, OpenInputFileRandomSeek) { +TEST_F(AzuriteFileSystemTest, OpenInputFileRandomSeek) { // Create a file large enough to make the random access tests non-trivial. auto constexpr kLineWidth = 100; auto constexpr kLineCount = 4096; @@ -445,14 +610,15 @@ TEST_F(TestAzureFileSystem, OpenInputFileRandomSeek) { } } -TEST_F(TestAzureFileSystem, OpenInputFileIoContext) { +TEST_F(AzuriteFileSystemTest, OpenInputFileIoContext) { // Create a test file. const auto path_to_file = "OpenInputFileIoContext/object-name"; const auto path = PreexistingContainerPath() + path_to_file; const std::string contents = "The quick brown fox jumps over the lazy dog"; - auto blob_client = service_client_->GetBlobContainerClient(PreexistingContainerName()) - .GetBlockBlobClient(path_to_file); + auto blob_client = + blob_service_client_->GetBlobContainerClient(PreexistingContainerName()) + .GetBlockBlobClient(path_to_file); blob_client.UploadFrom(reinterpret_cast(contents.data()), contents.length()); @@ -461,10 +627,8 @@ TEST_F(TestAzureFileSystem, OpenInputFileIoContext) { EXPECT_EQ(fs_->io_context().external_id(), file->io_context().external_id()); } -TEST_F(TestAzureFileSystem, OpenInputFileInfo) { - // TODO(GH-38335): When implemented use ASSERT_OK_AND_ASSIGN(info, - // fs->GetFileInfo(PreexistingObjectPath())); - arrow::fs::FileInfo info(PreexistingObjectPath(), FileType::File); +TEST_F(AzuriteFileSystemTest, OpenInputFileInfo) { + ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(PreexistingObjectPath())); std::shared_ptr file; ASSERT_OK_AND_ASSIGN(file, fs_->OpenInputFile(info)); @@ -478,23 +642,19 @@ TEST_F(TestAzureFileSystem, OpenInputFileInfo) { EXPECT_EQ(std::string(buffer.data(), size), expected); } -TEST_F(TestAzureFileSystem, OpenInputFileNotFound) { +TEST_F(AzuriteFileSystemTest, OpenInputFileNotFound) { ASSERT_RAISES(IOError, fs_->OpenInputFile(NotFoundObjectPath())); } -TEST_F(TestAzureFileSystem, OpenInputFileInfoInvalid) { - // TODO(GH-38335): When implemented use ASSERT_OK_AND_ASSIGN(info, - // fs->GetFileInfo(PreexistingContainerPath())); - arrow::fs::FileInfo info(PreexistingContainerPath(), FileType::File); +TEST_F(AzuriteFileSystemTest, OpenInputFileInfoInvalid) { + ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(PreexistingContainerPath())); ASSERT_RAISES(IOError, fs_->OpenInputFile(info)); - // TODO(GH-38335): When implemented use ASSERT_OK_AND_ASSIGN(info, - // fs->GetFileInfo(NotFoundObjectPath())); - arrow::fs::FileInfo info2(NotFoundObjectPath(), FileType::NotFound); + ASSERT_OK_AND_ASSIGN(auto info2, fs_->GetFileInfo(NotFoundObjectPath())); ASSERT_RAISES(IOError, fs_->OpenInputFile(info2)); } -TEST_F(TestAzureFileSystem, OpenInputFileClosed) { +TEST_F(AzuriteFileSystemTest, OpenInputFileClosed) { ASSERT_OK_AND_ASSIGN(auto stream, fs_->OpenInputFile(PreexistingObjectPath())); ASSERT_OK(stream->Close()); std::array buffer{}; diff --git a/cpp/src/arrow/filesystem/path_util.cc b/cpp/src/arrow/filesystem/path_util.cc index 90af3c66ff8d4..46ea436a9f31a 100644 --- a/cpp/src/arrow/filesystem/path_util.cc +++ b/cpp/src/arrow/filesystem/path_util.cc @@ -191,18 +191,15 @@ std::string_view RemoveLeadingSlash(std::string_view key) { } Status AssertNoTrailingSlash(std::string_view key) { - if (key.back() == '/') { + if (HasTrailingSlash(key)) { return NotAFile(key); } return Status::OK(); } -bool HasLeadingSlash(std::string_view key) { - if (key.front() != '/') { - return false; - } - return true; -} +bool HasTrailingSlash(std::string_view key) { return key.back() == '/'; } + +bool HasLeadingSlash(std::string_view key) { return key.front() == '/'; } Result MakeAbstractPathRelative(const std::string& base, const std::string& path) { diff --git a/cpp/src/arrow/filesystem/path_util.h b/cpp/src/arrow/filesystem/path_util.h index 13a74b7fa12c8..2c8c123e779f4 100644 --- a/cpp/src/arrow/filesystem/path_util.h +++ b/cpp/src/arrow/filesystem/path_util.h @@ -94,6 +94,9 @@ std::string_view RemoveTrailingSlash(std::string_view s, bool preserve_root = fa ARROW_EXPORT Status AssertNoTrailingSlash(std::string_view s); +ARROW_EXPORT +bool HasTrailingSlash(std::string_view s); + ARROW_EXPORT bool HasLeadingSlash(std::string_view s); From 9569be83adb917c8ac9e024e3e5d7bc78829608e Mon Sep 17 00:00:00 2001 From: Dane Pitkin <48041712+danepitkin@users.noreply.github.com> Date: Wed, 8 Nov 2023 23:39:33 -0500 Subject: [PATCH 24/57] GH-38439: [Java][CI] Use Eclipse Temurin for all Java CI linux jobs (#38440) ### Rationale for this change Use the same JDK vendor for all linux CI jobs and upgrade maven to latest version. ### What changes are included in this PR? * Eclipse Temurin image for Ubuntu is used for JDK 8 and 11 * maven is upgraded to 3.9.5 for all Ubuntu jobs ### Are these changes tested? This will be tested in CI. ### Are there any user-facing changes? No, just a developer change when using docker/archery. The image name is now `java` instead of `eclipse-java` and `debian-java`. * Closes: #38439 Authored-by: Dane Pitkin Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/java.yml | 23 ++++---------------- docker-compose.yml | 27 ++++++------------------ docs/source/developers/java/building.rst | 4 ++-- 3 files changed, 13 insertions(+), 41 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 444bec2c73b61..69adc184b7fe7 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -49,8 +49,8 @@ env: jobs: - debian: - name: ${{ matrix.title }} + ubuntu: + name: AMD64 Ubuntu 22.04 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 @@ -58,23 +58,8 @@ jobs: fail-fast: false matrix: jdk: [8, 11, 17, 21] - include: - - jdk: 8 - title: AMD64 Debian 9 Java JDK 8 Maven 3.5.4 - maven: 3.5.4 - image: debian-java - - jdk: 11 - title: AMD64 Debian 9 Java JDK 11 Maven 3.6.2 - maven: 3.6.2 - image: debian-java - - jdk: 17 - title: AMD64 Ubuntu 22.04 Java JDK 17 Maven 3.9.4 - maven: 3.9.4 - image: eclipse-java - - jdk: 21 - title: AMD64 Ubuntu 22.04 Java JDK 21 Maven 3.9.4 - maven: 3.9.4 - image: eclipse-java + maven: [3.9.5] + image: [java] env: JDK: ${{ matrix.jdk }} MAVEN: ${{ matrix.maven }} diff --git a/docker-compose.yml b/docker-compose.yml index bbe4e65c10c43..e2c993ee9ea41 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -135,11 +135,10 @@ x-hierarchy: - debian-go: - debian-go-cgo - debian-go-cgo-python - - debian-java - debian-js - - eclipse-java - fedora-cpp: - fedora-python + - java - python-sdist - ubuntu-cpp: - ubuntu-cpp-static @@ -1674,14 +1673,14 @@ services: ################################ Java ####################################### - debian-java: + java: # Usage: - # docker-compose build debian-java - # docker-compose run debian-java + # docker-compose build java + # docker-compose run java # Parameters: - # MAVEN: 3.5.4, 3.6.2 - # JDK: 8, 11 - image: ${ARCH}/maven:${MAVEN}-jdk-${JDK} + # MAVEN: 3.9.5 + # JDK: 8, 11, 17, 21 + image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} shm_size: *shm-size volumes: &java-volumes - .:/arrow:delegated @@ -1691,18 +1690,6 @@ services: /arrow/ci/scripts/java_build.sh /arrow /build && /arrow/ci/scripts/java_test.sh /arrow /build" - eclipse-java: - # Usage: - # docker-compose build eclipse-java - # docker-compose run eclipse-java - # Parameters: - # MAVEN: 3.9.4 - # JDK: 17, 21 - image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} - shm_size: *shm-size - volumes: *java-volumes - command: *java-command - ############################## Integration ################################## conda-integration: diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index 061c616d4b971..8b2a504631fdb 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -76,7 +76,7 @@ Docker compose $ cd arrow/java $ export JAVA_HOME= $ java --version - $ docker-compose run debian-java + $ docker-compose run java Archery ~~~~~~~ @@ -86,7 +86,7 @@ Archery $ cd arrow/java $ export JAVA_HOME= $ java --version - $ archery docker run debian-java + $ archery docker run java Building JNI Libraries (\*.dylib / \*.so / \*.dll) -------------------------------------------------- From 80503a6a1a024bbac32213b597a2b0e46bfcf390 Mon Sep 17 00:00:00 2001 From: Nick Hughes Date: Thu, 9 Nov 2023 07:39:22 -0800 Subject: [PATCH 25/57] GH-34017: [Python][FlightRPC][Doc] Fix `read_chunk` docstring for FlightStreamReader and MetadataRecordBatchReader (#38641) ### Rationale for this change The docs for `FlightStreamReader` and `MetadataRecordBatchReader` currently list an incorrect return type for the `read_chunk` method - see #34017. NB: this issue was partially addressed by #35583. ### What changes are included in this PR? Simple docstring update for `read_chunk`. ### Are these changes tested? No (v minor docstring only change) ### Are there any user-facing changes? Yes * Closes: #34017 Authored-by: Nick Hughes Signed-off-by: David Li --- python/pyarrow/_flight.pyx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/pyarrow/_flight.pyx b/python/pyarrow/_flight.pyx index 79aa24e4ce8e3..8fe9465a13d9c 100644 --- a/python/pyarrow/_flight.pyx +++ b/python/pyarrow/_flight.pyx @@ -1014,11 +1014,8 @@ cdef class _MetadataRecordBatchReader(_Weakrefable, _ReadPandasMixin): Returns ------- - data : FlightStreamChunk + chunk : FlightStreamChunk The next FlightStreamChunk in the stream. - app_metadata : Buffer or None - Application-specific metadata for the batch as defined by - Flight. Raises ------ From 1f71014435fd56c915aebe0a9ac982f6e8de6f94 Mon Sep 17 00:00:00 2001 From: Dane Pitkin <48041712+danepitkin@users.noreply.github.com> Date: Thu, 9 Nov 2023 10:53:55 -0500 Subject: [PATCH 26/57] GH-38648: [Java] Regenerate Flatbuffers (#38650) ### Rationale for this change Regenerate the Flatbuffers to include new formats. Flatbuffers are always forwards/backwards compatible as long as the code is generated from the same `flatc` compiler version. ### What changes are included in this PR? * Flatbuffers regenerated with `flatc` v1.12.0 ### Are these changes tested? Yes, via unit tests. ### Are there any user-facing changes? Yes, RecordBatch.java was modified. * Closes: #38648 Authored-by: Dane Pitkin Signed-off-by: David Li --- .../org/apache/arrow/flatbuf/BinaryView.java | 57 +++++++ .../apache/arrow/flatbuf/BodyCompression.java | 3 +- .../java/org/apache/arrow/flatbuf/Date.java | 4 +- .../apache/arrow/flatbuf/LargeListView.java | 52 ++++++ .../org/apache/arrow/flatbuf/ListView.java | 53 +++++++ .../org/apache/arrow/flatbuf/RecordBatch.java | 33 +++- .../apache/arrow/flatbuf/RunEndEncoded.java | 55 +++++++ .../java/org/apache/arrow/flatbuf/Time.java | 17 +- .../org/apache/arrow/flatbuf/Timestamp.java | 148 +++++++++++++----- .../java/org/apache/arrow/flatbuf/Type.java | 7 +- .../org/apache/arrow/flatbuf/Utf8View.java | 57 +++++++ 11 files changed, 435 insertions(+), 51 deletions(-) create mode 100644 java/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java create mode 100644 java/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java create mode 100644 java/format/src/main/java/org/apache/arrow/flatbuf/ListView.java create mode 100644 java/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java create mode 100644 java/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java b/java/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java new file mode 100644 index 0000000000000..56a8d329532c0 --- /dev/null +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// automatically generated by the FlatBuffers compiler, do not modify + +package org.apache.arrow.flatbuf; + +import java.nio.*; +import java.lang.*; +import java.util.*; +import com.google.flatbuffers.*; + +@SuppressWarnings("unused") +/** + * Logically the same as Binary, but the internal representation uses a view + * struct that contains the string length and either the string's entire data + * inline (for small strings) or an inlined prefix, an index of another buffer, + * and an offset pointing to a slice in that buffer (for non-small strings). + * + * Since it uses a variable number of data buffers, each Field with this type + * must have a corresponding entry in `variadicBufferCounts`. + */ +public final class BinaryView extends Table { + public static void ValidateVersion() { Constants.FLATBUFFERS_1_12_0(); } + public static BinaryView getRootAsBinaryView(ByteBuffer _bb) { return getRootAsBinaryView(_bb, new BinaryView()); } + public static BinaryView getRootAsBinaryView(ByteBuffer _bb, BinaryView obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } + public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } + public BinaryView __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + + public static void startBinaryView(FlatBufferBuilder builder) { builder.startTable(0); } + public static int endBinaryView(FlatBufferBuilder builder) { + int o = builder.endTable(); + return o; + } + + public static final class Vector extends BaseVector { + public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { __reset(_vector, _element_size, _bb); return this; } + + public BinaryView get(int j) { return get(new BinaryView(), j); } + public BinaryView get(BinaryView obj, int j) { return obj.__assign(__indirect(__element(j), bb), bb); } + } +} + diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java b/java/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java index 650454eb154b4..ed8ce0939a044 100644 --- a/java/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java @@ -37,7 +37,8 @@ public final class BodyCompression extends Table { public BodyCompression __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } /** - * Compressor library + * Compressor library. + * For LZ4_FRAME, each compressed buffer must consist of a single frame. */ public byte codec() { int o = __offset(4); return o != 0 ? bb.get(o + bb_pos) : 0; } /** diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/Date.java b/java/format/src/main/java/org/apache/arrow/flatbuf/Date.java index b2fcc9e39e38d..ac6e389835a43 100644 --- a/java/format/src/main/java/org/apache/arrow/flatbuf/Date.java +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/Date.java @@ -25,8 +25,8 @@ @SuppressWarnings("unused") /** - * Date is either a 32-bit or 64-bit type representing elapsed time since UNIX - * epoch (1970-01-01), stored in either of two units: + * Date is either a 32-bit or 64-bit signed integer type representing an + * elapsed time since UNIX epoch (1970-01-01), stored in either of two units: * * * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no * leap seconds), where the values are evenly divisible by 86400000 diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java b/java/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java new file mode 100644 index 0000000000000..08c31c23a943f --- /dev/null +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// automatically generated by the FlatBuffers compiler, do not modify + +package org.apache.arrow.flatbuf; + +import java.nio.*; +import java.lang.*; +import java.util.*; +import com.google.flatbuffers.*; + +@SuppressWarnings("unused") +/** + * Same as ListView, but with 64-bit offsets and sizes, allowing to represent + * extremely large data values. + */ +public final class LargeListView extends Table { + public static void ValidateVersion() { Constants.FLATBUFFERS_1_12_0(); } + public static LargeListView getRootAsLargeListView(ByteBuffer _bb) { return getRootAsLargeListView(_bb, new LargeListView()); } + public static LargeListView getRootAsLargeListView(ByteBuffer _bb, LargeListView obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } + public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } + public LargeListView __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + + public static void startLargeListView(FlatBufferBuilder builder) { builder.startTable(0); } + public static int endLargeListView(FlatBufferBuilder builder) { + int o = builder.endTable(); + return o; + } + + public static final class Vector extends BaseVector { + public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { __reset(_vector, _element_size, _bb); return this; } + + public LargeListView get(int j) { return get(new LargeListView(), j); } + public LargeListView get(LargeListView obj, int j) { return obj.__assign(__indirect(__element(j), bb), bb); } + } +} + diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/ListView.java b/java/format/src/main/java/org/apache/arrow/flatbuf/ListView.java new file mode 100644 index 0000000000000..2c9ad4c13d884 --- /dev/null +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/ListView.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// automatically generated by the FlatBuffers compiler, do not modify + +package org.apache.arrow.flatbuf; + +import java.nio.*; +import java.lang.*; +import java.util.*; +import com.google.flatbuffers.*; + +@SuppressWarnings("unused") +/** + * Represents the same logical types that List can, but contains offsets and + * sizes allowing for writes in any order and sharing of child values among + * list values. + */ +public final class ListView extends Table { + public static void ValidateVersion() { Constants.FLATBUFFERS_1_12_0(); } + public static ListView getRootAsListView(ByteBuffer _bb) { return getRootAsListView(_bb, new ListView()); } + public static ListView getRootAsListView(ByteBuffer _bb, ListView obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } + public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } + public ListView __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + + public static void startListView(FlatBufferBuilder builder) { builder.startTable(0); } + public static int endListView(FlatBufferBuilder builder) { + int o = builder.endTable(); + return o; + } + + public static final class Vector extends BaseVector { + public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { __reset(_vector, _element_size, _bb); return this; } + + public ListView get(int j) { return get(new ListView(), j); } + public ListView get(ListView obj, int j) { return obj.__assign(__indirect(__element(j), bb), bb); } + } +} + diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java b/java/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java index eb814e07dcceb..ce907ee0fdcda 100644 --- a/java/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java @@ -67,27 +67,54 @@ public final class RecordBatch extends Table { */ public org.apache.arrow.flatbuf.BodyCompression compression() { return compression(new org.apache.arrow.flatbuf.BodyCompression()); } public org.apache.arrow.flatbuf.BodyCompression compression(org.apache.arrow.flatbuf.BodyCompression obj) { int o = __offset(10); return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; } + /** + * Some types such as Utf8View are represented using a variable number of buffers. + * For each such Field in the pre-ordered flattened logical schema, there will be + * an entry in variadicBufferCounts to indicate the number of number of variadic + * buffers which belong to that Field in the current RecordBatch. + * + * For example, the schema + * col1: Struct + * col2: Utf8View + * contains two Fields with variadic buffers so variadicBufferCounts will have + * two entries, the first counting the variadic buffers of `col1.beta` and the + * second counting `col2`'s. + * + * This field may be omitted if and only if the schema contains no Fields with + * a variable number of buffers, such as BinaryView and Utf8View. + */ + public long variadicBufferCounts(int j) { int o = __offset(12); return o != 0 ? bb.getLong(__vector(o) + j * 8) : 0; } + public int variadicBufferCountsLength() { int o = __offset(12); return o != 0 ? __vector_len(o) : 0; } + public LongVector variadicBufferCountsVector() { return variadicBufferCountsVector(new LongVector()); } + public LongVector variadicBufferCountsVector(LongVector obj) { int o = __offset(12); return o != 0 ? obj.__assign(__vector(o), bb) : null; } + public ByteBuffer variadicBufferCountsAsByteBuffer() { return __vector_as_bytebuffer(12, 8); } + public ByteBuffer variadicBufferCountsInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 12, 8); } public static int createRecordBatch(FlatBufferBuilder builder, long length, int nodesOffset, int buffersOffset, - int compressionOffset) { - builder.startTable(4); + int compressionOffset, + int variadicBufferCountsOffset) { + builder.startTable(5); RecordBatch.addLength(builder, length); + RecordBatch.addVariadicBufferCounts(builder, variadicBufferCountsOffset); RecordBatch.addCompression(builder, compressionOffset); RecordBatch.addBuffers(builder, buffersOffset); RecordBatch.addNodes(builder, nodesOffset); return RecordBatch.endRecordBatch(builder); } - public static void startRecordBatch(FlatBufferBuilder builder) { builder.startTable(4); } + public static void startRecordBatch(FlatBufferBuilder builder) { builder.startTable(5); } public static void addLength(FlatBufferBuilder builder, long length) { builder.addLong(0, length, 0L); } public static void addNodes(FlatBufferBuilder builder, int nodesOffset) { builder.addOffset(1, nodesOffset, 0); } public static void startNodesVector(FlatBufferBuilder builder, int numElems) { builder.startVector(16, numElems, 8); } public static void addBuffers(FlatBufferBuilder builder, int buffersOffset) { builder.addOffset(2, buffersOffset, 0); } public static void startBuffersVector(FlatBufferBuilder builder, int numElems) { builder.startVector(16, numElems, 8); } public static void addCompression(FlatBufferBuilder builder, int compressionOffset) { builder.addOffset(3, compressionOffset, 0); } + public static void addVariadicBufferCounts(FlatBufferBuilder builder, int variadicBufferCountsOffset) { builder.addOffset(4, variadicBufferCountsOffset, 0); } + public static int createVariadicBufferCountsVector(FlatBufferBuilder builder, long[] data) { builder.startVector(8, data.length, 8); for (int i = data.length - 1; i >= 0; i--) builder.addLong(data[i]); return builder.endVector(); } + public static void startVariadicBufferCountsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(8, numElems, 8); } public static int endRecordBatch(FlatBufferBuilder builder) { int o = builder.endTable(); return o; diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java b/java/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java new file mode 100644 index 0000000000000..d48733ef0c826 --- /dev/null +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// automatically generated by the FlatBuffers compiler, do not modify + +package org.apache.arrow.flatbuf; + +import java.nio.*; +import java.lang.*; +import java.util.*; +import com.google.flatbuffers.*; + +@SuppressWarnings("unused") +/** + * Contains two child arrays, run_ends and values. + * The run_ends child array must be a 16/32/64-bit integer array + * which encodes the indices at which the run with the value in + * each corresponding index in the values child array ends. + * Like list/struct types, the value array can be of any type. + */ +public final class RunEndEncoded extends Table { + public static void ValidateVersion() { Constants.FLATBUFFERS_1_12_0(); } + public static RunEndEncoded getRootAsRunEndEncoded(ByteBuffer _bb) { return getRootAsRunEndEncoded(_bb, new RunEndEncoded()); } + public static RunEndEncoded getRootAsRunEndEncoded(ByteBuffer _bb, RunEndEncoded obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } + public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } + public RunEndEncoded __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + + public static void startRunEndEncoded(FlatBufferBuilder builder) { builder.startTable(0); } + public static int endRunEndEncoded(FlatBufferBuilder builder) { + int o = builder.endTable(); + return o; + } + + public static final class Vector extends BaseVector { + public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { __reset(_vector, _element_size, _bb); return this; } + + public RunEndEncoded get(int j) { return get(new RunEndEncoded(), j); } + public RunEndEncoded get(RunEndEncoded obj, int j) { return obj.__assign(__indirect(__element(j), bb), bb); } + } +} + diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/Time.java b/java/format/src/main/java/org/apache/arrow/flatbuf/Time.java index 596d403a3eae9..9acc3fc7a5ea1 100644 --- a/java/format/src/main/java/org/apache/arrow/flatbuf/Time.java +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/Time.java @@ -25,9 +25,20 @@ @SuppressWarnings("unused") /** - * Time type. The physical storage type depends on the unit - * - SECOND and MILLISECOND: 32 bits - * - MICROSECOND and NANOSECOND: 64 bits + * Time is either a 32-bit or 64-bit signed integer type representing an + * elapsed time since midnight, stored in either of four units: seconds, + * milliseconds, microseconds or nanoseconds. + * + * The integer `bitWidth` depends on the `unit` and must be one of the following: + * * SECOND and MILLISECOND: 32 bits + * * MICROSECOND and NANOSECOND: 64 bits + * + * The allowed values are between 0 (inclusive) and 86400 (=24*60*60) seconds + * (exclusive), adjusted for the time unit (for example, up to 86400000 + * exclusive for the MILLISECOND unit). + * This definition doesn't allow for leap seconds. Time values from + * measurements with leap seconds will need to be corrected when ingesting + * into Arrow (for example by replacing the value 86400 with 86399). */ public final class Time extends Table { public static void ValidateVersion() { Constants.FLATBUFFERS_1_12_0(); } diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java b/java/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java index 041452607c9e1..fe0c6aaea24fa 100644 --- a/java/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java @@ -25,37 +25,111 @@ @SuppressWarnings("unused") /** - * Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding - * leap seconds, as a 64-bit integer. Note that UNIX time does not include - * leap seconds. + * Timestamp is a 64-bit signed integer representing an elapsed time since a + * fixed epoch, stored in either of four units: seconds, milliseconds, + * microseconds or nanoseconds, and is optionally annotated with a timezone. + * + * Timestamp values do not include any leap seconds (in other words, all + * days are considered 86400 seconds long). + * + * Timestamps with a non-empty timezone + * ------------------------------------ + * + * If a Timestamp column has a non-empty timezone value, its epoch is + * 1970-01-01 00:00:00 (January 1st 1970, midnight) in the *UTC* timezone + * (the Unix epoch), regardless of the Timestamp's own timezone. + * + * Therefore, timestamp values with a non-empty timezone correspond to + * physical points in time together with some additional information about + * how the data was obtained and/or how to display it (the timezone). + * + * For example, the timestamp value 0 with the timezone string "Europe/Paris" + * corresponds to "January 1st 1970, 00h00" in the UTC timezone, but the + * application may prefer to display it as "January 1st 1970, 01h00" in + * the Europe/Paris timezone (which is the same physical point in time). + * + * One consequence is that timestamp values with a non-empty timezone + * can be compared and ordered directly, since they all share the same + * well-known point of reference (the Unix epoch). + * + * Timestamps with an unset / empty timezone + * ----------------------------------------- + * + * If a Timestamp column has no timezone value, its epoch is + * 1970-01-01 00:00:00 (January 1st 1970, midnight) in an *unknown* timezone. + * + * Therefore, timestamp values without a timezone cannot be meaningfully + * interpreted as physical points in time, but only as calendar / clock + * indications ("wall clock time") in an unspecified timezone. + * + * For example, the timestamp value 0 with an empty timezone string + * corresponds to "January 1st 1970, 00h00" in an unknown timezone: there + * is not enough information to interpret it as a well-defined physical + * point in time. + * + * One consequence is that timestamp values without a timezone cannot + * be reliably compared or ordered, since they may have different points of + * reference. In particular, it is *not* possible to interpret an unset + * or empty timezone as the same as "UTC". + * + * Conversion between timezones + * ---------------------------- + * + * If a Timestamp column has a non-empty timezone, changing the timezone + * to a different non-empty value is a metadata-only operation: + * the timestamp values need not change as their point of reference remains + * the same (the Unix epoch). + * + * However, if a Timestamp column has no timezone value, changing it to a + * non-empty value requires to think about the desired semantics. + * One possibility is to assume that the original timestamp values are + * relative to the epoch of the timezone being set; timestamp values should + * then adjusted to the Unix epoch (for example, changing the timezone from + * empty to "Europe/Paris" would require converting the timestamp values + * from "Europe/Paris" to "UTC", which seems counter-intuitive but is + * nevertheless correct). + * + * Guidelines for encoding data from external libraries + * ---------------------------------------------------- * * Date & time libraries often have multiple different data types for temporal - * data. In order to ease interoperability between different implementations the + * data. In order to ease interoperability between different implementations the * Arrow project has some recommendations for encoding these types into a Timestamp * column. * - * An "instant" represents a single moment in time that has no meaningful time zone - * or the time zone is unknown. A column of instants can also contain values from - * multiple time zones. To encode an instant set the timezone string to "UTC". - * - * A "zoned date-time" represents a single moment in time that has a meaningful - * reference time zone. To encode a zoned date-time as a Timestamp set the timezone - * string to the name of the timezone. There is some ambiguity between an instant - * and a zoned date-time with the UTC time zone. Both of these are stored the same. - * Typically, this distinction does not matter. If it does, then an application should - * use custom metadata or an extension type to distinguish between the two cases. - * - * An "offset date-time" represents a single moment in time combined with a meaningful - * offset from UTC. To encode an offset date-time as a Timestamp set the timezone string - * to the numeric time zone offset string (e.g. "+03:00"). - * - * A "local date-time" does not represent a single moment in time. It represents a wall - * clock time combined with a date. Because of daylight savings time there may multiple - * instants that correspond to a single local date-time in any given time zone. A - * local date-time is often stored as a struct or a Date32/Time64 pair. However, it can - * also be encoded into a Timestamp column. To do so the value should be the the time - * elapsed from the Unix epoch so that a wall clock in UTC would display the desired time. - * The timezone string should be set to null or the empty string. + * An "instant" represents a physical point in time that has no relevant timezone + * (for example, astronomical data). To encode an instant, use a Timestamp with + * the timezone string set to "UTC", and make sure the Timestamp values + * are relative to the UTC epoch (January 1st 1970, midnight). + * + * A "zoned date-time" represents a physical point in time annotated with an + * informative timezone (for example, the timezone in which the data was + * recorded). To encode a zoned date-time, use a Timestamp with the timezone + * string set to the name of the timezone, and make sure the Timestamp values + * are relative to the UTC epoch (January 1st 1970, midnight). + * + * (There is some ambiguity between an instant and a zoned date-time with the + * UTC timezone. Both of these are stored the same in Arrow. Typically, + * this distinction does not matter. If it does, then an application should + * use custom metadata or an extension type to distinguish between the two cases.) + * + * An "offset date-time" represents a physical point in time combined with an + * explicit offset from UTC. To encode an offset date-time, use a Timestamp + * with the timezone string set to the numeric timezone offset string + * (e.g. "+03:00"), and make sure the Timestamp values are relative to + * the UTC epoch (January 1st 1970, midnight). + * + * A "naive date-time" (also called "local date-time" in some libraries) + * represents a wall clock time combined with a calendar date, but with + * no indication of how to map this information to a physical point in time. + * Naive date-times must be handled with care because of this missing + * information, and also because daylight saving time (DST) may make + * some values ambiguous or non-existent. A naive date-time may be + * stored as a struct with Date and Time fields. However, it may also be + * encoded into a Timestamp column with an empty timezone. The timestamp + * values should be computed "as if" the timezone of the date-time values + * was UTC; for example, the naive date-time "January 1st 1970, 00h00" would + * be encoded as timestamp value 0. */ public final class Timestamp extends Table { public static void ValidateVersion() { Constants.FLATBUFFERS_1_12_0(); } @@ -66,24 +140,16 @@ public final class Timestamp extends Table { public short unit() { int o = __offset(4); return o != 0 ? bb.getShort(o + bb_pos) : 0; } /** - * The time zone is a string indicating the name of a time zone, one of: + * The timezone is an optional string indicating the name of a timezone, + * one of: * - * * As used in the Olson time zone database (the "tz database" or - * "tzdata"), such as "America/New_York" - * * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 + * * As used in the Olson timezone database (the "tz database" or + * "tzdata"), such as "America/New_York". + * * An absolute timezone offset of the form "+XX:XX" or "-XX:XX", + * such as "+07:30". * * Whether a timezone string is present indicates different semantics about - * the data: - * - * * If the time zone is null or an empty string, the data is a local date-time - * and does not represent a single moment in time. Instead it represents a wall clock - * time and care should be taken to avoid interpreting it semantically as an instant. - * - * * If the time zone is set to a valid value, values can be displayed as - * "localized" to that time zone, even though the underlying 64-bit - * integers are identical to the same data stored in UTC. Converting - * between time zones is a metadata-only operation and does not change the - * underlying values + * the data (see above). */ public String timezone() { int o = __offset(6); return o != 0 ? __string(o + bb_pos) : null; } public ByteBuffer timezoneAsByteBuffer() { return __vector_as_bytebuffer(6, 1); } diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/Type.java b/java/format/src/main/java/org/apache/arrow/flatbuf/Type.java index 5f1a550cffff4..29248bb23c303 100644 --- a/java/format/src/main/java/org/apache/arrow/flatbuf/Type.java +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/Type.java @@ -47,8 +47,13 @@ private Type() { } public static final byte LargeBinary = 19; public static final byte LargeUtf8 = 20; public static final byte LargeList = 21; + public static final byte RunEndEncoded = 22; + public static final byte BinaryView = 23; + public static final byte Utf8View = 24; + public static final byte ListView = 25; + public static final byte LargeListView = 26; - public static final String[] names = { "NONE", "Null", "Int", "FloatingPoint", "Binary", "Utf8", "Bool", "Decimal", "Date", "Time", "Timestamp", "Interval", "List", "Struct_", "Union", "FixedSizeBinary", "FixedSizeList", "Map", "Duration", "LargeBinary", "LargeUtf8", "LargeList", }; + public static final String[] names = { "NONE", "Null", "Int", "FloatingPoint", "Binary", "Utf8", "Bool", "Decimal", "Date", "Time", "Timestamp", "Interval", "List", "Struct_", "Union", "FixedSizeBinary", "FixedSizeList", "Map", "Duration", "LargeBinary", "LargeUtf8", "LargeList", "RunEndEncoded", "BinaryView", "Utf8View", "ListView", "LargeListView", }; public static String name(int e) { return names[e]; } } diff --git a/java/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java b/java/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java new file mode 100644 index 0000000000000..035c977576e43 --- /dev/null +++ b/java/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// automatically generated by the FlatBuffers compiler, do not modify + +package org.apache.arrow.flatbuf; + +import java.nio.*; +import java.lang.*; +import java.util.*; +import com.google.flatbuffers.*; + +@SuppressWarnings("unused") +/** + * Logically the same as Utf8, but the internal representation uses a view + * struct that contains the string length and either the string's entire data + * inline (for small strings) or an inlined prefix, an index of another buffer, + * and an offset pointing to a slice in that buffer (for non-small strings). + * + * Since it uses a variable number of data buffers, each Field with this type + * must have a corresponding entry in `variadicBufferCounts`. + */ +public final class Utf8View extends Table { + public static void ValidateVersion() { Constants.FLATBUFFERS_1_12_0(); } + public static Utf8View getRootAsUtf8View(ByteBuffer _bb) { return getRootAsUtf8View(_bb, new Utf8View()); } + public static Utf8View getRootAsUtf8View(ByteBuffer _bb, Utf8View obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } + public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } + public Utf8View __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + + public static void startUtf8View(FlatBufferBuilder builder) { builder.startTable(0); } + public static int endUtf8View(FlatBufferBuilder builder) { + int o = builder.endTable(); + return o; + } + + public static final class Vector extends BaseVector { + public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { __reset(_vector, _element_size, _bb); return this; } + + public Utf8View get(int j) { return get(new Utf8View(), j); } + public Utf8View get(Utf8View obj, int j) { return obj.__assign(__indirect(__element(j), bb), bb); } + } +} + From 49838858eab984fc4a7e8595d7dd1883b00350ea Mon Sep 17 00:00:00 2001 From: sgilmore10 <74676073+sgilmore10@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:14:31 -0500 Subject: [PATCH 27/57] GH-38630: [MATLAB] `arrow.array.BooleanArray`'s `toMATLAB` method does not take slice offsets into account (#38636) ### Rationale for this change While working on #38415, I noticed that the `toMATLAB` method of `arrow.array.BooleanArray` does not take the slice offset into account. This will cause the `toMATLAB` method to return the wrong value. ### What changes are included in this PR? 1. Updated `arrow::matlab::bit::unpack` function to accept a `start_offset` as input. 2. Updated clients (`BooleanArray::toMATLAB` and `Array::getValid`) to supply the array `offset` as the `start_offset`. ### Are these changes tested? The existing tests cover these changes. Additionally, the changes for #38415 will include tests that verify the `toMATLAB` method returns the correct MATLAB array when the underlying Arrow array has been sliced. ### Are there any user-facing changes? No. * Closes: #38630 Authored-by: Sarah Gilmore Signed-off-by: Kevin Gurney --- matlab/src/cpp/arrow/matlab/array/proxy/array.cc | 2 +- matlab/src/cpp/arrow/matlab/array/proxy/boolean_array.cc | 2 +- matlab/src/cpp/arrow/matlab/bit/unpack.cc | 3 +-- matlab/src/cpp/arrow/matlab/bit/unpack.h | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/matlab/src/cpp/arrow/matlab/array/proxy/array.cc b/matlab/src/cpp/arrow/matlab/array/proxy/array.cc index 4e52c990d3eae..09d9473df4795 100644 --- a/matlab/src/cpp/arrow/matlab/array/proxy/array.cc +++ b/matlab/src/cpp/arrow/matlab/array/proxy/array.cc @@ -100,7 +100,7 @@ namespace arrow::matlab::array::proxy { } auto validity_bitmap = array->null_bitmap(); - auto valid_elements_mda = bit::unpack(validity_bitmap, array_length); + auto valid_elements_mda = bit::unpack(validity_bitmap, array_length, array->offset()); context.outputs[0] = valid_elements_mda; } diff --git a/matlab/src/cpp/arrow/matlab/array/proxy/boolean_array.cc b/matlab/src/cpp/arrow/matlab/array/proxy/boolean_array.cc index 6a6e478274823..da3560ce522f3 100644 --- a/matlab/src/cpp/arrow/matlab/array/proxy/boolean_array.cc +++ b/matlab/src/cpp/arrow/matlab/array/proxy/boolean_array.cc @@ -53,7 +53,7 @@ namespace arrow::matlab::array::proxy { void BooleanArray::toMATLAB(libmexclass::proxy::method::Context& context) { auto array_length = array->length(); auto packed_logical_data_buffer = std::static_pointer_cast(array)->values(); - auto logical_array_mda = bit::unpack(packed_logical_data_buffer, array_length); + auto logical_array_mda = bit::unpack(packed_logical_data_buffer, array_length, array->offset()); context.outputs[0] = logical_array_mda; } } diff --git a/matlab/src/cpp/arrow/matlab/bit/unpack.cc b/matlab/src/cpp/arrow/matlab/bit/unpack.cc index 7135d593cf752..6cc88d48ede43 100644 --- a/matlab/src/cpp/arrow/matlab/bit/unpack.cc +++ b/matlab/src/cpp/arrow/matlab/bit/unpack.cc @@ -20,7 +20,7 @@ #include "arrow/util/bitmap_visit.h" namespace arrow::matlab::bit { - ::matlab::data::TypedArray unpack(const std::shared_ptr& packed_buffer, int64_t length) { + ::matlab::data::TypedArray unpack(const std::shared_ptr& packed_buffer, int64_t length, int64_t start_offset) { const auto packed_buffer_ptr = packed_buffer->data(); ::matlab::data::ArrayFactory factory; @@ -31,7 +31,6 @@ namespace arrow::matlab::bit { auto unpacked_buffer_ptr = unpacked_buffer.get(); auto visitFcn = [&](const bool is_valid) { *unpacked_buffer_ptr++ = is_valid; }; - const int64_t start_offset = 0; arrow::internal::VisitBitsUnrolled(packed_buffer_ptr, start_offset, length, visitFcn); ::matlab::data::TypedArray unpacked_matlab_logical_Array = factory.createArrayFromBuffer({array_length, 1}, std::move(unpacked_buffer)); diff --git a/matlab/src/cpp/arrow/matlab/bit/unpack.h b/matlab/src/cpp/arrow/matlab/bit/unpack.h index b6debb85f837b..6cd633e76fa56 100644 --- a/matlab/src/cpp/arrow/matlab/bit/unpack.h +++ b/matlab/src/cpp/arrow/matlab/bit/unpack.h @@ -22,6 +22,6 @@ #include "MatlabDataArray.hpp" namespace arrow::matlab::bit { - ::matlab::data::TypedArray unpack(const std::shared_ptr& packed_buffer, int64_t length); + ::matlab::data::TypedArray unpack(const std::shared_ptr& packed_buffer, int64_t length, int64_t start_offset); const uint8_t* extract_ptr(const ::matlab::data::TypedArray& unpacked_validity_bitmap); } From 79e328b3b7ce23002bc46904c1944654ce4cd0a3 Mon Sep 17 00:00:00 2001 From: Benjamin Schmidt Date: Thu, 9 Nov 2023 15:19:03 -0500 Subject: [PATCH 28/57] MINOR: [JS][Docs] Correct typo in README (#38658) ### Rationale for this change A demo variable is named 'uft8' but should clearly be 'utf8'. ### What changes are included in this PR? Two transpostions of two characters ### Are these changes tested? N/A ### Are there any user-facing changes? No Authored-by: Benjamin Schmidt Signed-off-by: Sutou Kouhei --- js/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/README.md b/js/README.md index f626ef7240fca..f3dd3fef890f2 100644 --- a/js/README.md +++ b/js/README.md @@ -151,7 +151,7 @@ Strings can be encoded as UTF-8 or dictionary encoded UTF-8. Dictionary encoding ```js import { makeVector, vectorFromArray, Dictionary, Uint8, Utf8 } from "apache-arrow"; -const uft8Vector = vectorFromArray(['foo', 'bar', 'baz'], new Utf8); +const utf8Vector = vectorFromArray(['foo', 'bar', 'baz'], new Utf8); const dictionaryVector1 = vectorFromArray( ['foo', 'bar', 'baz', 'foo', 'bar'] @@ -159,7 +159,7 @@ const dictionaryVector1 = vectorFromArray( const dictionaryVector2 = makeVector({ data: [0, 1, 2, 0, 1], // indexes into the dictionary - dictionary: uft8Vector, + dictionary: utf8Vector, type: new Dictionary(new Utf8, new Uint8) }); ``` From 0acf4c8271e65e103c9fbbf753b8759fd9058f45 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 9 Nov 2023 13:56:53 -0800 Subject: [PATCH 29/57] GH-38614: [Java] Add VarBinary and VarCharWriter helper methods to more writers (#38631) ### Rationale for this change Add the overrides for new convenience Writer methods added to VarCharWriter and VarBinaryWriter so that classes that use composition such as UnionWriter and PromotableWriter can invoke them properly. ### What changes are included in this PR? - Rename from writeTo$type to write$type for consistency with other methods - Add new helper methods to PromotableWriter - Add new helper methods to complex writers such as list and union ### Are these changes tested? Yes. New unit tests added for several Writer classes. **This PR includes breaking changes to public APIs.** The writeTo() and similar methods in Writers have been renamed to just write() * Closes: #38614 Authored-by: James Duong Signed-off-by: David Li --- .../templates/AbstractFieldWriter.java | 11 +- .../AbstractPromotableFieldWriter.java | 32 +++ .../codegen/templates/ComplexWriters.java | 16 +- .../templates/UnionFixedSizeListWriter.java | 56 +++++ .../codegen/templates/UnionListWriter.java | 37 +++ .../main/codegen/templates/UnionWriter.java | 36 +++ .../vector/complex/impl/PromotableWriter.java | 63 +++++- .../arrow/vector/TestFixedSizeListVector.java | 92 +++++++- .../complex/impl/TestPromotableWriter.java | 197 ++++++++++++++++ .../complex/writer/TestComplexWriter.java | 211 ++++++++++++++++++ .../complex/writer/TestSimpleWriter.java | 16 +- 11 files changed, 741 insertions(+), 26 deletions(-) diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java index bb4ee45eaa073..6c2368117f7c2 100644 --- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -27,6 +27,9 @@ /* * This class is generated using freemarker and the ${.template_name} template. + * Note that changes to the AbstractFieldWriter template should also get reflected in the + * AbstractPromotableFieldWriter, ComplexWriters, UnionFixedSizeListWriter, UnionListWriter + * and UnionWriter templates and the PromotableWriter concrete code. */ @SuppressWarnings("unused") abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter { @@ -125,19 +128,19 @@ public void write(${name}Holder holder) { <#if minor.class?ends_with("VarBinary")> - public void writeTo${minor.class}(byte[] value) { + public void write${minor.class}(byte[] value) { fail("${name}"); } - public void writeTo${minor.class}(byte[] value, int offset, int length) { + public void write${minor.class}(byte[] value, int offset, int length) { fail("${name}"); } - public void writeTo${minor.class}(ByteBuffer value) { + public void write${minor.class}(ByteBuffer value) { fail("${name}"); } - public void writeTo${minor.class}(ByteBuffer value, int offset, int length) { + public void write${minor.class}(ByteBuffer value, int offset, int length) { fail("${name}"); } diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index 2f963a9df0d05..59f9fb5b8098d 100644 --- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -221,6 +221,38 @@ public void write(${name}Holder holder) { } + <#if minor.class?ends_with("VarBinary")> + @Override + public void write${minor.class}(byte[] value) { + getWriter(MinorType.${name?upper_case}).write${minor.class}(value); + } + + @Override + public void write${minor.class}(byte[] value, int offset, int length) { + getWriter(MinorType.${name?upper_case}).write${minor.class}(value, offset, length); + } + + @Override + public void write${minor.class}(ByteBuffer value) { + getWriter(MinorType.${name?upper_case}).write${minor.class}(value); + } + + @Override + public void write${minor.class}(ByteBuffer value, int offset, int length) { + getWriter(MinorType.${name?upper_case}).write${minor.class}(value, offset, length); + } + <#elseif minor.class?ends_with("VarChar")> + @Override + public void write${minor.class}(Text value) { + getWriter(MinorType.${name?upper_case}).write${minor.class}(value); + } + + @Override + public void write${minor.class}(String value) { + getWriter(MinorType.${name?upper_case}).write${minor.class}(value); + } + + public void writeNull() { } diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 51a52a6e3070d..2e3caae1f0f22 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -194,22 +194,22 @@ public void writeNull() { <#if minor.class?ends_with("VarBinary")> - public void writeTo${minor.class}(byte[] value) { + public void write${minor.class}(byte[] value) { vector.setSafe(idx(), value); vector.setValueCount(idx() + 1); } - public void writeTo${minor.class}(byte[] value, int offset, int length) { + public void write${minor.class}(byte[] value, int offset, int length) { vector.setSafe(idx(), value, offset, length); vector.setValueCount(idx() + 1); } - public void writeTo${minor.class}(ByteBuffer value) { + public void write${minor.class}(ByteBuffer value) { vector.setSafe(idx(), value, 0, value.remaining()); vector.setValueCount(idx() + 1); } - public void writeTo${minor.class}(ByteBuffer value, int offset, int length) { + public void write${minor.class}(ByteBuffer value, int offset, int length) { vector.setSafe(idx(), value, offset, length); vector.setValueCount(idx() + 1); } @@ -259,13 +259,13 @@ public interface ${eName}Writer extends BaseWriter { <#if minor.class?ends_with("VarBinary")> - public void writeTo${minor.class}(byte[] value); + public void write${minor.class}(byte[] value); - public void writeTo${minor.class}(byte[] value, int offset, int length); + public void write${minor.class}(byte[] value, int offset, int length); - public void writeTo${minor.class}(ByteBuffer value); + public void write${minor.class}(ByteBuffer value); - public void writeTo${minor.class}(ByteBuffer value, int offset, int length); + public void write${minor.class}(ByteBuffer value, int offset, int length); <#if minor.class?ends_with("VarChar")> diff --git a/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java b/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java index 55c661bfc6023..3436e3a967651 100644 --- a/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java @@ -295,6 +295,62 @@ public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) { <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> + <#if minor.class?ends_with("VarBinary")> + @Override + public void write${minor.class}(byte[] value) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.write${minor.class}(value); + writer.setPosition(writer.idx() + 1); + } + + @Override + public void write${minor.class}(byte[] value, int offset, int length) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.write${minor.class}(value, offset, length); + writer.setPosition(writer.idx() + 1); + } + + @Override + public void write${minor.class}(ByteBuffer value) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.write${minor.class}(value); + writer.setPosition(writer.idx() + 1); + } + + @Override + public void write${minor.class}(ByteBuffer value, int offset, int length) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.write${minor.class}(value, offset, length); + writer.setPosition(writer.idx() + 1); + } + <#elseif minor.class?ends_with("VarChar")> + @Override + public void write${minor.class}(Text value) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.write${minor.class}(value); + writer.setPosition(writer.idx() + 1); + } + + @Override + public void write${minor.class}(String value) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.write${minor.class}(value); + writer.setPosition(writer.idx() + 1); + } + + <#if !minor.typeParams?? > @Override public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java index fac75a9ce5637..5c0565ee27175 100644 --- a/java/vector/src/main/codegen/templates/UnionListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionListWriter.java @@ -274,6 +274,43 @@ public void write(${name}Holder holder) { writer.write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); writer.setPosition(writer.idx()+1); } + + + <#if minor.class?ends_with("VarBinary")> + @Override + public void write${minor.class}(byte[] value) { + writer.write${minor.class}(value); + writer.setPosition(writer.idx() + 1); + } + + @Override + public void write${minor.class}(byte[] value, int offset, int length) { + writer.write${minor.class}(value, offset, length); + writer.setPosition(writer.idx() + 1); + } + + @Override + public void write${minor.class}(ByteBuffer value) { + writer.write${minor.class}(value); + writer.setPosition(writer.idx() + 1); + } + + @Override + public void write${minor.class}(ByteBuffer value, int offset, int length) { + writer.write${minor.class}(value, offset, length); + writer.setPosition(writer.idx() + 1); + } + <#elseif minor.class?ends_with("VarChar")> + @Override + public void write${minor.class}(Text value) { + writer.write${minor.class}(value); + writer.setPosition(writer.idx() + 1); + } + + public void write${minor.class}(String value) { + writer.write${minor.class}(value); + writer.setPosition(writer.idx() + 1); + } diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java index 4efd1026cac4a..08dbf24324b17 100644 --- a/java/vector/src/main/codegen/templates/UnionWriter.java +++ b/java/vector/src/main/codegen/templates/UnionWriter.java @@ -302,6 +302,42 @@ public void write(${name}Holder holder) { get${name}Writer(arrowType).setPosition(idx()); get${name}Writer(arrowType).writeBigEndianBytesTo${name}(value, arrowType); } + <#elseif minor.class?ends_with("VarBinary")> + @Override + public void write${minor.class}(byte[] value) { + get${name}Writer().setPosition(idx()); + get${name}Writer().write${minor.class}(value); + } + + @Override + public void write${minor.class}(byte[] value, int offset, int length) { + get${name}Writer().setPosition(idx()); + get${name}Writer().write${minor.class}(value, offset, length); + } + + @Override + public void write${minor.class}(ByteBuffer value) { + get${name}Writer().setPosition(idx()); + get${name}Writer().write${minor.class}(value); + } + + @Override + public void write${minor.class}(ByteBuffer value, int offset, int length) { + get${name}Writer().setPosition(idx()); + get${name}Writer().write${minor.class}(value, offset, length); + } + <#elseif minor.class?ends_with("VarChar")> + @Override + public void write${minor.class}(${friendlyType} value) { + get${name}Writer().setPosition(idx()); + get${name}Writer().write${minor.class}(value); + } + + @Override + public void write${minor.class}(String value) { + get${name}Writer().setPosition(idx()); + get${name}Writer().write${minor.class}(value); + } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index d99efceae3eca..f7be277f592a6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -18,6 +18,7 @@ package org.apache.arrow.vector.complex.impl; import java.math.BigDecimal; +import java.nio.ByteBuffer; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.vector.FieldVector; @@ -37,6 +38,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; /** @@ -378,7 +380,66 @@ public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) { /*bitWidth=*/256)).writeBigEndianBytesToDecimal256(value, arrowType); } - + @Override + public void writeVarBinary(byte[] value) { + getWriter(MinorType.VARBINARY).writeVarBinary(value); + } + + @Override + public void writeVarBinary(byte[] value, int offset, int length) { + getWriter(MinorType.VARBINARY).writeVarBinary(value, offset, length); + } + + @Override + public void writeVarBinary(ByteBuffer value) { + getWriter(MinorType.VARBINARY).writeVarBinary(value); + } + + @Override + public void writeVarBinary(ByteBuffer value, int offset, int length) { + getWriter(MinorType.VARBINARY).writeVarBinary(value, offset, length); + } + + @Override + public void writeLargeVarBinary(byte[] value) { + getWriter(MinorType.LARGEVARBINARY).writeLargeVarBinary(value); + } + + @Override + public void writeLargeVarBinary(byte[] value, int offset, int length) { + getWriter(MinorType.LARGEVARBINARY).writeLargeVarBinary(value, offset, length); + } + + @Override + public void writeLargeVarBinary(ByteBuffer value) { + getWriter(MinorType.LARGEVARBINARY).writeLargeVarBinary(value); + } + + @Override + public void writeLargeVarBinary(ByteBuffer value, int offset, int length) { + getWriter(MinorType.LARGEVARBINARY).writeLargeVarBinary(value, offset, length); + } + + @Override + public void writeVarChar(Text value) { + getWriter(MinorType.VARCHAR).writeVarChar(value); + } + + @Override + public void writeVarChar(String value) { + getWriter(MinorType.VARCHAR).writeVarChar(value); + } + + @Override + public void writeLargeVarChar(Text value) { + getWriter(MinorType.LARGEVARCHAR).writeLargeVarChar(value); + } + + @Override + public void writeLargeVarChar(String value) { + getWriter(MinorType.LARGEVARCHAR).writeLargeVarChar(value); + } + @Override public void allocate() { getWriter().allocate(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 9d7e413a739ad..0023b1dddb8e7 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import java.math.BigDecimal; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -37,6 +38,7 @@ import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; import org.junit.After; import org.junit.Assert; @@ -457,18 +459,98 @@ public void testVectorWithNulls() { assertEquals(4, vector1.getValueCount()); - List realValue1 = vector1.getObject(0); + List realValue1 = vector1.getObject(0); assertEquals(values1, realValue1); - List realValue2 = vector1.getObject(1); + List realValue2 = vector1.getObject(1); assertEquals(values2, realValue2); - List realValue3 = vector1.getObject(2); + List realValue3 = vector1.getObject(2); assertEquals(values3, realValue3); - List realValue4 = vector1.getObject(3); + List realValue4 = vector1.getObject(3); assertEquals(values4, realValue4); } } - private int[] convertListToIntArray(List list) { + @Test + public void testWriteVarCharHelpers() throws Exception { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/4, allocator)) { + + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + writer.startList(); + writer.writeVarChar("row1,1"); + writer.writeVarChar(new Text("row1,2")); + writer.writeNull(); + writer.writeNull(); + writer.endList(); + + assertEquals("row1,1", vector.getObject(0).get(0).toString()); + assertEquals("row1,2", vector.getObject(0).get(1).toString()); + } + } + + @Test + public void testWriteLargeVarCharHelpers() throws Exception { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/4, allocator)) { + + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + writer.startList(); + writer.writeLargeVarChar("row1,1"); + writer.writeLargeVarChar(new Text("row1,2")); + writer.writeNull(); + writer.writeNull(); + writer.endList(); + + assertEquals("row1,1", vector.getObject(0).get(0).toString()); + assertEquals("row1,2", vector.getObject(0).get(1).toString()); + } + } + + @Test + public void testWriteVarBinaryHelpers() throws Exception { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/4, allocator)) { + + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + writer.startList(); + writer.writeVarBinary("row1,1".getBytes()); + writer.writeVarBinary("row1,2".getBytes(), 0, "row1,2".getBytes().length); + writer.writeVarBinary(ByteBuffer.wrap("row1,3".getBytes())); + writer.writeVarBinary(ByteBuffer.wrap("row1,4".getBytes()), 0, "row1,4".getBytes().length); + writer.endList(); + + assertEquals("row1,1", new String((byte[]) (vector.getObject(0).get(0)))); + assertEquals("row1,2", new String((byte[]) (vector.getObject(0).get(1)))); + assertEquals("row1,3", new String((byte[]) (vector.getObject(0).get(2)))); + assertEquals("row1,4", new String((byte[]) (vector.getObject(0).get(3)))); + } + } + + @Test + public void testWriteLargeVarBinaryHelpers() throws Exception { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/4, allocator)) { + + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + writer.startList(); + writer.writeLargeVarBinary("row1,1".getBytes()); + writer.writeLargeVarBinary("row1,2".getBytes(), 0, "row1,2".getBytes().length); + writer.writeLargeVarBinary(ByteBuffer.wrap("row1,3".getBytes())); + writer.writeLargeVarBinary(ByteBuffer.wrap("row1,4".getBytes()), 0, "row1,4".getBytes().length); + writer.endList(); + + assertEquals("row1,1", new String((byte[]) (vector.getObject(0).get(0)))); + assertEquals("row1,2", new String((byte[]) (vector.getObject(0).get(1)))); + assertEquals("row1,3", new String((byte[]) (vector.getObject(0).get(2)))); + assertEquals("row1,4", new String((byte[]) (vector.getObject(0).get(3)))); + } + } + + private int[] convertListToIntArray(List list) { int[] values = new int[list.size()]; for (int i = 0; i < list.size(); i++) { values[i] = (int) list.get(i); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 1068f7c030eb5..4c8c96a0d74d3 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -28,6 +28,10 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.DirtyRootAllocator; +import org.apache.arrow.vector.LargeVarBinaryVector; +import org.apache.arrow.vector.LargeVarCharVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.NonNullableStructVector; import org.apache.arrow.vector.complex.StructVector; @@ -43,6 +47,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.Text; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -392,4 +397,196 @@ public void testNoPromoteFixedSizeBinaryToUnionWithNull() throws Exception { buf.close(); } } + + @Test + public void testPromoteLargeVarCharHelpersOnStruct() throws Exception { + try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final StructVector v = container.addOrGetStruct("test"); + final PromotableWriter writer = new PromotableWriter(v, container)) { + container.allocateNew(); + + writer.start(); + writer.setPosition(0); + writer.largeVarChar("c").writeLargeVarChar(new Text("foo")); + writer.setPosition(1); + writer.largeVarChar("c").writeLargeVarChar("foo2"); + writer.end(); + + final LargeVarCharVector uv = v.getChild("c", LargeVarCharVector.class); + assertEquals("foo", uv.getObject(0).toString()); + assertEquals("foo2", uv.getObject(1).toString()); + } + } + + @Test + public void testPromoteVarCharHelpersOnStruct() throws Exception { + try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final StructVector v = container.addOrGetStruct("test"); + final PromotableWriter writer = new PromotableWriter(v, container)) { + container.allocateNew(); + + writer.start(); + writer.setPosition(0); + writer.varChar("c").writeVarChar(new Text("foo")); + writer.setPosition(1); + writer.varChar("c").writeVarChar("foo2"); + writer.end(); + + final VarCharVector uv = v.getChild("c", VarCharVector.class); + assertEquals("foo", uv.getObject(0).toString()); + assertEquals("foo2", uv.getObject(1).toString()); + } + } + + @Test + public void testPromoteVarCharHelpersDirect() throws Exception { + try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final StructVector v = container.addOrGetStruct("test"); + final PromotableWriter writer = new PromotableWriter(v, container)) { + container.allocateNew(); + + writer.start(); + writer.setPosition(0); + writer.writeVarChar(new Text("foo")); + writer.setPosition(1); + writer.writeVarChar("foo2"); + writer.end(); + + // The "test" vector in the parent container should have been replaced with a UnionVector. + UnionVector promotedVector = container.getChild("test", UnionVector.class); + VarCharVector vector = promotedVector.getVarCharVector(); + assertEquals("foo", vector.getObject(0).toString()); + assertEquals("foo2", vector.getObject(1).toString()); + } + } + + @Test + public void testPromoteLargeVarCharHelpersDirect() throws Exception { + try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final StructVector v = container.addOrGetStruct("test"); + final PromotableWriter writer = new PromotableWriter(v, container)) { + container.allocateNew(); + + writer.start(); + writer.setPosition(0); + writer.writeLargeVarChar(new Text("foo")); + writer.setPosition(1); + writer.writeLargeVarChar("foo2"); + writer.end(); + + // The "test" vector in the parent container should have been replaced with a UnionVector. + UnionVector promotedVector = container.getChild("test", UnionVector.class); + LargeVarCharVector vector = promotedVector.getLargeVarCharVector(); + assertEquals("foo", vector.getObject(0).toString()); + assertEquals("foo2", vector.getObject(1).toString()); + } + } + + @Test + public void testPromoteVarBinaryHelpersOnStruct() throws Exception { + try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final StructVector v = container.addOrGetStruct("test"); + final PromotableWriter writer = new PromotableWriter(v, container)) { + container.allocateNew(); + + writer.start(); + writer.setPosition(0); + writer.varBinary("c").writeVarBinary("row1".getBytes()); + writer.setPosition(1); + writer.varBinary("c").writeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + writer.setPosition(2); + writer.varBinary("c").writeVarBinary(ByteBuffer.wrap("row3".getBytes())); + writer.setPosition(3); + writer.varBinary("c").writeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + writer.end(); + + final VarBinaryVector uv = v.getChild("c", VarBinaryVector.class); + assertEquals("row1", new String(uv.get(0))); + assertEquals("row2", new String(uv.get(1))); + assertEquals("row3", new String(uv.get(2))); + assertEquals("row4", new String(uv.get(3))); + } + } + + @Test + public void testPromoteVarBinaryHelpersDirect() throws Exception { + try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final StructVector v = container.addOrGetStruct("test"); + final PromotableWriter writer = new PromotableWriter(v, container)) { + container.allocateNew(); + + writer.start(); + writer.setPosition(0); + writer.writeVarBinary("row1".getBytes()); + writer.setPosition(1); + writer.writeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + writer.setPosition(2); + writer.writeVarBinary(ByteBuffer.wrap("row3".getBytes())); + writer.setPosition(3); + writer.writeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + writer.end(); + + // The "test" vector in the parent container should have been replaced with a UnionVector. + UnionVector promotedVector = container.getChild("test", UnionVector.class); + VarBinaryVector uv = promotedVector.getVarBinaryVector(); + assertEquals("row1", new String(uv.get(0))); + assertEquals("row2", new String(uv.get(1))); + assertEquals("row3", new String(uv.get(2))); + assertEquals("row4", new String(uv.get(3))); + } + } + + @Test + public void testPromoteLargeVarBinaryHelpersOnStruct() throws Exception { + try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final StructVector v = container.addOrGetStruct("test"); + final PromotableWriter writer = new PromotableWriter(v, container)) { + container.allocateNew(); + + writer.start(); + writer.setPosition(0); + writer.largeVarBinary("c").writeLargeVarBinary("row1".getBytes()); + writer.setPosition(1); + writer.largeVarBinary("c").writeLargeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + writer.setPosition(2); + writer.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes())); + writer.setPosition(3); + writer.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + writer.end(); + + final LargeVarBinaryVector uv = v.getChild("c", LargeVarBinaryVector.class); + assertEquals("row1", new String(uv.get(0))); + assertEquals("row2", new String(uv.get(1))); + assertEquals("row3", new String(uv.get(2))); + assertEquals("row4", new String(uv.get(3))); + } + } + + @Test + public void testPromoteLargeVarBinaryHelpersDirect() throws Exception { + try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final StructVector v = container.addOrGetStruct("test"); + final PromotableWriter writer = new PromotableWriter(v, container)) { + container.allocateNew(); + + writer.start(); + writer.setPosition(0); + writer.writeLargeVarBinary("row1".getBytes()); + writer.setPosition(1); + writer.writeLargeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + writer.setPosition(2); + writer.writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes())); + writer.setPosition(3); + writer.writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + writer.end(); + + // The "test" vector in the parent container should have been replaced with a UnionVector. + UnionVector promotedVector = container.getChild("test", UnionVector.class); + LargeVarBinaryVector uv = promotedVector.getLargeVarBinaryVector(); + assertEquals("row1", new String(uv.get(0))); + assertEquals("row2", new String(uv.get(1))); + assertEquals("row3", new String(uv.get(2))); + assertEquals("row4", new String(uv.get(3))); + } + } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 6f7f5abd30ac9..96d39e85f1f4a 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -20,6 +20,7 @@ import static org.junit.Assert.*; import java.math.BigDecimal; +import java.nio.ByteBuffer; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashSet; @@ -35,7 +36,11 @@ import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.LargeVarBinaryVector; +import org.apache.arrow.vector.LargeVarCharVector; import org.apache.arrow.vector.SchemaChangeCallBack; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.NonNullableStructVector; @@ -1667,4 +1672,210 @@ public void testMapWithStructKey() { assertEquals(1, mapReader.value().readInteger().intValue()); } } + + @Test + public void structWriterVarCharHelpers() { + try (NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator)) { + ComplexWriter writer = new ComplexWriterImpl("root", parent, false, true); + StructWriter rootWriter = writer.rootAsStruct(); + rootWriter.start(); + rootWriter.setPosition(0); + rootWriter.varChar("c").writeVarChar(new Text("row1")); + rootWriter.setPosition(1); + rootWriter.varChar("c").writeVarChar("row2"); + rootWriter.end(); + + VarCharVector vector = parent.getChild("root", StructVector.class).getChild("c", VarCharVector.class); + + assertEquals("row1", vector.getObject(0).toString()); + assertEquals("row2", vector.getObject(1).toString()); + } + } + + @Test + public void structWriterLargeVarCharHelpers() { + try (NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator)) { + ComplexWriter writer = new ComplexWriterImpl("root", parent, false, true); + StructWriter rootWriter = writer.rootAsStruct(); + rootWriter.start(); + rootWriter.setPosition(0); + rootWriter.largeVarChar("c").writeLargeVarChar(new Text("row1")); + rootWriter.setPosition(1); + rootWriter.largeVarChar("c").writeLargeVarChar("row2"); + rootWriter.end(); + + LargeVarCharVector vector = parent.getChild("root", StructVector.class).getChild("c", + LargeVarCharVector.class); + + assertEquals("row1", vector.getObject(0).toString()); + assertEquals("row2", vector.getObject(1).toString()); + } + } + + @Test + public void structWriterVarBinaryHelpers() { + try (NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator)) { + ComplexWriter writer = new ComplexWriterImpl("root", parent, false, true); + StructWriter rootWriter = writer.rootAsStruct(); + rootWriter.start(); + rootWriter.setPosition(0); + rootWriter.varBinary("c").writeVarBinary("row1".getBytes()); + rootWriter.setPosition(1); + rootWriter.varBinary("c").writeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + rootWriter.setPosition(2); + rootWriter.varBinary("c").writeVarBinary(ByteBuffer.wrap("row3".getBytes())); + rootWriter.setPosition(3); + rootWriter.varBinary("c").writeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + rootWriter.end(); + + VarBinaryVector uv = parent.getChild("root", StructVector.class).getChild("c", VarBinaryVector.class); + + assertEquals("row1", new String(uv.get(0))); + assertEquals("row2", new String(uv.get(1))); + assertEquals("row3", new String(uv.get(2))); + assertEquals("row4", new String(uv.get(3))); + } + } + + @Test + public void structWriterLargeVarBinaryHelpers() { + try (NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator)) { + ComplexWriter writer = new ComplexWriterImpl("root", parent, false, true); + StructWriter rootWriter = writer.rootAsStruct(); + rootWriter.start(); + rootWriter.setPosition(0); + rootWriter.largeVarBinary("c").writeLargeVarBinary("row1".getBytes()); + rootWriter.setPosition(1); + rootWriter.largeVarBinary("c").writeLargeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + rootWriter.setPosition(2); + rootWriter.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes())); + rootWriter.setPosition(3); + rootWriter.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, + "row4".getBytes().length); + rootWriter.end(); + + LargeVarBinaryVector uv = parent.getChild("root", StructVector.class).getChild("c", + LargeVarBinaryVector.class); + + assertEquals("row1", new String(uv.get(0))); + assertEquals("row2", new String(uv.get(1))); + assertEquals("row3", new String(uv.get(2))); + assertEquals("row4", new String(uv.get(3))); + } + } + + @Test + public void listVarCharHelpers() { + try (ListVector listVector = ListVector.empty("list", allocator)) { + listVector.allocateNew(); + UnionListWriter listWriter = new UnionListWriter(listVector); + listWriter.startList(); + listWriter.writeVarChar("row1"); + listWriter.writeVarChar(new Text("row2")); + listWriter.endList(); + listWriter.setValueCount(1); + assertEquals("row1", listVector.getObject(0).get(0).toString()); + assertEquals("row2", listVector.getObject(0).get(1).toString()); + } + } + + @Test + public void listLargeVarCharHelpers() { + try (ListVector listVector = ListVector.empty("list", allocator)) { + listVector.allocateNew(); + UnionListWriter listWriter = new UnionListWriter(listVector); + listWriter.startList(); + listWriter.writeLargeVarChar("row1"); + listWriter.writeLargeVarChar(new Text("row2")); + listWriter.endList(); + listWriter.setValueCount(1); + assertEquals("row1", listVector.getObject(0).get(0).toString()); + assertEquals("row2", listVector.getObject(0).get(1).toString()); + } + } + + @Test + public void listVarBinaryHelpers() { + try (ListVector listVector = ListVector.empty("list", allocator)) { + listVector.allocateNew(); + UnionListWriter listWriter = new UnionListWriter(listVector); + listWriter.startList(); + listWriter.writeVarBinary("row1".getBytes()); + listWriter.writeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + listWriter.writeVarBinary(ByteBuffer.wrap("row3".getBytes())); + listWriter.writeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + listWriter.endList(); + listWriter.setValueCount(1); + assertEquals("row1", new String((byte[]) listVector.getObject(0).get(0))); + assertEquals("row2", new String((byte[]) listVector.getObject(0).get(1))); + assertEquals("row3", new String((byte[]) listVector.getObject(0).get(2))); + assertEquals("row4", new String((byte[]) listVector.getObject(0).get(3))); + } + } + + @Test + public void listLargeVarBinaryHelpers() { + try (ListVector listVector = ListVector.empty("list", allocator)) { + listVector.allocateNew(); + UnionListWriter listWriter = new UnionListWriter(listVector); + listWriter.startList(); + listWriter.writeLargeVarBinary("row1".getBytes()); + listWriter.writeLargeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + listWriter.writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes())); + listWriter.writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + listWriter.endList(); + listWriter.setValueCount(1); + assertEquals("row1", new String((byte[]) listVector.getObject(0).get(0))); + assertEquals("row2", new String((byte[]) listVector.getObject(0).get(1))); + assertEquals("row3", new String((byte[]) listVector.getObject(0).get(2))); + assertEquals("row4", new String((byte[]) listVector.getObject(0).get(3))); + } + } + + @Test + public void unionWithVarCharAndBinaryHelpers() throws Exception { + try (UnionVector vector = new UnionVector("union", allocator, /* field type */ null, /* call-back */ null)) { + UnionWriter unionWriter = new UnionWriter(vector); + unionWriter.allocate(); + unionWriter.start(); + unionWriter.setPosition(0); + unionWriter.writeVarChar("row1"); + unionWriter.setPosition(1); + unionWriter.writeVarChar(new Text("row2")); + unionWriter.setPosition(2); + unionWriter.writeLargeVarChar("row3"); + unionWriter.setPosition(3); + unionWriter.writeLargeVarChar(new Text("row4")); + unionWriter.setPosition(4); + unionWriter.writeVarBinary("row5".getBytes()); + unionWriter.setPosition(5); + unionWriter.writeVarBinary("row6".getBytes(), 0, "row6".getBytes().length); + unionWriter.setPosition(6); + unionWriter.writeVarBinary(ByteBuffer.wrap("row7".getBytes())); + unionWriter.setPosition(7); + unionWriter.writeVarBinary(ByteBuffer.wrap("row8".getBytes()), 0, "row8".getBytes().length); + unionWriter.setPosition(8); + unionWriter.writeLargeVarBinary("row9".getBytes()); + unionWriter.setPosition(9); + unionWriter.writeLargeVarBinary("row10".getBytes(), 0, "row10".getBytes().length); + unionWriter.setPosition(10); + unionWriter.writeLargeVarBinary(ByteBuffer.wrap("row11".getBytes())); + unionWriter.setPosition(11); + unionWriter.writeLargeVarBinary(ByteBuffer.wrap("row12".getBytes()), 0, "row12".getBytes().length); + unionWriter.end(); + + assertEquals("row1", new String(vector.getVarCharVector().get(0))); + assertEquals("row2", new String(vector.getVarCharVector().get(1))); + assertEquals("row3", new String(vector.getLargeVarCharVector().get(2))); + assertEquals("row4", new String(vector.getLargeVarCharVector().get(3))); + assertEquals("row5", new String(vector.getVarBinaryVector().get(4))); + assertEquals("row6", new String(vector.getVarBinaryVector().get(5))); + assertEquals("row7", new String(vector.getVarBinaryVector().get(6))); + assertEquals("row8", new String(vector.getVarBinaryVector().get(7))); + assertEquals("row9", new String(vector.getLargeVarBinaryVector().get(8))); + assertEquals("row10", new String(vector.getLargeVarBinaryVector().get(9))); + assertEquals("row11", new String(vector.getLargeVarBinaryVector().get(10))); + assertEquals("row12", new String(vector.getLargeVarBinaryVector().get(11))); + } + } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java index ef918b13fb691..27b8f1796ee31 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java @@ -54,7 +54,7 @@ public void testWriteByteArrayToVarBinary() throws Exception { try (VarBinaryVector vector = new VarBinaryVector("test", allocator); VarBinaryWriter writer = new VarBinaryWriterImpl(vector)) { byte[] input = new byte[] { 0x01, 0x02 }; - writer.writeToVarBinary(input); + writer.writeVarBinary(input); byte[] result = vector.get(0); Assert.assertArrayEquals(input, result); } @@ -65,7 +65,7 @@ public void testWriteByteArrayWithOffsetToVarBinary() throws Exception { try (VarBinaryVector vector = new VarBinaryVector("test", allocator); VarBinaryWriter writer = new VarBinaryWriterImpl(vector)) { byte[] input = new byte[] { 0x01, 0x02 }; - writer.writeToVarBinary(input, 1, 1); + writer.writeVarBinary(input, 1, 1); byte[] result = vector.get(0); Assert.assertArrayEquals(new byte[] { 0x02 }, result); } @@ -77,7 +77,7 @@ public void testWriteByteBufferToVarBinary() throws Exception { VarBinaryWriter writer = new VarBinaryWriterImpl(vector)) { byte[] input = new byte[] { 0x01, 0x02 }; ByteBuffer buffer = ByteBuffer.wrap(input); - writer.writeToVarBinary(buffer); + writer.writeVarBinary(buffer); byte[] result = vector.get(0); Assert.assertArrayEquals(input, result); } @@ -89,7 +89,7 @@ public void testWriteByteBufferWithOffsetToVarBinary() throws Exception { VarBinaryWriter writer = new VarBinaryWriterImpl(vector)) { byte[] input = new byte[] { 0x01, 0x02 }; ByteBuffer buffer = ByteBuffer.wrap(input); - writer.writeToVarBinary(buffer, 1, 1); + writer.writeVarBinary(buffer, 1, 1); byte[] result = vector.get(0); Assert.assertArrayEquals(new byte[] { 0x02 }, result); } @@ -100,7 +100,7 @@ public void testWriteByteArrayToLargeVarBinary() throws Exception { try (LargeVarBinaryVector vector = new LargeVarBinaryVector("test", allocator); LargeVarBinaryWriter writer = new LargeVarBinaryWriterImpl(vector)) { byte[] input = new byte[] { 0x01, 0x02 }; - writer.writeToLargeVarBinary(input); + writer.writeLargeVarBinary(input); byte[] result = vector.get(0); Assert.assertArrayEquals(input, result); } @@ -111,7 +111,7 @@ public void testWriteByteArrayWithOffsetToLargeVarBinary() throws Exception { try (LargeVarBinaryVector vector = new LargeVarBinaryVector("test", allocator); LargeVarBinaryWriter writer = new LargeVarBinaryWriterImpl(vector)) { byte[] input = new byte[] { 0x01, 0x02 }; - writer.writeToLargeVarBinary(input, 1, 1); + writer.writeLargeVarBinary(input, 1, 1); byte[] result = vector.get(0); Assert.assertArrayEquals(new byte[] { 0x02 }, result); } @@ -123,7 +123,7 @@ public void testWriteByteBufferToLargeVarBinary() throws Exception { LargeVarBinaryWriter writer = new LargeVarBinaryWriterImpl(vector)) { byte[] input = new byte[] { 0x01, 0x02 }; ByteBuffer buffer = ByteBuffer.wrap(input); - writer.writeToLargeVarBinary(buffer); + writer.writeLargeVarBinary(buffer); byte[] result = vector.get(0); Assert.assertArrayEquals(input, result); } @@ -135,7 +135,7 @@ public void testWriteByteBufferWithOffsetToLargeVarBinary() throws Exception { LargeVarBinaryWriter writer = new LargeVarBinaryWriterImpl(vector)) { byte[] input = new byte[] { 0x01, 0x02 }; ByteBuffer buffer = ByteBuffer.wrap(input); - writer.writeToLargeVarBinary(buffer, 1, 1); + writer.writeLargeVarBinary(buffer, 1, 1); byte[] result = vector.get(0); Assert.assertArrayEquals(new byte[] { 0x02 }, result); } From 8fcaba7c10ed84bea00351133d924407dec1e18a Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 9 Nov 2023 21:57:54 -0400 Subject: [PATCH 30/57] MINOR: [R] Fix broken link in README.md (#38657) ### Rationale for this change There is a broken link in README.md that refers to an old URL in the Arrow documentation. This causes the CRAN incoming check to fail. ### What changes are included in this PR? The link was updated. ### Are these changes tested? Not needed (docs only) ### Are there any user-facing changes? No Lead-authored-by: Dewey Dunnington Co-authored-by: Sutou Kouhei Signed-off-by: Dewey Dunnington --- r/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/README.md b/r/README.md index 3c1e3570ffdd4..72c1574417a20 100644 --- a/r/README.md +++ b/r/README.md @@ -101,7 +101,7 @@ issue you want to create. Add a meaningful title prefixed with **`[R]`** followed by a space, the issue summary and select component **R** from the dropdown list. For more information, see the **Report bugs and propose features** section of the [Contributing to Apache -Arrow](https://arrow.apache.org/docs/developers/contributing.html) page +Arrow](https://arrow.apache.org/docs/developers/#contributing) page in the Arrow developer documentation. We welcome questions, discussion, and contributions from users of the From c260a24fb17a78cde65aabec724a2dace408e5cc Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 9 Nov 2023 22:00:37 -0400 Subject: [PATCH 31/57] GH-38430: [R] Add test + fix corner cases after nixlibs.R refactor (#38534) ### Rationale for this change A few rough edges exist after https://github.com/apache/arrow/pull/38236 including: - When zero or 1 nightly with the matching major version exist, detection of the latest nightly might fail - At least one CI job is pulling nightlies instead of using the version from the current commit ### What changes are included in this PR? - Clean up `find_latest_nightly()` + add test - Ensure all CI jobs are not using `find_latest_nightly()` ### Are these changes tested? Yes (test added) ### Are there any user-facing changes? No * Closes: #38430 Lead-authored-by: Dewey Dunnington Co-authored-by: Jonathan Keane Signed-off-by: Dewey Dunnington --- r/tools/nixlibs.R | 46 ++++++++++++++++++++--------- r/tools/test-nixlibs.R | 65 +++++++++++++++++++++++++++++++++++++++++ r/vignettes/install.Rmd | 1 + 3 files changed, 98 insertions(+), 14 deletions(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 63c185ce54563..436eefcfcfe3e 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -35,7 +35,9 @@ exit <- function(..., .status = 1) { # checks the nightly repo for the latest nightly version X.Y.Z.100 -find_latest_nightly <- function(description_version) { +find_latest_nightly <- function(description_version, + list_uri = "https://nightlies.apache.org/arrow/r/src/contrib/PACKAGES", + hush = quietly) { if (!startsWith(arrow_repo, "https://nightlies.apache.org/arrow/r")) { lg("Detected non standard dev repo: %s, not checking latest nightly version.", arrow_repo) return(description_version) @@ -44,17 +46,28 @@ find_latest_nightly <- function(description_version) { res <- try( { # Binaries are only uploaded if all jobs pass so can just look at the source versions. - urls <- readLines("https://nightlies.apache.org/arrow/r/src/contrib") - versions <- grep("arrow_.*\\.tar\\.gz", urls, value = TRUE) - versions <- sub(".*arrow_(.*)\\.tar\\.gz.*", "\\1", x = versions) - versions <- sapply(versions, package_version) - versions <- data.frame(do.call(rbind, versions)) - matching_major <- versions[versions$X1 == description_version[1, 1], ] - latest <- matching_major[which.max(matching_major$X4), ] - package_version(paste0(latest, collapse = ".")) + urls <- readLines(list_uri) + versions <- grep("Version:\\s*.*?", urls, value = TRUE) + versions <- sort(package_version(sub("Version:\\s*", "\\1", versions))) + major_versions <- versions$major + + description_version_major <- as.integer(description_version[1, 1]) + matching_major <- major_versions == description_version_major + if (!any(matching_major)) { + lg( + "No nightly binaries were found for version %s: falling back to libarrow build from source", + description_version + ) + + return(description_version) + } + + versions <- versions[matching_major] + max(versions) }, - silent = quietly + silent = hush ) + if (inherits(res, "try-error")) { lg("Failed to find latest nightly for %s", description_version) latest <- description_version @@ -832,16 +845,19 @@ quietly <- !env_is("ARROW_R_DEV", "true") not_cran <- env_is("NOT_CRAN", "true") -if (is_release & !test_mode) { +if (is_release) { VERSION <- VERSION[1, 1:3] arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/") -} else if(!test_mode) { +} else { not_cran <- TRUE arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/") +} + +if (!is_release && !test_mode) { VERSION <- find_latest_nightly(VERSION) } -# To collect dirs to rm on exit, use del() to add dirs +# To collect dirs to rm on exit, use cleanup() to add dirs # we reset it to avoid errors on reruns in the same session. options(.arrow.cleanup = character()) on.exit(unlink(getOption(".arrow.cleanup"), recursive = TRUE), add = TRUE) @@ -867,6 +883,8 @@ build_ok <- !env_is("LIBARROW_BUILD", "false") # https://arrow.apache.org/docs/developers/cpp/building.html#offline-builds) download_ok <- !test_mode && !env_is("TEST_OFFLINE_BUILD", "true") +download_libarrow_ok <- download_ok && !env_is("LIBARROW_DOWNLOAD", "false") + # This "tools/thirdparty_dependencies" path, within the tar file, might exist if # create_package_with_all_dependencies() was run, or if someone has created it # manually before running make build. @@ -904,7 +922,7 @@ if (!test_mode && !file.exists(api_h)) { lg("File not found: %s ($ARROW_DOWNLOADED_BINARIES)", bin_zip) bin_file <- NULL } - } else if (download_ok) { + } else if (download_libarrow_ok) { binary_flavor <- identify_binary() if (!is.null(binary_flavor)) { # The env vars say we can, and we've determined a lib that should work diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index c9571b58b7bbf..f97a80ccc2917 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -155,3 +155,68 @@ test_that("check_allowlist", { expect_true(check_allowlist("redhat", tempfile())) # remote allowlist doesn't exist, so we fall back to the default list, which contains redhat expect_false(check_allowlist("debian", tempfile())) }) + +test_that("find_latest_nightly()", { + tf <- tempfile() + tf_uri <- paste0("file://", tf) + on.exit(unlink(tf)) + + writeLines( + c( + "Version: 13.0.0.100000333", + "Version: 13.0.0.100000334", + "Version: 13.0.0.100000335", + "Version: 14.0.0.100000001" + ), + tf + ) + + expect_output( + expect_identical( + find_latest_nightly(package_version("13.0.1.9000"), list_uri = tf_uri), + package_version("13.0.0.100000335") + ), + "Found latest nightly" + ) + + expect_output( + expect_identical( + find_latest_nightly(package_version("14.0.0.9000"), list_uri = tf_uri), + package_version("14.0.0.100000001") + ), + "Found latest nightly" + ) + + expect_output( + expect_identical( + find_latest_nightly(package_version("15.0.0.9000"), list_uri = tf_uri), + package_version("15.0.0.9000") + ), + "No nightly binaries were found for version" + ) + + # Check empty input + writeLines(character(), tf) + expect_output( + expect_identical( + find_latest_nightly(package_version("15.0.0.9000"), list_uri = tf_uri), + package_version("15.0.0.9000") + ), + "No nightly binaries were found for version" + ) + + # Check input that will throw an error + expect_output( + expect_identical( + suppressWarnings( + find_latest_nightly( + package_version("15.0.0.9000"), + list_uri = "this is not a URI", + hush = TRUE + ) + ), + package_version("15.0.0.9000") + ), + "Failed to find latest nightly" + ) +}) diff --git a/r/vignettes/install.Rmd b/r/vignettes/install.Rmd index 20fe4ed8961c6..10155e3a8cd5b 100644 --- a/r/vignettes/install.Rmd +++ b/r/vignettes/install.Rmd @@ -283,6 +283,7 @@ the bundled build script. All boolean variables are case-insensitive. | --- | --- | :-: | | `LIBARROW_BUILD` | Allow building from source | `true` | | `LIBARROW_BINARY` | Try to install `libarrow` binary instead of building from source | (unset) | +| `LIBARROW_DOWNLOAD` | Set to `false` to explicitly forbid fetching a `libarrow` binary | (unset) | | `LIBARROW_MINIMAL` | Build with minimal features enabled | (unset) | | `NOT_CRAN` | Set `LIBARROW_BINARY=true` and `LIBARROW_MINIMAL=false` | `false` | | `ARROW_R_DEV` | More verbose messaging and regenerates some code | `false` | From e5b2bba2229836f7b1b37a06d62081cb214d598a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 10 Nov 2023 16:21:57 +0100 Subject: [PATCH 32/57] MINOR: [Release] Update versions for 14.0.1-SNAPSHOT --- c_glib/meson.build | 2 +- cpp/CMakeLists.txt | 2 +- cpp/vcpkg.json | 2 +- csharp/Directory.Build.props | 2 +- .../homebrew-formulae/apache-arrow-glib.rb | 2 +- dev/tasks/homebrew-formulae/apache-arrow.rb | 2 +- go/README.md | 2 +- go/arrow/_examples/helloworld/main.go | 8 ++--- go/arrow/_tools/tmpl/main.go | 2 +- go/arrow/array.go | 4 +-- go/arrow/array/array.go | 6 ++-- go/arrow/array/array_test.go | 10 +++--- go/arrow/array/binary.go | 4 +-- go/arrow/array/binary_test.go | 6 ++-- go/arrow/array/binarybuilder.go | 8 ++--- go/arrow/array/binarybuilder_test.go | 6 ++-- go/arrow/array/boolean.go | 8 ++--- go/arrow/array/boolean_test.go | 4 +-- go/arrow/array/booleanbuilder.go | 10 +++--- go/arrow/array/booleanbuilder_test.go | 6 ++-- go/arrow/array/bufferbuilder.go | 6 ++-- go/arrow/array/bufferbuilder_byte.go | 2 +- go/arrow/array/bufferbuilder_numeric.gen.go | 6 ++-- .../array/bufferbuilder_numeric.gen.go.tmpl | 6 ++-- go/arrow/array/bufferbuilder_numeric_test.go | 4 +-- go/arrow/array/builder.go | 8 ++--- go/arrow/array/builder_test.go | 4 +-- go/arrow/array/compare.go | 6 ++-- go/arrow/array/compare_test.go | 10 +++--- go/arrow/array/concat.go | 14 ++++---- go/arrow/array/concat_test.go | 10 +++--- go/arrow/array/data.go | 6 ++-- go/arrow/array/data_test.go | 4 +-- go/arrow/array/decimal128.go | 12 +++---- go/arrow/array/decimal128_test.go | 8 ++--- go/arrow/array/decimal256.go | 12 +++---- go/arrow/array/decimal256_test.go | 8 ++--- go/arrow/array/decimal_test.go | 12 +++---- go/arrow/array/dictionary.go | 20 ++++++------ go/arrow/array/dictionary_test.go | 14 ++++---- go/arrow/array/diff.go | 2 +- go/arrow/array/diff_test.go | 10 +++--- go/arrow/array/encoded.go | 12 +++---- go/arrow/array/encoded_test.go | 8 ++--- go/arrow/array/extension.go | 6 ++-- go/arrow/array/extension_test.go | 8 ++--- go/arrow/array/fixed_size_list.go | 10 +++--- go/arrow/array/fixed_size_list_test.go | 6 ++-- go/arrow/array/fixedsize_binary.go | 4 +-- go/arrow/array/fixedsize_binary_test.go | 6 ++-- go/arrow/array/fixedsize_binarybuilder.go | 8 ++--- .../array/fixedsize_binarybuilder_test.go | 4 +-- go/arrow/array/float16.go | 6 ++-- go/arrow/array/float16_builder.go | 12 +++---- go/arrow/array/float16_builder_test.go | 6 ++-- go/arrow/array/interval.go | 10 +++--- go/arrow/array/interval_test.go | 6 ++-- go/arrow/array/json_reader.go | 8 ++--- go/arrow/array/json_reader_test.go | 6 ++-- go/arrow/array/list.go | 10 +++--- go/arrow/array/list_test.go | 6 ++-- go/arrow/array/map.go | 6 ++-- go/arrow/array/map_test.go | 6 ++-- go/arrow/array/null.go | 8 ++--- go/arrow/array/null_test.go | 6 ++-- go/arrow/array/numeric.gen.go | 4 +-- go/arrow/array/numeric.gen.go.tmpl | 4 +-- go/arrow/array/numeric_test.go | 8 ++--- go/arrow/array/numericbuilder.gen.go | 10 +++--- go/arrow/array/numericbuilder.gen.go.tmpl | 10 +++--- go/arrow/array/numericbuilder.gen_test.go | 6 ++-- .../array/numericbuilder.gen_test.go.tmpl | 6 ++-- go/arrow/array/record.go | 8 ++--- go/arrow/array/record_test.go | 6 ++-- go/arrow/array/string.go | 6 ++-- go/arrow/array/string_test.go | 8 ++--- go/arrow/array/struct.go | 10 +++--- go/arrow/array/struct_test.go | 6 ++-- go/arrow/array/table.go | 4 +-- go/arrow/array/table_test.go | 6 ++-- go/arrow/array/timestamp.go | 10 +++--- go/arrow/array/timestamp_test.go | 6 ++-- go/arrow/array/union.go | 12 +++---- go/arrow/array/union_test.go | 6 ++-- go/arrow/array/util.go | 10 +++--- go/arrow/array/util_test.go | 14 ++++---- go/arrow/arrio/arrio.go | 2 +- go/arrow/arrio/arrio_test.go | 10 +++--- go/arrow/bitutil/bitmaps.go | 6 ++-- go/arrow/bitutil/bitmaps_test.go | 4 +-- go/arrow/bitutil/bitutil.go | 2 +- go/arrow/bitutil/bitutil_test.go | 4 +-- go/arrow/cdata/cdata.go | 8 ++--- go/arrow/cdata/cdata_exports.go | 10 +++--- go/arrow/cdata/cdata_test.go | 12 +++---- go/arrow/cdata/cdata_test_framework.go | 8 ++--- go/arrow/cdata/exports.go | 4 +-- go/arrow/cdata/import_allocator.go | 2 +- go/arrow/cdata/interface.go | 8 ++--- go/arrow/cdata/test/test_cimport.go | 8 ++--- go/arrow/compute/arithmetic.go | 12 +++---- go/arrow/compute/arithmetic_test.go | 20 ++++++------ go/arrow/compute/cast.go | 10 +++--- go/arrow/compute/cast_test.go | 20 ++++++------ go/arrow/compute/datum.go | 6 ++-- go/arrow/compute/example_test.go | 10 +++--- go/arrow/compute/exec.go | 6 ++-- go/arrow/compute/exec/kernel.go | 8 ++--- go/arrow/compute/exec/kernel_test.go | 12 +++---- go/arrow/compute/exec/span.go | 10 +++--- go/arrow/compute/exec/span_test.go | 16 +++++----- go/arrow/compute/exec/utils.go | 14 ++++---- go/arrow/compute/exec/utils_test.go | 8 ++--- go/arrow/compute/exec_internals_test.go | 14 ++++---- go/arrow/compute/exec_test.go | 12 +++---- go/arrow/compute/executor.go | 16 +++++----- go/arrow/compute/expression.go | 16 +++++----- go/arrow/compute/expression_test.go | 10 +++--- go/arrow/compute/exprs/builders.go | 4 +-- go/arrow/compute/exprs/builders_test.go | 4 +-- go/arrow/compute/exprs/exec.go | 18 +++++------ go/arrow/compute/exprs/exec_internal_test.go | 8 ++--- go/arrow/compute/exprs/exec_test.go | 12 +++---- go/arrow/compute/exprs/extension_types.go | 4 +-- go/arrow/compute/exprs/field_refs.go | 10 +++--- go/arrow/compute/exprs/types.go | 4 +-- go/arrow/compute/fieldref.go | 4 +-- go/arrow/compute/fieldref_test.go | 8 ++--- go/arrow/compute/functions.go | 4 +-- go/arrow/compute/functions_test.go | 4 +-- .../internal/kernels/base_arithmetic.go | 10 +++--- .../internal/kernels/base_arithmetic_amd64.go | 4 +-- .../kernels/base_arithmetic_avx2_amd64.go | 2 +- .../kernels/base_arithmetic_sse4_amd64.go | 2 +- .../kernels/basic_arithmetic_noasm.go | 2 +- .../compute/internal/kernels/boolean_cast.go | 6 ++-- go/arrow/compute/internal/kernels/cast.go | 6 ++-- .../compute/internal/kernels/cast_numeric.go | 2 +- .../kernels/cast_numeric_avx2_amd64.go | 2 +- .../kernels/cast_numeric_neon_arm64.go | 2 +- .../kernels/cast_numeric_sse4_amd64.go | 2 +- .../compute/internal/kernels/cast_temporal.go | 8 ++--- go/arrow/compute/internal/kernels/helpers.go | 14 ++++---- .../compute/internal/kernels/numeric_cast.go | 14 ++++---- go/arrow/compute/internal/kernels/rounding.go | 10 +++--- .../internal/kernels/scalar_arithmetic.go | 14 ++++---- .../internal/kernels/scalar_boolean.go | 6 ++-- .../kernels/scalar_comparison_amd64.go | 4 +-- .../kernels/scalar_comparison_avx2_amd64.go | 2 +- .../kernels/scalar_comparison_noasm.go | 2 +- .../kernels/scalar_comparison_sse4_amd64.go | 2 +- .../internal/kernels/scalar_comparisons.go | 16 +++++----- .../compute/internal/kernels/string_casts.go | 12 +++---- go/arrow/compute/internal/kernels/types.go | 8 ++--- .../compute/internal/kernels/vector_hash.go | 14 ++++---- .../internal/kernels/vector_run_end_encode.go | 16 +++++----- .../internal/kernels/vector_selection.go | 14 ++++---- go/arrow/compute/registry.go | 2 +- go/arrow/compute/registry_test.go | 6 ++-- go/arrow/compute/scalar_bool.go | 6 ++-- go/arrow/compute/scalar_bool_test.go | 10 +++--- go/arrow/compute/scalar_compare.go | 6 ++-- go/arrow/compute/scalar_compare_test.go | 18 +++++------ go/arrow/compute/selection.go | 8 ++--- go/arrow/compute/utils.go | 12 +++---- go/arrow/compute/vector_hash.go | 4 +-- go/arrow/compute/vector_hash_test.go | 14 ++++---- go/arrow/compute/vector_run_end_test.go | 14 ++++---- go/arrow/compute/vector_run_ends.go | 4 +-- go/arrow/compute/vector_selection_test.go | 18 +++++------ go/arrow/csv/common.go | 4 +-- go/arrow/csv/reader.go | 14 ++++---- go/arrow/csv/reader_test.go | 14 ++++---- go/arrow/csv/transformer.go | 4 +-- go/arrow/csv/writer.go | 2 +- go/arrow/csv/writer_test.go | 16 +++++----- go/arrow/datatype.go | 2 +- go/arrow/datatype_binary_test.go | 2 +- go/arrow/datatype_extension_test.go | 4 +-- go/arrow/datatype_fixedwidth.go | 2 +- go/arrow/datatype_fixedwidth_test.go | 2 +- go/arrow/datatype_nested.go | 2 +- go/arrow/datatype_null_test.go | 2 +- go/arrow/decimal128/decimal128.go | 2 +- go/arrow/decimal128/decimal128_test.go | 2 +- go/arrow/decimal256/decimal256.go | 4 +-- go/arrow/decimal256/decimal256_test.go | 2 +- go/arrow/doc.go | 2 +- go/arrow/encoded/ree_utils.go | 2 +- go/arrow/encoded/ree_utils_test.go | 8 ++--- go/arrow/endian/endian.go | 4 +-- go/arrow/example_test.go | 8 ++--- go/arrow/flight/basic_auth_flight_test.go | 2 +- go/arrow/flight/client.go | 2 +- go/arrow/flight/cookie_middleware_test.go | 2 +- go/arrow/flight/example_flight_server_test.go | 2 +- go/arrow/flight/flight_middleware_test.go | 4 +-- go/arrow/flight/flight_test.go | 10 +++--- go/arrow/flight/flightsql/client.go | 12 +++---- go/arrow/flight/flightsql/client_test.go | 12 +++---- go/arrow/flight/flightsql/column_metadata.go | 2 +- go/arrow/flight/flightsql/driver/README.md | 6 ++-- .../flight/flightsql/driver/config_test.go | 2 +- go/arrow/flight/flightsql/driver/driver.go | 10 +++--- .../flight/flightsql/driver/driver_test.go | 14 ++++---- go/arrow/flight/flightsql/driver/utils.go | 4 +-- .../flight/flightsql/driver/utils_test.go | 12 +++---- .../cmd/sqlite_flightsql_server/main.go | 6 ++-- .../flightsql/example/sql_batch_reader.go | 10 +++--- .../flight/flightsql/example/sqlite_info.go | 4 +-- .../flight/flightsql/example/sqlite_server.go | 14 ++++---- .../sqlite_tables_schema_batch_reader.go | 12 +++---- .../flight/flightsql/example/type_info.go | 8 ++--- .../flightsql/schema_ref/reference_schemas.go | 2 +- go/arrow/flight/flightsql/server.go | 16 +++++----- go/arrow/flight/flightsql/server_test.go | 12 +++---- go/arrow/flight/flightsql/sql_info.go | 4 +-- .../flight/flightsql/sqlite_server_test.go | 16 +++++----- go/arrow/flight/flightsql/types.go | 2 +- go/arrow/flight/record_batch_reader.go | 12 +++---- go/arrow/flight/record_batch_writer.go | 6 ++-- go/arrow/flight/server.go | 2 +- go/arrow/flight/server_example_test.go | 2 +- go/arrow/internal/arrdata/arrdata.go | 16 +++++----- go/arrow/internal/arrdata/ioutil.go | 10 +++--- go/arrow/internal/arrjson/arrjson.go | 20 ++++++------ go/arrow/internal/arrjson/arrjson_test.go | 6 ++-- go/arrow/internal/arrjson/option.go | 4 +-- go/arrow/internal/arrjson/reader.go | 10 +++--- go/arrow/internal/arrjson/writer.go | 10 +++--- .../internal/cdata_integration/entrypoints.go | 8 ++--- go/arrow/internal/dictutils/dict.go | 6 ++-- go/arrow/internal/dictutils/dict_test.go | 8 ++--- .../arrow-flight-integration-client/main.go | 2 +- .../arrow-flight-integration-server/main.go | 2 +- .../internal/flight_integration/scenario.go | 18 +++++------ .../internal/testing/gen/random_array_gen.go | 10 +++--- go/arrow/internal/testing/tools/bits_test.go | 2 +- go/arrow/internal/testing/tools/data_types.go | 4 +-- go/arrow/internal/utils.go | 4 +-- go/arrow/ipc/cmd/arrow-cat/main.go | 4 +-- go/arrow/ipc/cmd/arrow-cat/main_test.go | 8 ++--- go/arrow/ipc/cmd/arrow-file-to-stream/main.go | 6 ++-- .../ipc/cmd/arrow-file-to-stream/main_test.go | 4 +-- .../cmd/arrow-json-integration-test/main.go | 12 +++---- .../arrow-json-integration-test/main_test.go | 4 +-- go/arrow/ipc/cmd/arrow-ls/main.go | 4 +-- go/arrow/ipc/cmd/arrow-ls/main_test.go | 8 ++--- go/arrow/ipc/cmd/arrow-stream-to-file/main.go | 6 ++-- .../ipc/cmd/arrow-stream-to-file/main_test.go | 4 +-- go/arrow/ipc/compression.go | 6 ++-- go/arrow/ipc/endian_swap.go | 6 ++-- go/arrow/ipc/endian_swap_test.go | 10 +++--- go/arrow/ipc/file_reader.go | 16 +++++----- go/arrow/ipc/file_test.go | 6 ++-- go/arrow/ipc/file_writer.go | 10 +++--- go/arrow/ipc/ipc.go | 8 ++--- go/arrow/ipc/ipc_test.go | 8 ++--- go/arrow/ipc/message.go | 6 ++-- go/arrow/ipc/message_test.go | 6 ++-- go/arrow/ipc/metadata.go | 10 +++--- go/arrow/ipc/metadata_test.go | 12 +++---- go/arrow/ipc/reader.go | 14 ++++---- go/arrow/ipc/reader_test.go | 6 ++-- go/arrow/ipc/stream_test.go | 6 ++-- go/arrow/ipc/writer.go | 16 +++++----- go/arrow/ipc/writer_test.go | 10 +++--- go/arrow/math/float64.go | 2 +- go/arrow/math/float64_avx2_amd64.go | 2 +- go/arrow/math/float64_neon_arm64.go | 2 +- go/arrow/math/float64_sse4_amd64.go | 2 +- go/arrow/math/float64_test.go | 6 ++-- go/arrow/math/int64.go | 2 +- go/arrow/math/int64_avx2_amd64.go | 2 +- go/arrow/math/int64_neon_arm64.go | 2 +- go/arrow/math/int64_sse4_amd64.go | 2 +- go/arrow/math/int64_test.go | 6 ++-- go/arrow/math/type.go.tmpl | 2 +- go/arrow/math/type_simd_amd64.go.tmpl | 2 +- go/arrow/math/type_simd_arm64.go.tmpl | 2 +- go/arrow/math/type_test.go.tmpl | 6 ++-- go/arrow/math/uint64.go | 2 +- go/arrow/math/uint64_avx2_amd64.go | 2 +- go/arrow/math/uint64_neon_arm64.go | 2 +- go/arrow/math/uint64_sse4_amd64.go | 2 +- go/arrow/math/uint64_test.go | 6 ++-- go/arrow/memory/buffer.go | 2 +- go/arrow/memory/buffer_test.go | 2 +- go/arrow/memory/cgo_allocator.go | 2 +- go/arrow/memory/default_mallocator.go | 2 +- go/arrow/memory/default_mallocator_test.go | 4 +-- go/arrow/memory/mallocator/mallocator_test.go | 2 +- go/arrow/memory/memory_test.go | 2 +- go/arrow/record.go | 2 +- go/arrow/scalar/append.go | 10 +++--- go/arrow/scalar/append_test.go | 10 +++--- go/arrow/scalar/binary.go | 4 +-- go/arrow/scalar/compare.go | 2 +- go/arrow/scalar/nested.go | 8 ++--- go/arrow/scalar/numeric.gen.go | 6 ++-- go/arrow/scalar/numeric.gen_test.go | 4 +-- go/arrow/scalar/numeric.gen_test.go.tmpl | 4 +-- go/arrow/scalar/parse.go | 12 +++---- go/arrow/scalar/scalar.go | 20 ++++++------ go/arrow/scalar/scalar_test.go | 12 +++---- go/arrow/scalar/temporal.go | 2 +- go/arrow/schema.go | 2 +- go/arrow/schema_test.go | 2 +- go/arrow/table.go | 2 +- go/arrow/tensor/numeric.gen.go | 2 +- go/arrow/tensor/numeric.gen.go.tmpl | 4 +-- go/arrow/tensor/numeric.gen_test.go | 8 ++--- go/arrow/tensor/numeric.gen_test.go.tmpl | 8 ++--- go/arrow/tensor/tensor.go | 4 +-- go/arrow/tensor/tensor_test.go | 8 ++--- go/arrow/type_traits_boolean.go | 2 +- go/arrow/type_traits_decimal128.go | 4 +-- go/arrow/type_traits_decimal256.go | 4 +-- go/arrow/type_traits_float16.go | 4 +-- go/arrow/type_traits_interval.go | 4 +-- go/arrow/type_traits_numeric.gen.go | 2 +- go/arrow/type_traits_numeric.gen.go.tmpl | 2 +- go/arrow/type_traits_numeric.gen_test.go | 2 +- go/arrow/type_traits_numeric.gen_test.go.tmpl | 2 +- go/arrow/type_traits_test.go | 8 ++--- go/arrow/type_traits_timestamp.go | 2 +- go/arrow/util/byte_size.go | 6 ++-- go/arrow/util/byte_size_test.go | 8 ++--- go/go.mod | 2 +- go/internal/bitutils/bit_block_counter.go | 4 +-- .../bitutils/bit_block_counter_test.go | 6 ++-- go/internal/bitutils/bit_run_reader.go | 6 ++-- go/internal/bitutils/bit_run_reader_test.go | 6 ++-- go/internal/bitutils/bit_set_run_reader.go | 4 +-- .../bitutils/bit_set_run_reader_test.go | 6 ++-- go/internal/bitutils/bitmap_generate.go | 2 +- go/internal/bitutils/bitmap_generate_test.go | 2 +- go/internal/hashing/xxh3_memo_table.gen.go | 6 ++-- .../hashing/xxh3_memo_table.gen.go.tmpl | 4 +-- go/internal/types/extension_types.go | 6 ++-- go/internal/types/extension_types_test.go | 10 +++--- go/internal/utils/transpose_ints_def.go | 2 +- go/internal/utils/transpose_ints_test.go | 2 +- go/parquet/cmd/parquet_reader/dumper.go | 6 ++-- go/parquet/cmd/parquet_reader/main.go | 10 +++--- go/parquet/cmd/parquet_schema/main.go | 4 +-- go/parquet/compress/brotli.go | 2 +- go/parquet/compress/compress.go | 2 +- go/parquet/compress/compress_test.go | 2 +- go/parquet/compress/zstd.go | 2 +- go/parquet/doc.go | 6 ++-- go/parquet/encryption_properties.go | 2 +- go/parquet/encryption_properties_test.go | 4 +-- go/parquet/encryption_read_config_test.go | 8 ++--- go/parquet/encryption_write_config_test.go | 8 ++--- go/parquet/file/column_reader.go | 14 ++++---- go/parquet/file/column_reader_test.go | 12 +++---- go/parquet/file/column_reader_types.gen.go | 6 ++-- .../file/column_reader_types.gen.go.tmpl | 4 +-- go/parquet/file/column_writer.go | 16 +++++----- go/parquet/file/column_writer_test.go | 26 +++++++-------- go/parquet/file/column_writer_types.gen.go | 12 +++---- .../file/column_writer_types.gen.go.tmpl | 8 ++--- go/parquet/file/file_reader.go | 8 ++--- go/parquet/file/file_reader_mmap.go | 2 +- go/parquet/file/file_reader_mmap_windows.go | 2 +- go/parquet/file/file_reader_test.go | 20 ++++++------ go/parquet/file/file_writer.go | 10 +++--- go/parquet/file/file_writer_test.go | 14 ++++---- go/parquet/file/level_conversion.go | 10 +++--- go/parquet/file/level_conversion_test.go | 6 ++-- go/parquet/file/page_reader.go | 14 ++++---- go/parquet/file/page_writer.go | 18 +++++------ go/parquet/file/record_reader.go | 16 +++++----- go/parquet/file/row_group_reader.go | 8 ++--- go/parquet/file/row_group_writer.go | 8 ++--- go/parquet/file/row_group_writer_test.go | 8 ++--- go/parquet/internal/bmi/bmi_test.go | 2 +- .../internal/encoding/boolean_decoder.go | 8 ++--- .../internal/encoding/boolean_encoder.go | 8 ++--- .../internal/encoding/byte_array_decoder.go | 12 +++---- .../internal/encoding/byte_array_encoder.go | 10 +++--- go/parquet/internal/encoding/decoder.go | 20 ++++++------ .../internal/encoding/delta_bit_packing.go | 10 +++--- .../internal/encoding/delta_byte_array.go | 6 ++-- .../encoding/delta_byte_array_test.go | 4 +-- .../encoding/delta_length_byte_array.go | 6 ++-- go/parquet/internal/encoding/encoder.go | 16 +++++----- .../encoding/encoding_benchmarks_test.go | 16 +++++----- go/parquet/internal/encoding/encoding_test.go | 14 ++++---- .../encoding/fixed_len_byte_array_decoder.go | 4 +-- .../encoding/fixed_len_byte_array_encoder.go | 6 ++-- go/parquet/internal/encoding/levels.go | 10 +++--- go/parquet/internal/encoding/levels_test.go | 10 +++--- go/parquet/internal/encoding/memo_table.go | 10 +++--- .../internal/encoding/memo_table_test.go | 10 +++--- .../internal/encoding/memo_table_types.gen.go | 4 +-- .../encoding/memo_table_types.gen.go.tmpl | 2 +- .../encoding/plain_encoder_types.gen.go | 10 +++--- .../encoding/plain_encoder_types.gen.go.tmpl | 8 ++--- .../internal/encoding/typed_encoder.gen.go | 18 +++++------ .../encoding/typed_encoder.gen.go.tmpl | 14 ++++---- go/parquet/internal/encoding/types.go | 10 +++--- go/parquet/internal/encryption/aes.go | 2 +- go/parquet/internal/encryption/decryptor.go | 4 +-- go/parquet/internal/encryption/encryptor.go | 4 +-- go/parquet/internal/testutils/pagebuilder.go | 14 ++++---- .../internal/testutils/primitive_typed.go | 10 +++--- go/parquet/internal/testutils/random.go | 14 ++++---- go/parquet/internal/testutils/random_arrow.go | 6 ++-- go/parquet/internal/testutils/utils.go | 2 +- go/parquet/internal/thrift/helpers.go | 2 +- .../internal/utils/bit_benchmark_test.go | 6 ++-- .../internal/utils/bit_packing_arm64.go | 2 +- go/parquet/internal/utils/bit_reader.go | 8 ++--- go/parquet/internal/utils/bit_reader_test.go | 10 +++--- go/parquet/internal/utils/bit_writer.go | 2 +- go/parquet/internal/utils/bitmap_writer.go | 2 +- .../internal/utils/bitmap_writer_test.go | 4 +-- go/parquet/internal/utils/rle.go | 8 ++--- .../internal/utils/typed_rle_dict.gen.go | 6 ++-- .../internal/utils/typed_rle_dict.gen.go.tmpl | 6 ++-- go/parquet/metadata/app_version.go | 4 +-- go/parquet/metadata/column_chunk.go | 14 ++++---- go/parquet/metadata/file.go | 12 +++---- go/parquet/metadata/metadata_test.go | 6 ++-- go/parquet/metadata/row_group.go | 8 ++--- go/parquet/metadata/stat_compare_test.go | 4 +-- go/parquet/metadata/statistics.go | 16 +++++----- go/parquet/metadata/statistics_test.go | 10 +++--- go/parquet/metadata/statistics_types.gen.go | 16 +++++----- .../metadata/statistics_types.gen.go.tmpl | 14 ++++---- go/parquet/pqarrow/column_readers.go | 20 ++++++------ go/parquet/pqarrow/encode_arrow.go | 20 ++++++------ go/parquet/pqarrow/encode_arrow_test.go | 32 +++++++++---------- go/parquet/pqarrow/encode_dict_compute.go | 16 +++++----- go/parquet/pqarrow/encode_dict_nocompute.go | 4 +-- go/parquet/pqarrow/encode_dictionary_test.go | 16 +++++----- go/parquet/pqarrow/file_reader.go | 14 ++++---- go/parquet/pqarrow/file_reader_test.go | 14 ++++---- go/parquet/pqarrow/file_writer.go | 12 +++---- go/parquet/pqarrow/helpers.go | 2 +- go/parquet/pqarrow/path_builder.go | 10 +++--- go/parquet/pqarrow/path_builder_test.go | 8 ++--- go/parquet/pqarrow/properties.go | 6 ++-- go/parquet/pqarrow/reader_writer_test.go | 12 +++---- go/parquet/pqarrow/schema.go | 18 +++++------ go/parquet/pqarrow/schema_test.go | 18 +++++------ go/parquet/reader_properties.go | 4 +-- go/parquet/reader_writer_properties_test.go | 6 ++-- go/parquet/schema/column.go | 4 +-- go/parquet/schema/converted_types.go | 2 +- go/parquet/schema/converted_types_test.go | 2 +- go/parquet/schema/helpers.go | 2 +- go/parquet/schema/helpers_test.go | 4 +-- go/parquet/schema/logical_types.go | 8 ++--- go/parquet/schema/logical_types_test.go | 6 ++-- go/parquet/schema/node.go | 4 +-- go/parquet/schema/reflection.go | 4 +-- go/parquet/schema/reflection_test.go | 4 +-- go/parquet/schema/schema.go | 4 +-- go/parquet/schema/schema_element_test.go | 4 +-- go/parquet/schema/schema_flatten_test.go | 4 +-- go/parquet/schema/schema_test.go | 6 ++-- go/parquet/types.go | 4 +-- go/parquet/writer_properties.go | 6 ++-- java/adapter/avro/pom.xml | 2 +- java/adapter/jdbc/pom.xml | 2 +- java/adapter/orc/pom.xml | 2 +- java/algorithm/pom.xml | 2 +- java/bom/pom.xml | 2 +- java/c/pom.xml | 2 +- java/compression/pom.xml | 2 +- java/dataset/pom.xml | 2 +- java/flight/flight-core/pom.xml | 2 +- java/flight/flight-grpc/pom.xml | 2 +- java/flight/flight-integration-tests/pom.xml | 2 +- java/flight/flight-sql-jdbc-core/pom.xml | 2 +- java/flight/flight-sql-jdbc-driver/pom.xml | 2 +- java/flight/flight-sql/pom.xml | 2 +- java/flight/pom.xml | 2 +- java/format/pom.xml | 2 +- java/gandiva/pom.xml | 2 +- java/memory/memory-core/pom.xml | 2 +- java/memory/memory-netty/pom.xml | 2 +- java/memory/memory-unsafe/pom.xml | 2 +- java/memory/pom.xml | 2 +- java/performance/pom.xml | 2 +- java/pom.xml | 2 +- java/tools/pom.xml | 2 +- java/vector/pom.xml | 2 +- js/package.json | 2 +- matlab/CMakeLists.txt | 2 +- python/CMakeLists.txt | 2 +- python/setup.py | 2 +- r/NEWS.md | 2 ++ ruby/red-arrow-cuda/lib/arrow-cuda/version.rb | 2 +- .../lib/arrow-dataset/version.rb | 2 +- .../lib/arrow-flight-sql/version.rb | 2 +- .../lib/arrow-flight/version.rb | 2 +- ruby/red-arrow/lib/arrow/version.rb | 2 +- ruby/red-gandiva/lib/gandiva/version.rb | 2 +- ruby/red-parquet/lib/parquet/version.rb | 2 +- 503 files changed, 1732 insertions(+), 1730 deletions(-) diff --git a/c_glib/meson.build b/c_glib/meson.build index 7c495d2567d72..1aba855703828 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -24,7 +24,7 @@ project('arrow-glib', 'c', 'cpp', 'cpp_std=c++17', ]) -version = '15.0.0-SNAPSHOT' +version = '14.0.1-SNAPSHOT' if version.endswith('-SNAPSHOT') version_numbers = version.split('-')[0].split('.') version_tag = version.split('-')[1] diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bcb298407bd8b..d57fcebb498dc 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -71,7 +71,7 @@ if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() -set(ARROW_VERSION "15.0.0-SNAPSHOT") +set(ARROW_VERSION "14.0.1-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}") diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index c0bf5dce50e32..d1508658b8165 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -1,6 +1,6 @@ { "name": "arrow", - "version-string": "15.0.0-SNAPSHOT", + "version-string": "14.0.1-SNAPSHOT", "dependencies": [ "abseil", { diff --git a/csharp/Directory.Build.props b/csharp/Directory.Build.props index ae6edda0e2f0e..1047272ffd448 100644 --- a/csharp/Directory.Build.props +++ b/csharp/Directory.Build.props @@ -29,7 +29,7 @@ Apache Arrow library Copyright 2016-2019 The Apache Software Foundation The Apache Software Foundation - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT diff --git a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb index e29354def4c1c..e9fca5bc74216 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb @@ -29,7 +29,7 @@ class ApacheArrowGlib < Formula desc "GLib bindings for Apache Arrow" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-15.0.0-SNAPSHOT/apache-arrow-15.0.0-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-14.0.1-SNAPSHOT/apache-arrow-14.0.1-SNAPSHOT.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index 14d229b477dc8..ff7e54ba64bc0 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -29,7 +29,7 @@ class ApacheArrow < Formula desc "Columnar in-memory analytics layer designed to accelerate big data" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-15.0.0-SNAPSHOT/apache-arrow-15.0.0-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-14.0.1-SNAPSHOT/apache-arrow-14.0.1-SNAPSHOT.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/go/README.md b/go/README.md index c45bcd756f81c..660549cb1b366 100644 --- a/go/README.md +++ b/go/README.md @@ -20,7 +20,7 @@ Apache Arrow for Go =================== -[![Go Reference](https://pkg.go.dev/badge/github.com/apache/arrow/go/v15.svg)](https://pkg.go.dev/github.com/apache/arrow/go/v15) +[![Go Reference](https://pkg.go.dev/badge/github.com/apache/arrow/go/v14.svg)](https://pkg.go.dev/github.com/apache/arrow/go/v14) [Apache Arrow][arrow] is a cross-language development platform for in-memory data. It specifies a standardized language-independent columnar memory format diff --git a/go/arrow/_examples/helloworld/main.go b/go/arrow/_examples/helloworld/main.go index 156a52b86da1c..02601be694400 100644 --- a/go/arrow/_examples/helloworld/main.go +++ b/go/arrow/_examples/helloworld/main.go @@ -19,10 +19,10 @@ package main import ( "os" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/math" + "github.com/apache/arrow/go/v14/arrow/memory" ) func main() { diff --git a/go/arrow/_tools/tmpl/main.go b/go/arrow/_tools/tmpl/main.go index c591c3b96803c..87c4b6ed02c12 100644 --- a/go/arrow/_tools/tmpl/main.go +++ b/go/arrow/_tools/tmpl/main.go @@ -28,7 +28,7 @@ import ( "strings" "text/template" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/internal/json" ) const Ext = ".tmpl" diff --git a/go/arrow/array.go b/go/arrow/array.go index e07fa478aae57..7622e7503e0b6 100644 --- a/go/arrow/array.go +++ b/go/arrow/array.go @@ -19,8 +19,8 @@ package arrow import ( "fmt" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // ArrayData is the underlying memory and metadata of an Arrow array, corresponding diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go index bbe301ee661f3..1ee04c7aa2bcc 100644 --- a/go/arrow/array/array.go +++ b/go/arrow/array/array.go @@ -19,9 +19,9 @@ package array import ( "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) const ( diff --git a/go/arrow/array/array_test.go b/go/arrow/array/array_test.go index bbfbee83585da..6139548ddec7d 100644 --- a/go/arrow/array/array_test.go +++ b/go/arrow/array/array_test.go @@ -19,11 +19,11 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/binary.go b/go/arrow/array/binary.go index bf27139fddbaa..e9e6e66e7e813 100644 --- a/go/arrow/array/binary.go +++ b/go/arrow/array/binary.go @@ -23,8 +23,8 @@ import ( "strings" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/internal/json" ) type BinaryLike interface { diff --git a/go/arrow/array/binary_test.go b/go/arrow/array/binary_test.go index 9c1770950a8b5..5febb55691d7d 100644 --- a/go/arrow/array/binary_test.go +++ b/go/arrow/array/binary_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/binarybuilder.go b/go/arrow/array/binarybuilder.go index a51bc799e4965..3cb709b45b7a1 100644 --- a/go/arrow/array/binarybuilder.go +++ b/go/arrow/array/binarybuilder.go @@ -24,10 +24,10 @@ import ( "reflect" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // A BinaryBuilder is used to build a Binary array using the Append methods. diff --git a/go/arrow/array/binarybuilder_test.go b/go/arrow/array/binarybuilder_test.go index 96be73da6516d..c63307433bcca 100644 --- a/go/arrow/array/binarybuilder_test.go +++ b/go/arrow/array/binarybuilder_test.go @@ -20,9 +20,9 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go index 43bac64a4c990..464cef48bdf32 100644 --- a/go/arrow/array/boolean.go +++ b/go/arrow/array/boolean.go @@ -21,10 +21,10 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // A type which represents an immutable sequence of boolean values. diff --git a/go/arrow/array/boolean_test.go b/go/arrow/array/boolean_test.go index bcd17ee5967d6..c49ec08624cc6 100644 --- a/go/arrow/array/boolean_test.go +++ b/go/arrow/array/boolean_test.go @@ -22,8 +22,8 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/booleanbuilder.go b/go/arrow/array/booleanbuilder.go index cd0cffd5e43e2..10b7405aa5b2a 100644 --- a/go/arrow/array/booleanbuilder.go +++ b/go/arrow/array/booleanbuilder.go @@ -23,11 +23,11 @@ import ( "strconv" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) type BooleanBuilder struct { diff --git a/go/arrow/array/booleanbuilder_test.go b/go/arrow/array/booleanbuilder_test.go index f6f6c80dab37f..e270636a87b1c 100644 --- a/go/arrow/array/booleanbuilder_test.go +++ b/go/arrow/array/booleanbuilder_test.go @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/bufferbuilder.go b/go/arrow/array/bufferbuilder.go index cb381e25b32a2..e023b0d907421 100644 --- a/go/arrow/array/bufferbuilder.go +++ b/go/arrow/array/bufferbuilder.go @@ -19,9 +19,9 @@ package array import ( "sync/atomic" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" ) type bufBuilder interface { diff --git a/go/arrow/array/bufferbuilder_byte.go b/go/arrow/array/bufferbuilder_byte.go index 9b2b559ba9f68..00a0d1c21b3ef 100644 --- a/go/arrow/array/bufferbuilder_byte.go +++ b/go/arrow/array/bufferbuilder_byte.go @@ -16,7 +16,7 @@ package array -import "github.com/apache/arrow/go/v15/arrow/memory" +import "github.com/apache/arrow/go/v14/arrow/memory" type byteBufferBuilder struct { bufferBuilder diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go b/go/arrow/array/bufferbuilder_numeric.gen.go index a7961166c0edd..879bc9f571152 100644 --- a/go/arrow/array/bufferbuilder_numeric.gen.go +++ b/go/arrow/array/bufferbuilder_numeric.gen.go @@ -19,9 +19,9 @@ package array import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" ) type int64BufferBuilder struct { diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl b/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl index 845d7ef01c89a..e859b5bff02a1 100644 --- a/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl +++ b/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl @@ -17,9 +17,9 @@ package array import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" ) {{range .In}} diff --git a/go/arrow/array/bufferbuilder_numeric_test.go b/go/arrow/array/bufferbuilder_numeric_test.go index b51e9ae9207ea..ba0a60e5b703b 100644 --- a/go/arrow/array/bufferbuilder_numeric_test.go +++ b/go/arrow/array/bufferbuilder_numeric_test.go @@ -20,8 +20,8 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/endian" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/builder.go b/go/arrow/array/builder.go index bb15298e03ccf..2f15ac965e07c 100644 --- a/go/arrow/array/builder.go +++ b/go/arrow/array/builder.go @@ -20,10 +20,10 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) const ( diff --git a/go/arrow/array/builder_test.go b/go/arrow/array/builder_test.go index 7bec86d86cc8b..3cacb54f725e7 100644 --- a/go/arrow/array/builder_test.go +++ b/go/arrow/array/builder_test.go @@ -19,8 +19,8 @@ package array import ( "testing" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/compare.go b/go/arrow/array/compare.go index 778de41e32c67..e70716bee91a7 100644 --- a/go/arrow/array/compare.go +++ b/go/arrow/array/compare.go @@ -20,9 +20,9 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/internal/bitutils" ) // RecordEqual reports whether the two provided records are equal. diff --git a/go/arrow/array/compare_test.go b/go/arrow/array/compare_test.go index 4fc9cf50e8643..51421aa04fe9a 100644 --- a/go/arrow/array/compare_test.go +++ b/go/arrow/array/compare_test.go @@ -22,11 +22,11 @@ import ( "sort" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/concat.go b/go/arrow/array/concat.go index 53c5be06895b9..9d815023c4b76 100644 --- a/go/arrow/array/concat.go +++ b/go/arrow/array/concat.go @@ -23,13 +23,13 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/encoded" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/encoded" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/utils" ) // Concatenate creates a new arrow.Array which is the concatenation of the diff --git a/go/arrow/array/concat_test.go b/go/arrow/array/concat_test.go index 1cc484ad1a923..c80844f05bacd 100644 --- a/go/arrow/array/concat_test.go +++ b/go/arrow/array/concat_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/array/data.go b/go/arrow/array/data.go index 8cce49182b879..49df06fb1174a 100644 --- a/go/arrow/array/data.go +++ b/go/arrow/array/data.go @@ -22,9 +22,9 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" ) // Data represents the memory and metadata of an Arrow array. diff --git a/go/arrow/array/data_test.go b/go/arrow/array/data_test.go index b7b0f396470d7..2773096995fb6 100644 --- a/go/arrow/array/data_test.go +++ b/go/arrow/array/data_test.go @@ -19,8 +19,8 @@ package array import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/decimal128.go b/go/arrow/array/decimal128.go index 16a492db09c67..3317531687ca4 100644 --- a/go/arrow/array/decimal128.go +++ b/go/arrow/array/decimal128.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // A type which represents an immutable sequence of 128-bit decimal values. diff --git a/go/arrow/array/decimal128_test.go b/go/arrow/array/decimal128_test.go index 8c26d00cdc18e..8ef09b68c176a 100644 --- a/go/arrow/array/decimal128_test.go +++ b/go/arrow/array/decimal128_test.go @@ -19,10 +19,10 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/decimal256.go b/go/arrow/array/decimal256.go index 8f72e414d1959..d63544f784a09 100644 --- a/go/arrow/array/decimal256.go +++ b/go/arrow/array/decimal256.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // Decimal256 is a type that represents an immutable sequence of 256-bit decimal values. diff --git a/go/arrow/array/decimal256_test.go b/go/arrow/array/decimal256_test.go index 6085d6b5a6a59..a02098e79103f 100644 --- a/go/arrow/array/decimal256_test.go +++ b/go/arrow/array/decimal256_test.go @@ -19,10 +19,10 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/decimal_test.go b/go/arrow/array/decimal_test.go index 67900447be1cf..2203b2702c09a 100644 --- a/go/arrow/array/decimal_test.go +++ b/go/arrow/array/decimal_test.go @@ -21,12 +21,12 @@ import ( "math/big" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/array/dictionary.go b/go/arrow/array/dictionary.go index 856f91605ff53..d0a1c4dc97e1d 100644 --- a/go/arrow/array/dictionary.go +++ b/go/arrow/array/dictionary.go @@ -25,16 +25,16 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/hashing" + "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v14/internal/utils" ) // Dictionary represents the type for dictionary-encoded data with a data diff --git a/go/arrow/array/dictionary_test.go b/go/arrow/array/dictionary_test.go index 5a3e0e10c23f3..d0878fa3b0329 100644 --- a/go/arrow/array/dictionary_test.go +++ b/go/arrow/array/dictionary_test.go @@ -24,13 +24,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/array/diff.go b/go/arrow/array/diff.go index 6bf6372531fd7..026a27b983f59 100644 --- a/go/arrow/array/diff.go +++ b/go/arrow/array/diff.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) // Edit represents one entry in the edit script to compare two arrays. diff --git a/go/arrow/array/diff_test.go b/go/arrow/array/diff_test.go index 17539c38d282f..0eff8dc4f0577 100644 --- a/go/arrow/array/diff_test.go +++ b/go/arrow/array/diff_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v14/internal/types" ) type diffTestCase struct { diff --git a/go/arrow/array/encoded.go b/go/arrow/array/encoded.go index fa5fa7addf34c..bf4a942cf1c35 100644 --- a/go/arrow/array/encoded.go +++ b/go/arrow/array/encoded.go @@ -23,12 +23,12 @@ import ( "reflect" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/encoded" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/encoded" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v14/internal/utils" ) // RunEndEncoded represents an array containing two children: diff --git a/go/arrow/array/encoded_test.go b/go/arrow/array/encoded_test.go index 5bfac7a1a96e6..57c01cf22d2d6 100644 --- a/go/arrow/array/encoded_test.go +++ b/go/arrow/array/encoded_test.go @@ -20,10 +20,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/arrow/array/extension.go b/go/arrow/array/extension.go index 021b8e7bc451b..03e8c1734b9d6 100644 --- a/go/arrow/array/extension.go +++ b/go/arrow/array/extension.go @@ -20,9 +20,9 @@ import ( "fmt" "reflect" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // ExtensionArray is the interface that needs to be implemented to handle diff --git a/go/arrow/array/extension_test.go b/go/arrow/array/extension_test.go index a8e2b0dfd59bb..5b473fed428ef 100644 --- a/go/arrow/array/extension_test.go +++ b/go/arrow/array/extension_test.go @@ -19,10 +19,10 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/array/fixed_size_list.go b/go/arrow/array/fixed_size_list.go index 5923d68590b15..62c3213861a7d 100644 --- a/go/arrow/array/fixed_size_list.go +++ b/go/arrow/array/fixed_size_list.go @@ -22,11 +22,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // FixedSizeList represents an immutable sequence of N array values. diff --git a/go/arrow/array/fixed_size_list_test.go b/go/arrow/array/fixed_size_list_test.go index 5c01199ddf987..83dff923fe120 100644 --- a/go/arrow/array/fixed_size_list_test.go +++ b/go/arrow/array/fixed_size_list_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/fixedsize_binary.go b/go/arrow/array/fixedsize_binary.go index 6cdaeace939fd..5466156d5a256 100644 --- a/go/arrow/array/fixedsize_binary.go +++ b/go/arrow/array/fixedsize_binary.go @@ -22,8 +22,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/internal/json" ) // A type which represents an immutable sequence of fixed-length binary strings. diff --git a/go/arrow/array/fixedsize_binary_test.go b/go/arrow/array/fixedsize_binary_test.go index 785725537cbdd..b65c7051a9b3e 100644 --- a/go/arrow/array/fixedsize_binary_test.go +++ b/go/arrow/array/fixedsize_binary_test.go @@ -21,9 +21,9 @@ import ( "github.com/stretchr/testify/assert" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestFixedSizeBinary(t *testing.T) { diff --git a/go/arrow/array/fixedsize_binarybuilder.go b/go/arrow/array/fixedsize_binarybuilder.go index 230a65fd2d352..ba4b474a89fd0 100644 --- a/go/arrow/array/fixedsize_binarybuilder.go +++ b/go/arrow/array/fixedsize_binarybuilder.go @@ -23,10 +23,10 @@ import ( "reflect" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // A FixedSizeBinaryBuilder is used to build a FixedSizeBinary array using the Append methods. diff --git a/go/arrow/array/fixedsize_binarybuilder_test.go b/go/arrow/array/fixedsize_binarybuilder_test.go index 8e4a0ac1e46a7..e3962ad3b9372 100644 --- a/go/arrow/array/fixedsize_binarybuilder_test.go +++ b/go/arrow/array/fixedsize_binarybuilder_test.go @@ -19,8 +19,8 @@ package array import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/float16.go b/go/arrow/array/float16.go index 4260f8e3774b4..de499e26706cc 100644 --- a/go/arrow/array/float16.go +++ b/go/arrow/array/float16.go @@ -20,9 +20,9 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/internal/json" ) // A type which represents an immutable sequence of Float16 values. diff --git a/go/arrow/array/float16_builder.go b/go/arrow/array/float16_builder.go index 033b9fa2d8028..f96ab6037e002 100644 --- a/go/arrow/array/float16_builder.go +++ b/go/arrow/array/float16_builder.go @@ -23,12 +23,12 @@ import ( "strconv" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) type Float16Builder struct { diff --git a/go/arrow/array/float16_builder_test.go b/go/arrow/array/float16_builder_test.go index f8c5890179869..6ee028d978fb0 100644 --- a/go/arrow/array/float16_builder_test.go +++ b/go/arrow/array/float16_builder_test.go @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/interval.go b/go/arrow/array/interval.go index 2a5529f1c30f7..ff059c92c8583 100644 --- a/go/arrow/array/interval.go +++ b/go/arrow/array/interval.go @@ -23,11 +23,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) func NewIntervalData(data arrow.ArrayData) arrow.Array { diff --git a/go/arrow/array/interval_test.go b/go/arrow/array/interval_test.go index f83fc52dfa34e..50a96e4779f03 100644 --- a/go/arrow/array/interval_test.go +++ b/go/arrow/array/interval_test.go @@ -20,9 +20,9 @@ import ( "math" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/json_reader.go b/go/arrow/array/json_reader.go index 2f05d4b70dd76..e09717c4199b7 100644 --- a/go/arrow/array/json_reader.go +++ b/go/arrow/array/json_reader.go @@ -22,10 +22,10 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) type Option func(config) diff --git a/go/arrow/array/json_reader_test.go b/go/arrow/array/json_reader_test.go index 7f12bf211dd04..3a095a0d33189 100644 --- a/go/arrow/array/json_reader_test.go +++ b/go/arrow/array/json_reader_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/list.go b/go/arrow/array/list.go index f10e2072c43a2..d8d8b8c76165a 100644 --- a/go/arrow/array/list.go +++ b/go/arrow/array/list.go @@ -23,11 +23,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) type ListLike interface { diff --git a/go/arrow/array/list_test.go b/go/arrow/array/list_test.go index 11404b2d8bb95..bf3555b3f6603 100644 --- a/go/arrow/array/list_test.go +++ b/go/arrow/array/list_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/map.go b/go/arrow/array/map.go index fe07a68785067..9945a90ce495e 100644 --- a/go/arrow/array/map.go +++ b/go/arrow/array/map.go @@ -20,9 +20,9 @@ import ( "bytes" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // Map represents an immutable sequence of Key/Value structs. It is a diff --git a/go/arrow/array/map_test.go b/go/arrow/array/map_test.go index a7ecc032682bc..3fe78549ec803 100644 --- a/go/arrow/array/map_test.go +++ b/go/arrow/array/map_test.go @@ -20,9 +20,9 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/null.go b/go/arrow/array/null.go index 2735a88a92cb3..150a1030eb49d 100644 --- a/go/arrow/array/null.go +++ b/go/arrow/array/null.go @@ -23,10 +23,10 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // Null represents an immutable, degenerate array with no physical storage. diff --git a/go/arrow/array/null_test.go b/go/arrow/array/null_test.go index 5d230ec5cec71..e1bf1e0345b84 100644 --- a/go/arrow/array/null_test.go +++ b/go/arrow/array/null_test.go @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go index 59c9a979768d5..a3e1101516aa6 100644 --- a/go/arrow/array/numeric.gen.go +++ b/go/arrow/array/numeric.gen.go @@ -23,8 +23,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/internal/json" ) // A type which represents an immutable sequence of int64 values. diff --git a/go/arrow/array/numeric.gen.go.tmpl b/go/arrow/array/numeric.gen.go.tmpl index 027456009daad..34d17fbfc8854 100644 --- a/go/arrow/array/numeric.gen.go.tmpl +++ b/go/arrow/array/numeric.gen.go.tmpl @@ -21,8 +21,8 @@ import ( "strings" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/internal/json" ) {{range .In}} diff --git a/go/arrow/array/numeric_test.go b/go/arrow/array/numeric_test.go index 91dd724c8da50..34f59c238cbef 100644 --- a/go/arrow/array/numeric_test.go +++ b/go/arrow/array/numeric_test.go @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go index 52b189d6ed453..7f01180f55957 100644 --- a/go/arrow/array/numericbuilder.gen.go +++ b/go/arrow/array/numericbuilder.gen.go @@ -27,11 +27,11 @@ import ( "sync/atomic" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) type Int64Builder struct { diff --git a/go/arrow/array/numericbuilder.gen.go.tmpl b/go/arrow/array/numericbuilder.gen.go.tmpl index 82ac35465d424..cf663c031f616 100644 --- a/go/arrow/array/numericbuilder.gen.go.tmpl +++ b/go/arrow/array/numericbuilder.gen.go.tmpl @@ -17,11 +17,11 @@ package array import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) {{range .In}} diff --git a/go/arrow/array/numericbuilder.gen_test.go b/go/arrow/array/numericbuilder.gen_test.go index e1f72773403d8..b5986775cd995 100644 --- a/go/arrow/array/numericbuilder.gen_test.go +++ b/go/arrow/array/numericbuilder.gen_test.go @@ -21,9 +21,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numericbuilder.gen_test.go.tmpl b/go/arrow/array/numericbuilder.gen_test.go.tmpl index eddd884e2eddf..bc8c993374671 100644 --- a/go/arrow/array/numericbuilder.gen_test.go.tmpl +++ b/go/arrow/array/numericbuilder.gen_test.go.tmpl @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/record.go b/go/arrow/array/record.go index d080f726e472d..0b0fe4c3847fc 100644 --- a/go/arrow/array/record.go +++ b/go/arrow/array/record.go @@ -22,10 +22,10 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // RecordReader reads a stream of records. diff --git a/go/arrow/array/record_test.go b/go/arrow/array/record_test.go index 7d438d1f1f81e..da50a30b646f1 100644 --- a/go/arrow/array/record_test.go +++ b/go/arrow/array/record_test.go @@ -21,9 +21,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/string.go b/go/arrow/array/string.go index 9ab7c938ef5d8..86e27c970cbe9 100644 --- a/go/arrow/array/string.go +++ b/go/arrow/array/string.go @@ -23,9 +23,9 @@ import ( "strings" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // String represents an immutable sequence of variable-length UTF-8 strings. diff --git a/go/arrow/array/string_test.go b/go/arrow/array/string_test.go index d743a3ec7f37f..fbc106b098332 100644 --- a/go/arrow/array/string_test.go +++ b/go/arrow/array/string_test.go @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go index 94052953852c2..248a25bf6044d 100644 --- a/go/arrow/array/struct.go +++ b/go/arrow/array/struct.go @@ -23,11 +23,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // Struct represents an ordered sequence of relative types. diff --git a/go/arrow/array/struct_test.go b/go/arrow/array/struct_test.go index 1b0dc5a3e4b19..86a5b311fcc21 100644 --- a/go/arrow/array/struct_test.go +++ b/go/arrow/array/struct_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/table.go b/go/arrow/array/table.go index a987dd057f82c..6456992e34973 100644 --- a/go/arrow/array/table.go +++ b/go/arrow/array/table.go @@ -23,8 +23,8 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) // NewColumnSlice returns a new zero-copy slice of the column with the indicated diff --git a/go/arrow/array/table_test.go b/go/arrow/array/table_test.go index 9535ae6b089bb..8e22d53963ba4 100644 --- a/go/arrow/array/table_test.go +++ b/go/arrow/array/table_test.go @@ -22,9 +22,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestChunked(t *testing.T) { diff --git a/go/arrow/array/timestamp.go b/go/arrow/array/timestamp.go index 6ffb43e067af0..2928b1fc77ff6 100644 --- a/go/arrow/array/timestamp.go +++ b/go/arrow/array/timestamp.go @@ -24,11 +24,11 @@ import ( "sync/atomic" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) // Timestamp represents an immutable sequence of arrow.Timestamp values. diff --git a/go/arrow/array/timestamp_test.go b/go/arrow/array/timestamp_test.go index acbad8b586dd4..1e55364e8da0d 100644 --- a/go/arrow/array/timestamp_test.go +++ b/go/arrow/array/timestamp_test.go @@ -20,9 +20,9 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/union.go b/go/arrow/array/union.go index c0a5050560634..869355ac7108a 100644 --- a/go/arrow/array/union.go +++ b/go/arrow/array/union.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/json" ) // Union is a convenience interface to encompass both Sparse and Dense diff --git a/go/arrow/array/union_test.go b/go/arrow/array/union_test.go index e876f5def26d3..2f2bf3b84e9f7 100644 --- a/go/arrow/array/union_test.go +++ b/go/arrow/array/union_test.go @@ -21,9 +21,9 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/array/util.go b/go/arrow/array/util.go index a1b3cc7d4e5f7..54d15a8095c80 100644 --- a/go/arrow/array/util.go +++ b/go/arrow/array/util.go @@ -22,11 +22,11 @@ import ( "io" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/hashing" + "github.com/apache/arrow/go/v14/internal/json" ) func min(a, b int) int { diff --git a/go/arrow/array/util_test.go b/go/arrow/array/util_test.go index 84a6debdf3946..d3d2b37365513 100644 --- a/go/arrow/array/util_test.go +++ b/go/arrow/array/util_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/arrio/arrio.go b/go/arrow/arrio/arrio.go index 51cf6dc46d44a..466a93a68400a 100644 --- a/go/arrow/arrio/arrio.go +++ b/go/arrow/arrio/arrio.go @@ -22,7 +22,7 @@ import ( "errors" "io" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) // Reader is the interface that wraps the Read method. diff --git a/go/arrow/arrio/arrio_test.go b/go/arrow/arrio/arrio_test.go index c80d5d2569d67..7e9aac8fbe520 100644 --- a/go/arrow/arrio/arrio_test.go +++ b/go/arrow/arrio/arrio_test.go @@ -22,11 +22,11 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) type copyKind int diff --git a/go/arrow/bitutil/bitmaps.go b/go/arrow/bitutil/bitmaps.go index d7516771def7f..2e9c0601c3c45 100644 --- a/go/arrow/bitutil/bitmaps.go +++ b/go/arrow/bitutil/bitmaps.go @@ -22,9 +22,9 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" ) // BitmapReader is a simple bitmap reader for a byte slice. diff --git a/go/arrow/bitutil/bitmaps_test.go b/go/arrow/bitutil/bitmaps_test.go index c926bff39e09a..60022a179d1da 100644 --- a/go/arrow/bitutil/bitmaps_test.go +++ b/go/arrow/bitutil/bitmaps_test.go @@ -22,8 +22,8 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/bitutil/bitutil.go b/go/arrow/bitutil/bitutil.go index dc510a8b374c4..a4a1519b8f940 100644 --- a/go/arrow/bitutil/bitutil.go +++ b/go/arrow/bitutil/bitutil.go @@ -22,7 +22,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/memory" ) var ( diff --git a/go/arrow/bitutil/bitutil_test.go b/go/arrow/bitutil/bitutil_test.go index 189c8541f4925..9e29e9d04d1db 100644 --- a/go/arrow/bitutil/bitutil_test.go +++ b/go/arrow/bitutil/bitutil_test.go @@ -21,8 +21,8 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/cdata/cdata.go b/go/arrow/cdata/cdata.go index f9693851d7483..1d454c57f722e 100644 --- a/go/arrow/cdata/cdata.go +++ b/go/arrow/cdata/cdata.go @@ -46,10 +46,10 @@ import ( "syscall" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/cdata/cdata_exports.go b/go/arrow/cdata/cdata_exports.go index 91f1b352e0327..187c2deb9755f 100644 --- a/go/arrow/cdata/cdata_exports.go +++ b/go/arrow/cdata/cdata_exports.go @@ -45,11 +45,11 @@ import ( "strings" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/internal" + "github.com/apache/arrow/go/v14/arrow/ipc" ) func encodeCMetadata(keys, values []string) []byte { diff --git a/go/arrow/cdata/cdata_test.go b/go/arrow/cdata/cdata_test.go index 607cfe53217a6..f09fa3ff2f6fa 100644 --- a/go/arrow/cdata/cdata_test.go +++ b/go/arrow/cdata/cdata_test.go @@ -35,12 +35,12 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/memory/mallocator" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/cdata/cdata_test_framework.go b/go/arrow/cdata/cdata_test_framework.go index 1251b20201e41..2df52dcb3cd40 100644 --- a/go/arrow/cdata/cdata_test_framework.go +++ b/go/arrow/cdata/cdata_test_framework.go @@ -69,10 +69,10 @@ import ( "runtime/cgo" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/internal" + "github.com/apache/arrow/go/v14/arrow/memory/mallocator" ) const ( diff --git a/go/arrow/cdata/exports.go b/go/arrow/cdata/exports.go index 7353df62d113a..9d2576818e31c 100644 --- a/go/arrow/cdata/exports.go +++ b/go/arrow/cdata/exports.go @@ -20,8 +20,8 @@ import ( "runtime/cgo" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" ) // #include diff --git a/go/arrow/cdata/import_allocator.go b/go/arrow/cdata/import_allocator.go index cf1c6a961ff37..eff8c7517caef 100644 --- a/go/arrow/cdata/import_allocator.go +++ b/go/arrow/cdata/import_allocator.go @@ -20,7 +20,7 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) // #include "arrow/c/helpers.h" diff --git a/go/arrow/cdata/interface.go b/go/arrow/cdata/interface.go index 8ce06280a0bf5..bf5c5270bae2f 100644 --- a/go/arrow/cdata/interface.go +++ b/go/arrow/cdata/interface.go @@ -22,10 +22,10 @@ package cdata import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/cdata/test/test_cimport.go b/go/arrow/cdata/test/test_cimport.go index 147c3691f0c71..a7c5dc034496a 100644 --- a/go/arrow/cdata/test/test_cimport.go +++ b/go/arrow/cdata/test/test_cimport.go @@ -23,10 +23,10 @@ import ( "fmt" "runtime" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/cdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/cdata" + "github.com/apache/arrow/go/v14/arrow/memory" ) // #include diff --git a/go/arrow/compute/arithmetic.go b/go/arrow/compute/arithmetic.go index 1ee1959b2ddc8..2fb95f06ff84a 100644 --- a/go/arrow/compute/arithmetic.go +++ b/go/arrow/compute/arithmetic.go @@ -22,12 +22,12 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/scalar" ) type ( diff --git a/go/arrow/compute/arithmetic_test.go b/go/arrow/compute/arithmetic_test.go index 821ffd9e068d4..a99b2074eb69f 100644 --- a/go/arrow/compute/arithmetic_test.go +++ b/go/arrow/compute/arithmetic_test.go @@ -26,16 +26,16 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/klauspost/cpuid/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/arrow/compute/cast.go b/go/arrow/compute/cast.go index 133a983038ce1..8b720a2bad5d8 100644 --- a/go/arrow/compute/cast.go +++ b/go/arrow/compute/cast.go @@ -23,11 +23,11 @@ import ( "fmt" "sync" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/cast_test.go b/go/arrow/compute/cast_test.go index 10957a45167aa..cc301cc35e6e5 100644 --- a/go/arrow/compute/cast_test.go +++ b/go/arrow/compute/cast_test.go @@ -26,16 +26,16 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v14/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/compute/datum.go b/go/arrow/compute/datum.go index 98bd1f1a0a326..1d3c1b4dab700 100644 --- a/go/arrow/compute/datum.go +++ b/go/arrow/compute/datum.go @@ -21,9 +21,9 @@ package compute import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/scalar" ) //go:generate go run golang.org/x/tools/cmd/stringer -type=DatumKind -linecomment diff --git a/go/arrow/compute/example_test.go b/go/arrow/compute/example_test.go index e2b07b7e191ae..f0db9f377868e 100644 --- a/go/arrow/compute/example_test.go +++ b/go/arrow/compute/example_test.go @@ -23,11 +23,11 @@ import ( "fmt" "log" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/memory" ) // This example demonstrates how to register a custom scalar function. diff --git a/go/arrow/compute/exec.go b/go/arrow/compute/exec.go index eba47e64bd509..84e3310cc2df8 100644 --- a/go/arrow/compute/exec.go +++ b/go/arrow/compute/exec.go @@ -22,9 +22,9 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) func haveChunkedArray(values []Datum) bool { diff --git a/go/arrow/compute/exec/kernel.go b/go/arrow/compute/exec/kernel.go index 766857f63e565..327426da68eb9 100644 --- a/go/arrow/compute/exec/kernel.go +++ b/go/arrow/compute/exec/kernel.go @@ -24,10 +24,10 @@ import ( "hash/maphash" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/exec/kernel_test.go b/go/arrow/compute/exec/kernel_test.go index 4df6b42ff9408..16a29ea2deda6 100644 --- a/go/arrow/compute/exec/kernel_test.go +++ b/go/arrow/compute/exec/kernel_test.go @@ -22,12 +22,12 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exec/span.go b/go/arrow/compute/exec/span.go index 0b5f6208227f5..b6d240fa4a0be 100644 --- a/go/arrow/compute/exec/span.go +++ b/go/arrow/compute/exec/span.go @@ -23,11 +23,11 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" ) // BufferSpan is a lightweight Buffer holder for ArraySpans that does not diff --git a/go/arrow/compute/exec/span_test.go b/go/arrow/compute/exec/span_test.go index 474c005b44642..c642be1c30c23 100644 --- a/go/arrow/compute/exec/span_test.go +++ b/go/arrow/compute/exec/span_test.go @@ -24,14 +24,14 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v14/internal/types" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exec/utils.go b/go/arrow/compute/exec/utils.go index 17bc30ef2adb0..6d83b75da1313 100644 --- a/go/arrow/compute/exec/utils.go +++ b/go/arrow/compute/exec/utils.go @@ -25,13 +25,13 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/memory" "golang.org/x/exp/constraints" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/exec/utils_test.go b/go/arrow/compute/exec/utils_test.go index b26e4ff41e79f..e9bb07177f2ac 100644 --- a/go/arrow/compute/exec/utils_test.go +++ b/go/arrow/compute/exec/utils_test.go @@ -21,10 +21,10 @@ package exec_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exec_internals_test.go b/go/arrow/compute/exec_internals_test.go index bae32268862ff..eb9273d23389f 100644 --- a/go/arrow/compute/exec_internals_test.go +++ b/go/arrow/compute/exec_internals_test.go @@ -24,13 +24,13 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/compute/exec_test.go b/go/arrow/compute/exec_test.go index a37f67c03e8ce..30bfbc0246a1c 100644 --- a/go/arrow/compute/exec_test.go +++ b/go/arrow/compute/exec_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/compute/executor.go b/go/arrow/compute/executor.go index 1cba0b1e19f69..6da7ed1293065 100644 --- a/go/arrow/compute/executor.go +++ b/go/arrow/compute/executor.go @@ -25,14 +25,14 @@ import ( "runtime" "sync" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" ) // ExecCtx holds simple contextual information for execution diff --git a/go/arrow/compute/expression.go b/go/arrow/compute/expression.go index fbb6c502d98ab..9f20c9704dca7 100644 --- a/go/arrow/compute/expression.go +++ b/go/arrow/compute/expression.go @@ -28,14 +28,14 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" ) var hashSeed = maphash.MakeSeed() diff --git a/go/arrow/compute/expression_test.go b/go/arrow/compute/expression_test.go index b3b44ae1faa61..71b1b64c4b0c1 100644 --- a/go/arrow/compute/expression_test.go +++ b/go/arrow/compute/expression_test.go @@ -22,11 +22,11 @@ package compute_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exprs/builders.go b/go/arrow/compute/exprs/builders.go index 0b694525d1b47..39606c27e714a 100644 --- a/go/arrow/compute/exprs/builders.go +++ b/go/arrow/compute/exprs/builders.go @@ -25,8 +25,8 @@ import ( "strings" "unicode" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute" "github.com/substrait-io/substrait-go/expr" "github.com/substrait-io/substrait-go/extensions" "github.com/substrait-io/substrait-go/types" diff --git a/go/arrow/compute/exprs/builders_test.go b/go/arrow/compute/exprs/builders_test.go index 69501622359d5..e42d7569a8f03 100644 --- a/go/arrow/compute/exprs/builders_test.go +++ b/go/arrow/compute/exprs/builders_test.go @@ -21,8 +21,8 @@ package exprs_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exprs" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exprs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/substrait-io/substrait-go/expr" diff --git a/go/arrow/compute/exprs/exec.go b/go/arrow/compute/exprs/exec.go index 800ffe62f2559..d63752696082b 100644 --- a/go/arrow/compute/exprs/exec.go +++ b/go/arrow/compute/exprs/exec.go @@ -23,15 +23,15 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/substrait-io/substrait-go/expr" "github.com/substrait-io/substrait-go/extensions" "github.com/substrait-io/substrait-go/types" diff --git a/go/arrow/compute/exprs/exec_internal_test.go b/go/arrow/compute/exprs/exec_internal_test.go index 680bf36f11958..9475a9a8cbca9 100644 --- a/go/arrow/compute/exprs/exec_internal_test.go +++ b/go/arrow/compute/exprs/exec_internal_test.go @@ -23,10 +23,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/compute/exprs/exec_test.go b/go/arrow/compute/exprs/exec_test.go index 65cafc8e62ddb..e30d2afdb73f8 100644 --- a/go/arrow/compute/exprs/exec_test.go +++ b/go/arrow/compute/exprs/exec_test.go @@ -23,12 +23,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exprs" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exprs" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/substrait-io/substrait-go/expr" diff --git a/go/arrow/compute/exprs/extension_types.go b/go/arrow/compute/exprs/extension_types.go index db7992b8f089f..ce375492d67f1 100644 --- a/go/arrow/compute/exprs/extension_types.go +++ b/go/arrow/compute/exprs/extension_types.go @@ -24,8 +24,8 @@ import ( "reflect" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" ) type simpleExtensionTypeFactory[P comparable] struct { diff --git a/go/arrow/compute/exprs/field_refs.go b/go/arrow/compute/exprs/field_refs.go index e95e3c8c9abb9..fd4313818195e 100644 --- a/go/arrow/compute/exprs/field_refs.go +++ b/go/arrow/compute/exprs/field_refs.go @@ -21,11 +21,11 @@ package exprs import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/substrait-io/substrait-go/expr" ) diff --git a/go/arrow/compute/exprs/types.go b/go/arrow/compute/exprs/types.go index 6a5b81d11b3eb..9f807e1e1ecbc 100644 --- a/go/arrow/compute/exprs/types.go +++ b/go/arrow/compute/exprs/types.go @@ -24,8 +24,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute" "github.com/substrait-io/substrait-go/expr" "github.com/substrait-io/substrait-go/extensions" "github.com/substrait-io/substrait-go/types" diff --git a/go/arrow/compute/fieldref.go b/go/arrow/compute/fieldref.go index 565ae3bfadbd0..ee6f39948ebf1 100644 --- a/go/arrow/compute/fieldref.go +++ b/go/arrow/compute/fieldref.go @@ -27,8 +27,8 @@ import ( "unicode" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" ) var ( diff --git a/go/arrow/compute/fieldref_test.go b/go/arrow/compute/fieldref_test.go index c4fa72182835f..c38ad8ac1fe30 100644 --- a/go/arrow/compute/fieldref_test.go +++ b/go/arrow/compute/fieldref_test.go @@ -19,10 +19,10 @@ package compute_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/functions.go b/go/arrow/compute/functions.go index b85062721d9b8..a1905f91850f0 100644 --- a/go/arrow/compute/functions.go +++ b/go/arrow/compute/functions.go @@ -23,8 +23,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exec" ) type Function interface { diff --git a/go/arrow/compute/functions_test.go b/go/arrow/compute/functions_test.go index 1e6bbd598e0c9..b37da285badd0 100644 --- a/go/arrow/compute/functions_test.go +++ b/go/arrow/compute/functions_test.go @@ -21,8 +21,8 @@ package compute_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/compute/internal/kernels/base_arithmetic.go b/go/arrow/compute/internal/kernels/base_arithmetic.go index c7950877264df..67e80af74f511 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic.go @@ -24,11 +24,11 @@ import ( "math/bits" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/internal/debug" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go b/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go index 942b8e4ff5600..0e78e6c9183ff 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go @@ -21,8 +21,8 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/debug" "golang.org/x/exp/constraints" "golang.org/x/sys/cpu" ) diff --git a/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go b/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go index 6814e834b4bf3..29cce7830e0f4 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go b/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go index 633ec5f4f18e5..e9b03551f81b1 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go b/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go index 8e46ca030c8b7..4f160a1411091 100644 --- a/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go +++ b/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go @@ -19,7 +19,7 @@ package kernels import ( - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/exec" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/boolean_cast.go b/go/arrow/compute/internal/kernels/boolean_cast.go index 923c5b3f54512..18d04c845c41d 100644 --- a/go/arrow/compute/internal/kernels/boolean_cast.go +++ b/go/arrow/compute/internal/kernels/boolean_cast.go @@ -22,9 +22,9 @@ import ( "strconv" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" ) func isNonZero[T exec.FixedWidthTypes](ctx *exec.KernelCtx, in []T, out []byte) error { diff --git a/go/arrow/compute/internal/kernels/cast.go b/go/arrow/compute/internal/kernels/cast.go index bc4ee3abd128c..5a71206b2476c 100644 --- a/go/arrow/compute/internal/kernels/cast.go +++ b/go/arrow/compute/internal/kernels/cast.go @@ -19,9 +19,9 @@ package kernels import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute/exec" ) type CastOptions struct { diff --git a/go/arrow/compute/internal/kernels/cast_numeric.go b/go/arrow/compute/internal/kernels/cast_numeric.go index 2e893c7205f6a..4e5c5c1d9ea77 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric.go +++ b/go/arrow/compute/internal/kernels/cast_numeric.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) var castNumericUnsafe func(itype, otype arrow.Type, in, out []byte, len int) = castNumericGo diff --git a/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go b/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go index eafa4b41001ae..6b28441ec083e 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go +++ b/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go b/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go index 75ad79fd0e7f3..d53a4486f6983 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go +++ b/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" "golang.org/x/sys/cpu" ) diff --git a/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go b/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go index d04f393c4c099..1cbea033dec27 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go +++ b/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/cast_temporal.go b/go/arrow/compute/internal/kernels/cast_temporal.go index 542a8a4590b28..82fce1e33c959 100644 --- a/go/arrow/compute/internal/kernels/cast_temporal.go +++ b/go/arrow/compute/internal/kernels/cast_temporal.go @@ -24,10 +24,10 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) const millisecondsInDay = 86400000 diff --git a/go/arrow/compute/internal/kernels/helpers.go b/go/arrow/compute/internal/kernels/helpers.go index 686c4b3e0c29a..ed25071c9d87f 100644 --- a/go/arrow/compute/internal/kernels/helpers.go +++ b/go/arrow/compute/internal/kernels/helpers.go @@ -22,13 +22,13 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v14/internal/bitutils" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/numeric_cast.go b/go/arrow/compute/internal/kernels/numeric_cast.go index c055552bf7ff5..8e5350754723d 100644 --- a/go/arrow/compute/internal/kernels/numeric_cast.go +++ b/go/arrow/compute/internal/kernels/numeric_cast.go @@ -23,13 +23,13 @@ import ( "strconv" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/internal/bitutils" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/rounding.go b/go/arrow/compute/internal/kernels/rounding.go index 8a1bec1180ac8..2f58a9faea8a2 100644 --- a/go/arrow/compute/internal/kernels/rounding.go +++ b/go/arrow/compute/internal/kernels/rounding.go @@ -22,11 +22,11 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/scalar" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/scalar_arithmetic.go b/go/arrow/compute/internal/kernels/scalar_arithmetic.go index cf17e9fd9548b..9cb32ae64a4a9 100644 --- a/go/arrow/compute/internal/kernels/scalar_arithmetic.go +++ b/go/arrow/compute/internal/kernels/scalar_arithmetic.go @@ -22,13 +22,13 @@ import ( "fmt" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/scalar" ) // scalar kernel that ignores (assumed all-null inputs) and returns null diff --git a/go/arrow/compute/internal/kernels/scalar_boolean.go b/go/arrow/compute/internal/kernels/scalar_boolean.go index 0707c92e6a198..812f4ad1bd363 100644 --- a/go/arrow/compute/internal/kernels/scalar_boolean.go +++ b/go/arrow/compute/internal/kernels/scalar_boolean.go @@ -19,9 +19,9 @@ package kernels import ( - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/scalar" ) type computeWordFN func(leftTrue, leftFalse, rightTrue, rightFalse uint64) (outValid, outData uint64) diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go b/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go index 8e5ce1ab7c1ad..585d1bff01ab6 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go @@ -21,8 +21,8 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exec" "golang.org/x/sys/cpu" ) diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go b/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go index cf9fc1eeedbb8..868179057ccf4 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go b/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go index c0aef5a04e9b8..56abad42b1ffc 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go @@ -18,7 +18,7 @@ package kernels -import "github.com/apache/arrow/go/v15/arrow/compute/exec" +import "github.com/apache/arrow/go/v14/arrow/compute/exec" func genCompareKernel[T exec.NumericTypes](op CompareOperator) *CompareData { return genGoCompareKernel(getCmpOp[T](op)) diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go b/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go index f8b36a1e4be76..7cf96a4171d6c 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/scalar_comparisons.go b/go/arrow/compute/internal/kernels/scalar_comparisons.go index 9a7640a8d8a39..8a957eafd85da 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparisons.go +++ b/go/arrow/compute/internal/kernels/scalar_comparisons.go @@ -23,14 +23,14 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v14/internal/bitutils" ) type binaryKernel func(left, right, out []byte, offset int) diff --git a/go/arrow/compute/internal/kernels/string_casts.go b/go/arrow/compute/internal/kernels/string_casts.go index 76da901e33f8d..3070514665d7d 100644 --- a/go/arrow/compute/internal/kernels/string_casts.go +++ b/go/arrow/compute/internal/kernels/string_casts.go @@ -23,12 +23,12 @@ import ( "strconv" "unicode/utf8" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/internal/bitutils" ) func validateUtf8Fsb(input *exec.ArraySpan) error { diff --git a/go/arrow/compute/internal/kernels/types.go b/go/arrow/compute/internal/kernels/types.go index 481eab36059f8..2788fb7054c53 100644 --- a/go/arrow/compute/internal/kernels/types.go +++ b/go/arrow/compute/internal/kernels/types.go @@ -21,10 +21,10 @@ package kernels import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/scalar" ) var ( diff --git a/go/arrow/compute/internal/kernels/vector_hash.go b/go/arrow/compute/internal/kernels/vector_hash.go index 9401e31cc5b09..e0ede826e876e 100644 --- a/go/arrow/compute/internal/kernels/vector_hash.go +++ b/go/arrow/compute/internal/kernels/vector_hash.go @@ -21,13 +21,13 @@ package kernels import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/hashing" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/hashing" ) type HashState interface { diff --git a/go/arrow/compute/internal/kernels/vector_run_end_encode.go b/go/arrow/compute/internal/kernels/vector_run_end_encode.go index a147bf7d50170..e073ff1ff5308 100644 --- a/go/arrow/compute/internal/kernels/vector_run_end_encode.go +++ b/go/arrow/compute/internal/kernels/vector_run_end_encode.go @@ -24,14 +24,14 @@ import ( "sort" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" ) type RunEndEncodeState struct { diff --git a/go/arrow/compute/internal/kernels/vector_selection.go b/go/arrow/compute/internal/kernels/vector_selection.go index 714e452325bfd..c7a902bd008d0 100644 --- a/go/arrow/compute/internal/kernels/vector_selection.go +++ b/go/arrow/compute/internal/kernels/vector_selection.go @@ -22,13 +22,13 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" ) type NullSelectionBehavior int8 diff --git a/go/arrow/compute/registry.go b/go/arrow/compute/registry.go index 379e0ccbe86a9..4f1c435fb680a 100644 --- a/go/arrow/compute/registry.go +++ b/go/arrow/compute/registry.go @@ -21,7 +21,7 @@ package compute import ( "sync" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/debug" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/registry_test.go b/go/arrow/compute/registry_test.go index 4e4f44f1d39b6..428f7cf1d8fda 100644 --- a/go/arrow/compute/registry_test.go +++ b/go/arrow/compute/registry_test.go @@ -23,9 +23,9 @@ import ( "errors" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exec" "github.com/stretchr/testify/assert" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/scalar_bool.go b/go/arrow/compute/scalar_bool.go index 1f28a6e2bfcb2..49c745688284e 100644 --- a/go/arrow/compute/scalar_bool.go +++ b/go/arrow/compute/scalar_bool.go @@ -21,9 +21,9 @@ package compute import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/scalar_bool_test.go b/go/arrow/compute/scalar_bool_test.go index bd4f3c5c0df2b..6eadc973d6965 100644 --- a/go/arrow/compute/scalar_bool_test.go +++ b/go/arrow/compute/scalar_bool_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/compute/scalar_compare.go b/go/arrow/compute/scalar_compare.go index 24a4191a10999..476f37711e289 100644 --- a/go/arrow/compute/scalar_compare.go +++ b/go/arrow/compute/scalar_compare.go @@ -21,9 +21,9 @@ package compute import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" ) type compareFunction struct { diff --git a/go/arrow/compute/scalar_compare_test.go b/go/arrow/compute/scalar_compare_test.go index d209f72c800b0..13fdd24fdd6b3 100644 --- a/go/arrow/compute/scalar_compare_test.go +++ b/go/arrow/compute/scalar_compare_test.go @@ -24,15 +24,15 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/compute/selection.go b/go/arrow/compute/selection.go index 90bc5280ef2a7..ed6d80418d3f6 100644 --- a/go/arrow/compute/selection.go +++ b/go/arrow/compute/selection.go @@ -22,10 +22,10 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" "golang.org/x/sync/errgroup" ) diff --git a/go/arrow/compute/utils.go b/go/arrow/compute/utils.go index 9de6523fd0b5f..cc4d6edc48ec8 100644 --- a/go/arrow/compute/utils.go +++ b/go/arrow/compute/utils.go @@ -24,12 +24,12 @@ import ( "math" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/compute/vector_hash.go b/go/arrow/compute/vector_hash.go index 144c123380fe2..5f9aec55c55bc 100644 --- a/go/arrow/compute/vector_hash.go +++ b/go/arrow/compute/vector_hash.go @@ -21,8 +21,8 @@ package compute import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/vector_hash_test.go b/go/arrow/compute/vector_hash_test.go index 9410720de7941..50c96ddc04f6c 100644 --- a/go/arrow/compute/vector_hash_test.go +++ b/go/arrow/compute/vector_hash_test.go @@ -23,13 +23,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/compute/vector_run_end_test.go b/go/arrow/compute/vector_run_end_test.go index 51c0f834ceb6a..8d894d96b8f0a 100644 --- a/go/arrow/compute/vector_run_end_test.go +++ b/go/arrow/compute/vector_run_end_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/compute/vector_run_ends.go b/go/arrow/compute/vector_run_ends.go index 3e47c67de3c35..48f3dcba8c481 100644 --- a/go/arrow/compute/vector_run_ends.go +++ b/go/arrow/compute/vector_run_ends.go @@ -21,8 +21,8 @@ package compute import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/vector_selection_test.go b/go/arrow/compute/vector_selection_test.go index f44840ba72034..10788d60e10d2 100644 --- a/go/arrow/compute/vector_selection_test.go +++ b/go/arrow/compute/vector_selection_test.go @@ -24,15 +24,15 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v14/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/csv/common.go b/go/arrow/csv/common.go index 2a1f7300a986f..91f42897a24a6 100644 --- a/go/arrow/csv/common.go +++ b/go/arrow/csv/common.go @@ -22,8 +22,8 @@ import ( "errors" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" ) var ( diff --git a/go/arrow/csv/reader.go b/go/arrow/csv/reader.go index e58b426d837fb..80af520a12b7f 100644 --- a/go/arrow/csv/reader.go +++ b/go/arrow/csv/reader.go @@ -29,13 +29,13 @@ import ( "time" "unicode/utf8" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" ) // Reader wraps encoding/csv.Reader and creates array.Records from a schema. diff --git a/go/arrow/csv/reader_test.go b/go/arrow/csv/reader_test.go index dfcb6625bd7cc..f481cd2e961d3 100644 --- a/go/arrow/csv/reader_test.go +++ b/go/arrow/csv/reader_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/csv" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/csv" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/csv/transformer.go b/go/arrow/csv/transformer.go index 0f0181520b847..7c8eafcee1a59 100644 --- a/go/arrow/csv/transformer.go +++ b/go/arrow/csv/transformer.go @@ -25,8 +25,8 @@ import ( "math/big" "strconv" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" ) func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array) []string { diff --git a/go/arrow/csv/writer.go b/go/arrow/csv/writer.go index a672008b58a07..e558364ecf16d 100644 --- a/go/arrow/csv/writer.go +++ b/go/arrow/csv/writer.go @@ -22,7 +22,7 @@ import ( "strconv" "sync" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) // Writer wraps encoding/csv.Writer and writes arrow.Record based on a schema. diff --git a/go/arrow/csv/writer_test.go b/go/arrow/csv/writer_test.go index 1918e2e492dff..f945023790426 100644 --- a/go/arrow/csv/writer_test.go +++ b/go/arrow/csv/writer_test.go @@ -25,14 +25,14 @@ import ( "log" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/csv" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/csv" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" "github.com/google/uuid" ) diff --git a/go/arrow/datatype.go b/go/arrow/datatype.go index 24113b55899dc..f0fb24ec873c5 100644 --- a/go/arrow/datatype.go +++ b/go/arrow/datatype.go @@ -21,7 +21,7 @@ import ( "hash/maphash" "strings" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) // Type is a logical type. They can be expressed as diff --git a/go/arrow/datatype_binary_test.go b/go/arrow/datatype_binary_test.go index 25ba6e8db4ba4..c47df3da1db9c 100644 --- a/go/arrow/datatype_binary_test.go +++ b/go/arrow/datatype_binary_test.go @@ -19,7 +19,7 @@ package arrow_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) func TestBinaryType(t *testing.T) { diff --git a/go/arrow/datatype_extension_test.go b/go/arrow/datatype_extension_test.go index 9811800400506..c542b22b769e6 100644 --- a/go/arrow/datatype_extension_test.go +++ b/go/arrow/datatype_extension_test.go @@ -20,8 +20,8 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go index bcbc8ef6aec87..fc0b3aea56e70 100644 --- a/go/arrow/datatype_fixedwidth.go +++ b/go/arrow/datatype_fixedwidth.go @@ -21,7 +21,7 @@ import ( "strconv" "time" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/internal/json" "golang.org/x/xerrors" ) diff --git a/go/arrow/datatype_fixedwidth_test.go b/go/arrow/datatype_fixedwidth_test.go index 918572d40b8f4..1d3a07de09c1d 100644 --- a/go/arrow/datatype_fixedwidth_test.go +++ b/go/arrow/datatype_fixedwidth_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/datatype_nested.go b/go/arrow/datatype_nested.go index e381cd7047e45..4ae4880334620 100644 --- a/go/arrow/datatype_nested.go +++ b/go/arrow/datatype_nested.go @@ -22,7 +22,7 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) type ( diff --git a/go/arrow/datatype_null_test.go b/go/arrow/datatype_null_test.go index 57cddfadb8ef8..286a72fb7fea3 100644 --- a/go/arrow/datatype_null_test.go +++ b/go/arrow/datatype_null_test.go @@ -19,7 +19,7 @@ package arrow_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) func TestNullType(t *testing.T) { diff --git a/go/arrow/decimal128/decimal128.go b/go/arrow/decimal128/decimal128.go index 7bde39d327417..ff9c088b9d638 100644 --- a/go/arrow/decimal128/decimal128.go +++ b/go/arrow/decimal128/decimal128.go @@ -23,7 +23,7 @@ import ( "math/big" "math/bits" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) const ( diff --git a/go/arrow/decimal128/decimal128_test.go b/go/arrow/decimal128/decimal128_test.go index 879f2849028f8..102eede6d3b28 100644 --- a/go/arrow/decimal128/decimal128_test.go +++ b/go/arrow/decimal128/decimal128_test.go @@ -22,7 +22,7 @@ import ( "math/big" "testing" - "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal128" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/decimal256/decimal256.go b/go/arrow/decimal256/decimal256.go index 5f2ad5f32165c..cf2157fdd48ed 100644 --- a/go/arrow/decimal256/decimal256.go +++ b/go/arrow/decimal256/decimal256.go @@ -23,8 +23,8 @@ import ( "math/big" "math/bits" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) const ( diff --git a/go/arrow/decimal256/decimal256_test.go b/go/arrow/decimal256/decimal256_test.go index 3be6a1944406f..9c734e23aa0d6 100644 --- a/go/arrow/decimal256/decimal256_test.go +++ b/go/arrow/decimal256/decimal256_test.go @@ -23,7 +23,7 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/decimal256" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/doc.go b/go/arrow/doc.go index c8da330e4f8a1..87aea1d41035c 100644 --- a/go/arrow/doc.go +++ b/go/arrow/doc.go @@ -36,7 +36,7 @@ To build with tinygo include the noasm build tag. */ package arrow -const PkgVersion = "15.0.0-SNAPSHOT" +const PkgVersion = "14.0.1-SNAPSHOT" //go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl //go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl diff --git a/go/arrow/encoded/ree_utils.go b/go/arrow/encoded/ree_utils.go index 2ff4e7cadd38b..1f71e7b5210cf 100644 --- a/go/arrow/encoded/ree_utils.go +++ b/go/arrow/encoded/ree_utils.go @@ -20,7 +20,7 @@ import ( "math" "sort" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) // FindPhysicalIndex performs a binary search on the run-ends to return diff --git a/go/arrow/encoded/ree_utils_test.go b/go/arrow/encoded/ree_utils_test.go index 9470331002b80..785a3b105a47d 100644 --- a/go/arrow/encoded/ree_utils_test.go +++ b/go/arrow/encoded/ree_utils_test.go @@ -21,10 +21,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/encoded" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/encoded" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/endian/endian.go b/go/arrow/endian/endian.go index d4865025b1fb0..3ecda7b3600b7 100644 --- a/go/arrow/endian/endian.go +++ b/go/arrow/endian/endian.go @@ -17,8 +17,8 @@ package endian import ( - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" ) type Endianness flatbuf.Endianness diff --git a/go/arrow/example_test.go b/go/arrow/example_test.go index b28cc9093739f..10b3633f65536 100644 --- a/go/arrow/example_test.go +++ b/go/arrow/example_test.go @@ -20,10 +20,10 @@ import ( "fmt" "log" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/tensor" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/tensor" ) // This example demonstrates how to build an array of int64 values using a builder and Append. diff --git a/go/arrow/flight/basic_auth_flight_test.go b/go/arrow/flight/basic_auth_flight_test.go index 4097bf02edeff..0f63a400353d2 100755 --- a/go/arrow/flight/basic_auth_flight_test.go +++ b/go/arrow/flight/basic_auth_flight_test.go @@ -22,7 +22,7 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/go/arrow/flight/client.go b/go/arrow/flight/client.go index 8c400eb66b64d..c9a1417accab4 100644 --- a/go/arrow/flight/client.go +++ b/go/arrow/flight/client.go @@ -26,7 +26,7 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" diff --git a/go/arrow/flight/cookie_middleware_test.go b/go/arrow/flight/cookie_middleware_test.go index bdfe4fb32c1a5..2bde37fc0bb85 100644 --- a/go/arrow/flight/cookie_middleware_test.go +++ b/go/arrow/flight/cookie_middleware_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/go/arrow/flight/example_flight_server_test.go b/go/arrow/flight/example_flight_server_test.go index d3a71b3b67ade..09f01361d13a0 100755 --- a/go/arrow/flight/example_flight_server_test.go +++ b/go/arrow/flight/example_flight_server_test.go @@ -23,7 +23,7 @@ import ( "io" "log" - "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/go/arrow/flight/flight_middleware_test.go b/go/arrow/flight/flight_middleware_test.go index e6faa1b7df1d2..f2b240d1f2e45 100755 --- a/go/arrow/flight/flight_middleware_test.go +++ b/go/arrow/flight/flight_middleware_test.go @@ -23,8 +23,8 @@ import ( sync "sync" "testing" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/go/arrow/flight/flight_test.go b/go/arrow/flight/flight_test.go index cd37658603d5b..7cd724b76b8ce 100755 --- a/go/arrow/flight/flight_test.go +++ b/go/arrow/flight/flight_test.go @@ -23,11 +23,11 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/go/arrow/flight/flightsql/client.go b/go/arrow/flight/flightsql/client.go index c0c7e2cf20a28..13d046661e96d 100644 --- a/go/arrow/flight/flightsql/client.go +++ b/go/arrow/flight/flightsql/client.go @@ -22,12 +22,12 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" diff --git a/go/arrow/flight/flightsql/client_test.go b/go/arrow/flight/flightsql/client_test.go index 3efe4ba4049b8..bb4ae29477997 100644 --- a/go/arrow/flight/flightsql/client_test.go +++ b/go/arrow/flight/flightsql/client_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "google.golang.org/grpc" diff --git a/go/arrow/flight/flightsql/column_metadata.go b/go/arrow/flight/flightsql/column_metadata.go index 8e0a69b99d055..d9785892888b2 100644 --- a/go/arrow/flight/flightsql/column_metadata.go +++ b/go/arrow/flight/flightsql/column_metadata.go @@ -19,7 +19,7 @@ package flightsql import ( "strconv" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) const ( diff --git a/go/arrow/flight/flightsql/driver/README.md b/go/arrow/flight/flightsql/driver/README.md index f1447a7d24256..b8850527c19c1 100644 --- a/go/arrow/flight/flightsql/driver/README.md +++ b/go/arrow/flight/flightsql/driver/README.md @@ -36,7 +36,7 @@ connection pooling, transactions combined with ease of use (see (#usage)). ## Prerequisites * Go 1.17+ -* Installation via `go get -u github.com/apache/arrow/go/v15/arrow/flight/flightsql` +* Installation via `go get -u github.com/apache/arrow/go/v14/arrow/flight/flightsql` * Backend speaking FlightSQL --------------------------------------- @@ -55,7 +55,7 @@ import ( "database/sql" "time" - _ "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + _ "github.com/apache/arrow/go/v14/arrow/flight/flightsql" ) // Open the connection to an SQLite backend @@ -141,7 +141,7 @@ import ( "log" "time" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" ) func main() { diff --git a/go/arrow/flight/flightsql/driver/config_test.go b/go/arrow/flight/flightsql/driver/config_test.go index 6b24b535585d9..abcececeaf214 100644 --- a/go/arrow/flight/flightsql/driver/config_test.go +++ b/go/arrow/flight/flightsql/driver/config_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/driver" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/driver" ) func TestConfigTLSRegistry(t *testing.T) { diff --git a/go/arrow/flight/flightsql/driver/driver.go b/go/arrow/flight/flightsql/driver/driver.go index e325489236c6d..b4cb0d2dbda2e 100644 --- a/go/arrow/flight/flightsql/driver/driver.go +++ b/go/arrow/flight/flightsql/driver/driver.go @@ -25,11 +25,11 @@ import ( "sort" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow/memory" "google.golang.org/grpc" "google.golang.org/grpc/credentials" diff --git a/go/arrow/flight/flightsql/driver/driver_test.go b/go/arrow/flight/flightsql/driver/driver_test.go index a388bf155ec99..4449b390974a7 100644 --- a/go/arrow/flight/flightsql/driver/driver_test.go +++ b/go/arrow/flight/flightsql/driver/driver_test.go @@ -33,13 +33,13 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/driver" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/driver" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v14/arrow/memory" ) const defaultTableName = "drivertest" diff --git a/go/arrow/flight/flightsql/driver/utils.go b/go/arrow/flight/flightsql/driver/utils.go index 022e9da4925f9..99352e7251c53 100644 --- a/go/arrow/flight/flightsql/driver/utils.go +++ b/go/arrow/flight/flightsql/driver/utils.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" ) // *** GRPC helpers *** diff --git a/go/arrow/flight/flightsql/driver/utils_test.go b/go/arrow/flight/flightsql/driver/utils_test.go index 8f5ea031adf28..355c888f56cc4 100644 --- a/go/arrow/flight/flightsql/driver/utils_test.go +++ b/go/arrow/flight/flightsql/driver/utils_test.go @@ -22,12 +22,12 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go b/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go index 1a50422f826f3..6508833876fa5 100644 --- a/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go +++ b/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go @@ -27,9 +27,9 @@ import ( "os" "strconv" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/example" ) func main() { diff --git a/go/arrow/flight/flightsql/example/sql_batch_reader.go b/go/arrow/flight/flightsql/example/sql_batch_reader.go index 36a0d7b424544..7101b8f830a6f 100644 --- a/go/arrow/flight/flightsql/example/sql_batch_reader.go +++ b/go/arrow/flight/flightsql/example/sql_batch_reader.go @@ -26,11 +26,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/wrapperspb" diff --git a/go/arrow/flight/flightsql/example/sqlite_info.go b/go/arrow/flight/flightsql/example/sqlite_info.go index c993af97f7449..bf29265853c65 100644 --- a/go/arrow/flight/flightsql/example/sqlite_info.go +++ b/go/arrow/flight/flightsql/example/sqlite_info.go @@ -20,8 +20,8 @@ package example import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" ) func SqlInfoResultMap() flightsql.SqlInfoResultMap { diff --git a/go/arrow/flight/flightsql/example/sqlite_server.go b/go/arrow/flight/flightsql/example/sqlite_server.go index f06dd0210655f..63b41ee5e0a4a 100644 --- a/go/arrow/flight/flightsql/example/sqlite_server.go +++ b/go/arrow/flight/flightsql/example/sqlite_server.go @@ -45,13 +45,13 @@ import ( "strings" "sync" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" diff --git a/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go b/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go index e07a98fffdbf7..507abf622d890 100644 --- a/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go +++ b/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" sqlite3 "modernc.org/sqlite/lib" ) diff --git a/go/arrow/flight/flightsql/example/type_info.go b/go/arrow/flight/flightsql/example/type_info.go index 0180fc1f23af5..6d6ceb339e4b0 100644 --- a/go/arrow/flight/flightsql/example/type_info.go +++ b/go/arrow/flight/flightsql/example/type_info.go @@ -22,10 +22,10 @@ package example import ( "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v14/arrow/memory" ) func GetTypeInfoResult(mem memory.Allocator) arrow.Record { diff --git a/go/arrow/flight/flightsql/schema_ref/reference_schemas.go b/go/arrow/flight/flightsql/schema_ref/reference_schemas.go index b65688306473b..98d61f8110d45 100644 --- a/go/arrow/flight/flightsql/schema_ref/reference_schemas.go +++ b/go/arrow/flight/flightsql/schema_ref/reference_schemas.go @@ -18,7 +18,7 @@ // by FlightSQL servers and clients. package schema_ref -import "github.com/apache/arrow/go/v15/arrow" +import "github.com/apache/arrow/go/v14/arrow" var ( Catalogs = arrow.NewSchema( diff --git a/go/arrow/flight/flightsql/server.go b/go/arrow/flight/flightsql/server.go index a086610433eae..e9cbc177da2c4 100644 --- a/go/arrow/flight/flightsql/server.go +++ b/go/arrow/flight/flightsql/server.go @@ -20,14 +20,14 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" + pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" diff --git a/go/arrow/flight/flightsql/server_test.go b/go/arrow/flight/flightsql/server_test.go index e444da4aaf4a2..33e9b8236a261 100644 --- a/go/arrow/flight/flightsql/server_test.go +++ b/go/arrow/flight/flightsql/server_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/go/arrow/flight/flightsql/sql_info.go b/go/arrow/flight/flightsql/sql_info.go index 5f78e1facd581..a12c4fbdaeda9 100644 --- a/go/arrow/flight/flightsql/sql_info.go +++ b/go/arrow/flight/flightsql/sql_info.go @@ -17,8 +17,8 @@ package flightsql import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" ) const ( diff --git a/go/arrow/flight/flightsql/sqlite_server_test.go b/go/arrow/flight/flightsql/sqlite_server_test.go index e6fa798c55cf1..03ae3d7a8313a 100644 --- a/go/arrow/flight/flightsql/sqlite_server_test.go +++ b/go/arrow/flight/flightsql/sqlite_server_test.go @@ -26,14 +26,14 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "google.golang.org/grpc/codes" diff --git a/go/arrow/flight/flightsql/types.go b/go/arrow/flight/flightsql/types.go index 2b7419482e3e3..98086a61fe067 100644 --- a/go/arrow/flight/flightsql/types.go +++ b/go/arrow/flight/flightsql/types.go @@ -17,7 +17,7 @@ package flightsql import ( - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) diff --git a/go/arrow/flight/record_batch_reader.go b/go/arrow/flight/record_batch_reader.go index f137c75172368..b726180627e6f 100644 --- a/go/arrow/flight/record_batch_reader.go +++ b/go/arrow/flight/record_batch_reader.go @@ -21,12 +21,12 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) // DataStreamReader is an interface for receiving flight data messages on a stream diff --git a/go/arrow/flight/record_batch_writer.go b/go/arrow/flight/record_batch_writer.go index 426dca6c3e907..3ae17bcf48042 100644 --- a/go/arrow/flight/record_batch_writer.go +++ b/go/arrow/flight/record_batch_writer.go @@ -19,9 +19,9 @@ package flight import ( "bytes" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) // DataStreamWriter is an interface that represents an Arrow Flight stream diff --git a/go/arrow/flight/server.go b/go/arrow/flight/server.go index 8676b15644e47..cf471f9b3a2b6 100644 --- a/go/arrow/flight/server.go +++ b/go/arrow/flight/server.go @@ -22,7 +22,7 @@ import ( "os" "os/signal" - "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" "google.golang.org/grpc" ) diff --git a/go/arrow/flight/server_example_test.go b/go/arrow/flight/server_example_test.go index 4e2e4f4c6b587..b82e2cb7f7164 100644 --- a/go/arrow/flight/server_example_test.go +++ b/go/arrow/flight/server_example_test.go @@ -21,7 +21,7 @@ import ( "fmt" "net" - "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/health" diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go index 6631e4245c19d..0851bff0fe0da 100644 --- a/go/arrow/internal/arrdata/arrdata.go +++ b/go/arrow/internal/arrdata/arrdata.go @@ -21,14 +21,14 @@ import ( "fmt" "sort" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" ) var ( diff --git a/go/arrow/internal/arrdata/ioutil.go b/go/arrow/internal/arrdata/ioutil.go index a6becc2151ef3..1a2ce0e5b5eca 100644 --- a/go/arrow/internal/arrdata/ioutil.go +++ b/go/arrow/internal/arrdata/ioutil.go @@ -23,11 +23,11 @@ import ( "sync" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) // CheckArrowFile checks whether a given ARROW file contains the expected list of records. diff --git a/go/arrow/internal/arrjson/arrjson.go b/go/arrow/internal/arrjson/arrjson.go index 87bdc1f44d875..ad87b73fc4ddb 100644 --- a/go/arrow/internal/arrjson/arrjson.go +++ b/go/arrow/internal/arrjson/arrjson.go @@ -26,16 +26,16 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" ) type Schema struct { diff --git a/go/arrow/internal/arrjson/arrjson_test.go b/go/arrow/internal/arrjson/arrjson_test.go index 7beadee370edb..ee85d431805ab 100644 --- a/go/arrow/internal/arrjson/arrjson_test.go +++ b/go/arrow/internal/arrjson/arrjson_test.go @@ -22,9 +22,9 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/internal/arrjson/option.go b/go/arrow/internal/arrjson/option.go index 47d40f857c54e..7ae2058d471a7 100644 --- a/go/arrow/internal/arrjson/option.go +++ b/go/arrow/internal/arrjson/option.go @@ -17,8 +17,8 @@ package arrjson import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" ) type config struct { diff --git a/go/arrow/internal/arrjson/reader.go b/go/arrow/internal/arrjson/reader.go index 226fa1b1919f9..c8056ef1dc744 100644 --- a/go/arrow/internal/arrjson/reader.go +++ b/go/arrow/internal/arrjson/reader.go @@ -20,11 +20,11 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/internal/json" ) type Reader struct { diff --git a/go/arrow/internal/arrjson/writer.go b/go/arrow/internal/arrjson/writer.go index af7032d581f4d..ddac32609b604 100644 --- a/go/arrow/internal/arrjson/writer.go +++ b/go/arrow/internal/arrjson/writer.go @@ -20,11 +20,11 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/internal/json" ) const ( diff --git a/go/arrow/internal/cdata_integration/entrypoints.go b/go/arrow/internal/cdata_integration/entrypoints.go index a40db8316f848..629b8a762a689 100644 --- a/go/arrow/internal/cdata_integration/entrypoints.go +++ b/go/arrow/internal/cdata_integration/entrypoints.go @@ -25,10 +25,10 @@ import ( "runtime" "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/cdata" - "github.com/apache/arrow/go/v15/arrow/internal/arrjson" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/cdata" + "github.com/apache/arrow/go/v14/arrow/internal/arrjson" + "github.com/apache/arrow/go/v14/arrow/memory" ) // #include diff --git a/go/arrow/internal/dictutils/dict.go b/go/arrow/internal/dictutils/dict.go index d31369f7d25b3..e09a2f4a0ecd1 100644 --- a/go/arrow/internal/dictutils/dict.go +++ b/go/arrow/internal/dictutils/dict.go @@ -21,9 +21,9 @@ import ( "fmt" "hash/maphash" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" ) type Kind int8 diff --git a/go/arrow/internal/dictutils/dict_test.go b/go/arrow/internal/dictutils/dict_test.go index 9d2f7ae4782c7..6cafacbd1df00 100644 --- a/go/arrow/internal/dictutils/dict_test.go +++ b/go/arrow/internal/dictutils/dict_test.go @@ -20,10 +20,10 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestDictMemo(t *testing.T) { diff --git a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go index 1afb53c5294f4..8da39858fabab 100755 --- a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go +++ b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - "github.com/apache/arrow/go/v15/arrow/internal/flight_integration" + "github.com/apache/arrow/go/v14/arrow/internal/flight_integration" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) diff --git a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go index d382ff9e88670..9cebb2c695ece 100644 --- a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go +++ b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go @@ -23,7 +23,7 @@ import ( "os" "syscall" - "github.com/apache/arrow/go/v15/arrow/internal/flight_integration" + "github.com/apache/arrow/go/v14/arrow/internal/flight_integration" ) var ( diff --git a/go/arrow/internal/flight_integration/scenario.go b/go/arrow/internal/flight_integration/scenario.go index 3ec905e2d659c..4f47d7fd506ee 100644 --- a/go/arrow/internal/flight_integration/scenario.go +++ b/go/arrow/internal/flight_integration/scenario.go @@ -31,15 +31,15 @@ import ( "strings" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v15/arrow/internal/arrjson" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v14/arrow/internal/arrjson" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" "golang.org/x/xerrors" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/go/arrow/internal/testing/gen/random_array_gen.go b/go/arrow/internal/testing/gen/random_array_gen.go index b42273ff93fac..41f2578209a7f 100644 --- a/go/arrow/internal/testing/gen/random_array_gen.go +++ b/go/arrow/internal/testing/gen/random_array_gen.go @@ -19,11 +19,11 @@ package gen import ( "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" "golang.org/x/exp/rand" "gonum.org/v1/gonum/stat/distuv" ) diff --git a/go/arrow/internal/testing/tools/bits_test.go b/go/arrow/internal/testing/tools/bits_test.go index 6ad8ac5b12b68..7d0fccb5f999e 100644 --- a/go/arrow/internal/testing/tools/bits_test.go +++ b/go/arrow/internal/testing/tools/bits_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/internal/testing/tools/data_types.go b/go/arrow/internal/testing/tools/data_types.go index 161cfa431491d..654780bc3e853 100644 --- a/go/arrow/internal/testing/tools/data_types.go +++ b/go/arrow/internal/testing/tools/data_types.go @@ -21,8 +21,8 @@ package tools import ( "reflect" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/float16" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/internal/utils.go b/go/arrow/internal/utils.go index 619eebd97dc78..265f030dfd19a 100644 --- a/go/arrow/internal/utils.go +++ b/go/arrow/internal/utils.go @@ -17,8 +17,8 @@ package internal import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" ) const CurMetadataVersion = flatbuf.MetadataVersionV5 diff --git a/go/arrow/ipc/cmd/arrow-cat/main.go b/go/arrow/ipc/cmd/arrow-cat/main.go index 0251b08c09bff..af53c9b4e5f44 100644 --- a/go/arrow/ipc/cmd/arrow-cat/main.go +++ b/go/arrow/ipc/cmd/arrow-cat/main.go @@ -63,8 +63,8 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-cat/main_test.go b/go/arrow/ipc/cmd/arrow-cat/main_test.go index b6528b85b416d..22e4f6ffe0778 100644 --- a/go/arrow/ipc/cmd/arrow-cat/main_test.go +++ b/go/arrow/ipc/cmd/arrow-cat/main_test.go @@ -23,10 +23,10 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestCatStream(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-file-to-stream/main.go b/go/arrow/ipc/cmd/arrow-file-to-stream/main.go index 6187b94465211..7e86263d6d5b8 100644 --- a/go/arrow/ipc/cmd/arrow-file-to-stream/main.go +++ b/go/arrow/ipc/cmd/arrow-file-to-stream/main.go @@ -24,9 +24,9 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go b/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go index 2f66a8b48c59d..30db24715386f 100644 --- a/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go +++ b/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go @@ -21,8 +21,8 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestFileToStream(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-json-integration-test/main.go b/go/arrow/ipc/cmd/arrow-json-integration-test/main.go index 7db2fa1d1c5db..91ba6a4ad9b61 100644 --- a/go/arrow/ipc/cmd/arrow-json-integration-test/main.go +++ b/go/arrow/ipc/cmd/arrow-json-integration-test/main.go @@ -22,12 +22,12 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/arrjson" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/internal/arrjson" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/internal/types" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go b/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go index eb702f0aa2bc5..140f9f62478c0 100644 --- a/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go +++ b/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go @@ -20,8 +20,8 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestIntegration(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-ls/main.go b/go/arrow/ipc/cmd/arrow-ls/main.go index 4230ae2449985..f739c9ae7da8e 100644 --- a/go/arrow/ipc/cmd/arrow-ls/main.go +++ b/go/arrow/ipc/cmd/arrow-ls/main.go @@ -61,8 +61,8 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-ls/main_test.go b/go/arrow/ipc/cmd/arrow-ls/main_test.go index bda1c2dc4d3dc..2c038dae97b52 100644 --- a/go/arrow/ipc/cmd/arrow-ls/main_test.go +++ b/go/arrow/ipc/cmd/arrow-ls/main_test.go @@ -23,10 +23,10 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestLsStream(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-stream-to-file/main.go b/go/arrow/ipc/cmd/arrow-stream-to-file/main.go index 7ed3f6a281d9f..758231ea21cf5 100644 --- a/go/arrow/ipc/cmd/arrow-stream-to-file/main.go +++ b/go/arrow/ipc/cmd/arrow-stream-to-file/main.go @@ -24,9 +24,9 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go b/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go index 73196551054ac..4354b46e8964b 100644 --- a/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go +++ b/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go @@ -21,8 +21,8 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestStreamToFile(t *testing.T) { diff --git a/go/arrow/ipc/compression.go b/go/arrow/ipc/compression.go index 8856b732f9c5d..73fb916500804 100644 --- a/go/arrow/ipc/compression.go +++ b/go/arrow/ipc/compression.go @@ -19,9 +19,9 @@ package ipc import ( "io" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/klauspost/compress/zstd" "github.com/pierrec/lz4/v4" ) diff --git a/go/arrow/ipc/endian_swap.go b/go/arrow/ipc/endian_swap.go index d2e0948434abc..d98fec1089f04 100644 --- a/go/arrow/ipc/endian_swap.go +++ b/go/arrow/ipc/endian_swap.go @@ -20,9 +20,9 @@ import ( "errors" "math/bits" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" ) // swap the endianness of the array's buffers as needed in-place to save diff --git a/go/arrow/ipc/endian_swap_test.go b/go/arrow/ipc/endian_swap_test.go index 1c724103f4441..4f02bb53d61a4 100644 --- a/go/arrow/ipc/endian_swap_test.go +++ b/go/arrow/ipc/endian_swap_test.go @@ -20,11 +20,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go index 330355d3a60c3..10cb2cae764e6 100644 --- a/go/arrow/ipc/file_reader.go +++ b/go/arrow/ipc/file_reader.go @@ -23,14 +23,14 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/internal" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" ) // FileReader is an Arrow file reader. diff --git a/go/arrow/ipc/file_test.go b/go/arrow/ipc/file_test.go index 5f4dac1f899bb..1b22719dc8097 100644 --- a/go/arrow/ipc/file_test.go +++ b/go/arrow/ipc/file_test.go @@ -21,9 +21,9 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestFile(t *testing.T) { diff --git a/go/arrow/ipc/file_writer.go b/go/arrow/ipc/file_writer.go index 55e4d7c2dc22c..12384225b70a1 100644 --- a/go/arrow/ipc/file_writer.go +++ b/go/arrow/ipc/file_writer.go @@ -21,11 +21,11 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" ) // PayloadWriter is an interface for injecting a different payloadwriter diff --git a/go/arrow/ipc/ipc.go b/go/arrow/ipc/ipc.go index 8cf56e8331fcb..6c04b6f5ad2d6 100644 --- a/go/arrow/ipc/ipc.go +++ b/go/arrow/ipc/ipc.go @@ -19,10 +19,10 @@ package ipc import ( "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" ) const ( diff --git a/go/arrow/ipc/ipc_test.go b/go/arrow/ipc/ipc_test.go index d02d8734b56d4..551d325dc1212 100644 --- a/go/arrow/ipc/ipc_test.go +++ b/go/arrow/ipc/ipc_test.go @@ -29,10 +29,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestArrow12072(t *testing.T) { diff --git a/go/arrow/ipc/message.go b/go/arrow/ipc/message.go index 709aa5aa2dba4..c5d0ec68d19c4 100644 --- a/go/arrow/ipc/message.go +++ b/go/arrow/ipc/message.go @@ -22,9 +22,9 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" ) // MetadataVersion represents the Arrow metadata version. diff --git a/go/arrow/ipc/message_test.go b/go/arrow/ipc/message_test.go index 912d112229f0c..cc41fdd3fe13d 100644 --- a/go/arrow/ipc/message_test.go +++ b/go/arrow/ipc/message_test.go @@ -22,9 +22,9 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestMessageReaderBodyInAllocator(t *testing.T) { diff --git a/go/arrow/ipc/metadata.go b/go/arrow/ipc/metadata.go index bd437834c3d06..9bab47d6fa0cd 100644 --- a/go/arrow/ipc/metadata.go +++ b/go/arrow/ipc/metadata.go @@ -23,11 +23,11 @@ import ( "io" "sort" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" flatbuffers "github.com/google/flatbuffers/go" ) diff --git a/go/arrow/ipc/metadata_test.go b/go/arrow/ipc/metadata_test.go index a35068813ba56..3038abdf5f791 100644 --- a/go/arrow/ipc/metadata_test.go +++ b/go/arrow/ipc/metadata_test.go @@ -21,12 +21,12 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" flatbuffers "github.com/google/flatbuffers/go" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/ipc/reader.go b/go/arrow/ipc/reader.go index 826062fafdb46..1f684c1f6c034 100644 --- a/go/arrow/ipc/reader.go +++ b/go/arrow/ipc/reader.go @@ -23,13 +23,13 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" ) // Reader reads records from an io.Reader. diff --git a/go/arrow/ipc/reader_test.go b/go/arrow/ipc/reader_test.go index f00f3bb3da476..229374e94ec74 100644 --- a/go/arrow/ipc/reader_test.go +++ b/go/arrow/ipc/reader_test.go @@ -20,9 +20,9 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/ipc/stream_test.go b/go/arrow/ipc/stream_test.go index 201bfb82a5677..d8bbd8f04b14d 100644 --- a/go/arrow/ipc/stream_test.go +++ b/go/arrow/ipc/stream_test.go @@ -22,9 +22,9 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" ) func TestStream(t *testing.T) { diff --git a/go/arrow/ipc/writer.go b/go/arrow/ipc/writer.go index 58c56d2d16ccf..a97f47ef4aa43 100644 --- a/go/arrow/ipc/writer.go +++ b/go/arrow/ipc/writer.go @@ -26,14 +26,14 @@ import ( "sync" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/dictutils" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" ) type swriter struct { diff --git a/go/arrow/ipc/writer_test.go b/go/arrow/ipc/writer_test.go index ea7592554c888..da461c3d52272 100644 --- a/go/arrow/ipc/writer_test.go +++ b/go/arrow/ipc/writer_test.go @@ -24,11 +24,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/math/float64.go b/go/arrow/math/float64.go index 8d72ecf1b2668..81de594df6ce6 100644 --- a/go/arrow/math/float64.go +++ b/go/arrow/math/float64.go @@ -19,7 +19,7 @@ package math import ( - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) type Float64Funcs struct { diff --git a/go/arrow/math/float64_avx2_amd64.go b/go/arrow/math/float64_avx2_amd64.go index e78d4affe9f6f..8a131a2bc8fc5 100644 --- a/go/arrow/math/float64_avx2_amd64.go +++ b/go/arrow/math/float64_avx2_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) //go:noescape diff --git a/go/arrow/math/float64_neon_arm64.go b/go/arrow/math/float64_neon_arm64.go index a5013960d9ec7..ad9b6435b26fc 100755 --- a/go/arrow/math/float64_neon_arm64.go +++ b/go/arrow/math/float64_neon_arm64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) //go:noescape diff --git a/go/arrow/math/float64_sse4_amd64.go b/go/arrow/math/float64_sse4_amd64.go index cdd88f91c2c2b..a2be5de43f5bb 100644 --- a/go/arrow/math/float64_sse4_amd64.go +++ b/go/arrow/math/float64_sse4_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) //go:noescape diff --git a/go/arrow/math/float64_test.go b/go/arrow/math/float64_test.go index 3a7c247c5df8c..b10214f6d892f 100644 --- a/go/arrow/math/float64_test.go +++ b/go/arrow/math/float64_test.go @@ -21,9 +21,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/math" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/math/int64.go b/go/arrow/math/int64.go index b8236dfbb6552..2adff970c2233 100644 --- a/go/arrow/math/int64.go +++ b/go/arrow/math/int64.go @@ -19,7 +19,7 @@ package math import ( - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) type Int64Funcs struct { diff --git a/go/arrow/math/int64_avx2_amd64.go b/go/arrow/math/int64_avx2_amd64.go index 5c16e790c1e55..ac0b2e1ff662a 100644 --- a/go/arrow/math/int64_avx2_amd64.go +++ b/go/arrow/math/int64_avx2_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) //go:noescape diff --git a/go/arrow/math/int64_neon_arm64.go b/go/arrow/math/int64_neon_arm64.go index 00ec48b6e4735..4ab5fb771a526 100755 --- a/go/arrow/math/int64_neon_arm64.go +++ b/go/arrow/math/int64_neon_arm64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) //go:noescape diff --git a/go/arrow/math/int64_sse4_amd64.go b/go/arrow/math/int64_sse4_amd64.go index a894bad2ef6a3..85304210eb448 100644 --- a/go/arrow/math/int64_sse4_amd64.go +++ b/go/arrow/math/int64_sse4_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) //go:noescape diff --git a/go/arrow/math/int64_test.go b/go/arrow/math/int64_test.go index 3781fd3570785..06b42ffe9351b 100644 --- a/go/arrow/math/int64_test.go +++ b/go/arrow/math/int64_test.go @@ -21,9 +21,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/math" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/math/type.go.tmpl b/go/arrow/math/type.go.tmpl index f1a4dfa1362f8..7ae142c46f12b 100644 --- a/go/arrow/math/type.go.tmpl +++ b/go/arrow/math/type.go.tmpl @@ -17,7 +17,7 @@ package math import ( - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) {{$def := .D}} diff --git a/go/arrow/math/type_simd_amd64.go.tmpl b/go/arrow/math/type_simd_amd64.go.tmpl index 77dee758b05cd..72d40ea386a8f 100644 --- a/go/arrow/math/type_simd_amd64.go.tmpl +++ b/go/arrow/math/type_simd_amd64.go.tmpl @@ -21,7 +21,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) {{$name := printf "%s_%s" .In.Type .D.arch}} diff --git a/go/arrow/math/type_simd_arm64.go.tmpl b/go/arrow/math/type_simd_arm64.go.tmpl index 77dee758b05cd..72d40ea386a8f 100755 --- a/go/arrow/math/type_simd_arm64.go.tmpl +++ b/go/arrow/math/type_simd_arm64.go.tmpl @@ -21,7 +21,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) {{$name := printf "%s_%s" .In.Type .D.arch}} diff --git a/go/arrow/math/type_test.go.tmpl b/go/arrow/math/type_test.go.tmpl index 969377e454e52..160fc34e4d4d4 100644 --- a/go/arrow/math/type_test.go.tmpl +++ b/go/arrow/math/type_test.go.tmpl @@ -19,9 +19,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/math" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/math/uint64.go b/go/arrow/math/uint64.go index b9a70360224ea..64ac63a3d1853 100644 --- a/go/arrow/math/uint64.go +++ b/go/arrow/math/uint64.go @@ -19,7 +19,7 @@ package math import ( - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) type Uint64Funcs struct { diff --git a/go/arrow/math/uint64_avx2_amd64.go b/go/arrow/math/uint64_avx2_amd64.go index c1f48efefd8eb..2a35bebc980b3 100644 --- a/go/arrow/math/uint64_avx2_amd64.go +++ b/go/arrow/math/uint64_avx2_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) //go:noescape diff --git a/go/arrow/math/uint64_neon_arm64.go b/go/arrow/math/uint64_neon_arm64.go index 37ca4ecad04e7..f6588694bb1fd 100755 --- a/go/arrow/math/uint64_neon_arm64.go +++ b/go/arrow/math/uint64_neon_arm64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) //go:noescape diff --git a/go/arrow/math/uint64_sse4_amd64.go b/go/arrow/math/uint64_sse4_amd64.go index 7163c1de171de..8c6621464f118 100644 --- a/go/arrow/math/uint64_sse4_amd64.go +++ b/go/arrow/math/uint64_sse4_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow/array" ) //go:noescape diff --git a/go/arrow/math/uint64_test.go b/go/arrow/math/uint64_test.go index e8ba42b59aa10..aac768e599cf8 100644 --- a/go/arrow/math/uint64_test.go +++ b/go/arrow/math/uint64_test.go @@ -21,9 +21,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/math" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/buffer.go b/go/arrow/memory/buffer.go index ffdb41e3dbe9e..5a2b4297031f0 100644 --- a/go/arrow/memory/buffer.go +++ b/go/arrow/memory/buffer.go @@ -19,7 +19,7 @@ package memory import ( "sync/atomic" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) // Buffer is a wrapper type for a buffer of bytes. diff --git a/go/arrow/memory/buffer_test.go b/go/arrow/memory/buffer_test.go index 92bb071d0abde..25a0b39848519 100644 --- a/go/arrow/memory/buffer_test.go +++ b/go/arrow/memory/buffer_test.go @@ -19,7 +19,7 @@ package memory_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/cgo_allocator.go b/go/arrow/memory/cgo_allocator.go index af25d1899a6fe..85ee445216f6a 100644 --- a/go/arrow/memory/cgo_allocator.go +++ b/go/arrow/memory/cgo_allocator.go @@ -22,7 +22,7 @@ package memory import ( "runtime" - cga "github.com/apache/arrow/go/v15/arrow/memory/internal/cgoalloc" + cga "github.com/apache/arrow/go/v14/arrow/memory/internal/cgoalloc" ) // CgoArrowAllocator is an allocator which exposes the C++ memory pool class diff --git a/go/arrow/memory/default_mallocator.go b/go/arrow/memory/default_mallocator.go index 4a9ef942fd08d..12ad0846677b1 100644 --- a/go/arrow/memory/default_mallocator.go +++ b/go/arrow/memory/default_mallocator.go @@ -19,7 +19,7 @@ package memory import ( - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v14/arrow/memory/mallocator" ) // DefaultAllocator is a default implementation of Allocator and can be used anywhere diff --git a/go/arrow/memory/default_mallocator_test.go b/go/arrow/memory/default_mallocator_test.go index 5a38e8b4e843c..b209ced68a067 100644 --- a/go/arrow/memory/default_mallocator_test.go +++ b/go/arrow/memory/default_mallocator_test.go @@ -21,8 +21,8 @@ package memory_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/memory/mallocator" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/mallocator/mallocator_test.go b/go/arrow/memory/mallocator/mallocator_test.go index 91b74383ed494..5b09355033f70 100644 --- a/go/arrow/memory/mallocator/mallocator_test.go +++ b/go/arrow/memory/mallocator/mallocator_test.go @@ -23,7 +23,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v14/arrow/memory/mallocator" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/memory_test.go b/go/arrow/memory/memory_test.go index adaa8359369be..dff2e0bfd0712 100644 --- a/go/arrow/memory/memory_test.go +++ b/go/arrow/memory/memory_test.go @@ -19,7 +19,7 @@ package memory_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/record.go b/go/arrow/record.go index 885af38034938..d98c7732ef798 100644 --- a/go/arrow/record.go +++ b/go/arrow/record.go @@ -16,7 +16,7 @@ package arrow -import "github.com/apache/arrow/go/v15/internal/json" +import "github.com/apache/arrow/go/v14/internal/json" // Record is a collection of equal-length arrays matching a particular Schema. // Also known as a RecordBatch in the spec and in some implementations. diff --git a/go/arrow/scalar/append.go b/go/arrow/scalar/append.go index fe8dd32e9eae5..9bcfe3e22dc63 100644 --- a/go/arrow/scalar/append.go +++ b/go/arrow/scalar/append.go @@ -21,11 +21,11 @@ package scalar import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/scalar/append_test.go b/go/arrow/scalar/append_test.go index e509bca1fb7b0..497a6a2d02933 100644 --- a/go/arrow/scalar/append_test.go +++ b/go/arrow/scalar/append_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/scalar/binary.go b/go/arrow/scalar/binary.go index b6abe9cba7b1c..3c041210fde9c 100644 --- a/go/arrow/scalar/binary.go +++ b/go/arrow/scalar/binary.go @@ -21,8 +21,8 @@ import ( "fmt" "unicode/utf8" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" ) type BinaryScalar interface { diff --git a/go/arrow/scalar/compare.go b/go/arrow/scalar/compare.go index b4a3fe08d26fe..be7fa4d01cc19 100644 --- a/go/arrow/scalar/compare.go +++ b/go/arrow/scalar/compare.go @@ -16,7 +16,7 @@ package scalar -import "github.com/apache/arrow/go/v15/arrow" +import "github.com/apache/arrow/go/v14/arrow" // Equals returns true if two scalars are equal, which means they have the same // datatype, validity and value. diff --git a/go/arrow/scalar/nested.go b/go/arrow/scalar/nested.go index 8250beb5ed90d..87e84210f58a1 100644 --- a/go/arrow/scalar/nested.go +++ b/go/arrow/scalar/nested.go @@ -21,10 +21,10 @@ import ( "errors" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/scalar/numeric.gen.go b/go/arrow/scalar/numeric.gen.go index da088162f4017..90ae2a7004ce3 100644 --- a/go/arrow/scalar/numeric.gen.go +++ b/go/arrow/scalar/numeric.gen.go @@ -24,9 +24,9 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" ) type Int8 struct { diff --git a/go/arrow/scalar/numeric.gen_test.go b/go/arrow/scalar/numeric.gen_test.go index 807857c1c98c2..987cd66c75f3c 100644 --- a/go/arrow/scalar/numeric.gen_test.go +++ b/go/arrow/scalar/numeric.gen_test.go @@ -21,8 +21,8 @@ package scalar_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/scalar/numeric.gen_test.go.tmpl b/go/arrow/scalar/numeric.gen_test.go.tmpl index 7f45898a20d29..c975cc9db6e1c 100644 --- a/go/arrow/scalar/numeric.gen_test.go.tmpl +++ b/go/arrow/scalar/numeric.gen_test.go.tmpl @@ -19,8 +19,8 @@ package scalar_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/scalar/parse.go b/go/arrow/scalar/parse.go index 5002f98a65c42..fcffe1ea6adf8 100644 --- a/go/arrow/scalar/parse.go +++ b/go/arrow/scalar/parse.go @@ -25,12 +25,12 @@ import ( "strings" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/memory" ) type TypeToScalar interface { diff --git a/go/arrow/scalar/scalar.go b/go/arrow/scalar/scalar.go index 8b0d3ace2ad78..395771fa89c94 100644 --- a/go/arrow/scalar/scalar.go +++ b/go/arrow/scalar/scalar.go @@ -26,16 +26,16 @@ import ( "strconv" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/encoded" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/encoded" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/scalar/scalar_test.go b/go/arrow/scalar/scalar_test.go index ce8170301b0a2..405571a14880d 100644 --- a/go/arrow/scalar/scalar_test.go +++ b/go/arrow/scalar/scalar_test.go @@ -25,12 +25,12 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/scalar/temporal.go b/go/arrow/scalar/temporal.go index ee13c84429e8e..880416f7dd5a3 100644 --- a/go/arrow/scalar/temporal.go +++ b/go/arrow/scalar/temporal.go @@ -22,7 +22,7 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) func temporalToString(s TemporalScalar) string { diff --git a/go/arrow/schema.go b/go/arrow/schema.go index e84f350a53637..a7fa434131925 100644 --- a/go/arrow/schema.go +++ b/go/arrow/schema.go @@ -21,7 +21,7 @@ import ( "sort" "strings" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/endian" ) type Metadata struct { diff --git a/go/arrow/schema_test.go b/go/arrow/schema_test.go index fddf1d7f131ec..5e79723f90352 100644 --- a/go/arrow/schema_test.go +++ b/go/arrow/schema_test.go @@ -21,7 +21,7 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/endian" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/table.go b/go/arrow/table.go index 82dc283706b65..5a68085f8df93 100644 --- a/go/arrow/table.go +++ b/go/arrow/table.go @@ -20,7 +20,7 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) // Table represents a logical sequence of chunked arrays of equal length. It is diff --git a/go/arrow/tensor/numeric.gen.go b/go/arrow/tensor/numeric.gen.go index d207f0bfa2c9c..5b7bbde150fe2 100644 --- a/go/arrow/tensor/numeric.gen.go +++ b/go/arrow/tensor/numeric.gen.go @@ -19,7 +19,7 @@ package tensor import ( - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) // Int8 is an n-dim array of int8s. diff --git a/go/arrow/tensor/numeric.gen.go.tmpl b/go/arrow/tensor/numeric.gen.go.tmpl index e03f986da3f8f..c693082192247 100644 --- a/go/arrow/tensor/numeric.gen.go.tmpl +++ b/go/arrow/tensor/numeric.gen.go.tmpl @@ -17,8 +17,8 @@ package tensor import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" ) {{range .In}} diff --git a/go/arrow/tensor/numeric.gen_test.go b/go/arrow/tensor/numeric.gen_test.go index 3a7c3570c216b..00922a84b6b03 100644 --- a/go/arrow/tensor/numeric.gen_test.go +++ b/go/arrow/tensor/numeric.gen_test.go @@ -23,10 +23,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/tensor" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/tensor" ) func TestTensorInt8(t *testing.T) { diff --git a/go/arrow/tensor/numeric.gen_test.go.tmpl b/go/arrow/tensor/numeric.gen_test.go.tmpl index 34f01b40d8084..665bb8dc68da9 100644 --- a/go/arrow/tensor/numeric.gen_test.go.tmpl +++ b/go/arrow/tensor/numeric.gen_test.go.tmpl @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/tensor" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/tensor" ) {{range .In}} diff --git a/go/arrow/tensor/tensor.go b/go/arrow/tensor/tensor.go index 1f2ed7e82141b..27c380345a498 100644 --- a/go/arrow/tensor/tensor.go +++ b/go/arrow/tensor/tensor.go @@ -21,8 +21,8 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) // Interface represents an n-dimensional array of numerical data. diff --git a/go/arrow/tensor/tensor_test.go b/go/arrow/tensor/tensor_test.go index 552c4c1c223e0..4cfe69f780421 100644 --- a/go/arrow/tensor/tensor_test.go +++ b/go/arrow/tensor/tensor_test.go @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/tensor" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/tensor" ) func TestTensor(t *testing.T) { diff --git a/go/arrow/type_traits_boolean.go b/go/arrow/type_traits_boolean.go index c164d45954fc0..6a46bdec702ff 100644 --- a/go/arrow/type_traits_boolean.go +++ b/go/arrow/type_traits_boolean.go @@ -17,7 +17,7 @@ package arrow import ( - "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/bitutil" ) type booleanTraits struct{} diff --git a/go/arrow/type_traits_decimal128.go b/go/arrow/type_traits_decimal128.go index f573ad3c65a4c..d2d3aae371663 100644 --- a/go/arrow/type_traits_decimal128.go +++ b/go/arrow/type_traits_decimal128.go @@ -20,8 +20,8 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/endian" ) // Decimal128 traits diff --git a/go/arrow/type_traits_decimal256.go b/go/arrow/type_traits_decimal256.go index adf3cc3e0bc31..256ed68fffc8d 100644 --- a/go/arrow/type_traits_decimal256.go +++ b/go/arrow/type_traits_decimal256.go @@ -20,8 +20,8 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/endian" ) // Decimal256 traits diff --git a/go/arrow/type_traits_float16.go b/go/arrow/type_traits_float16.go index e59efd4c248d8..c40363d3785da 100644 --- a/go/arrow/type_traits_float16.go +++ b/go/arrow/type_traits_float16.go @@ -20,8 +20,8 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/float16" ) // Float16 traits diff --git a/go/arrow/type_traits_interval.go b/go/arrow/type_traits_interval.go index 5fbd7a5248918..35e6057090f10 100644 --- a/go/arrow/type_traits_interval.go +++ b/go/arrow/type_traits_interval.go @@ -20,8 +20,8 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/internal/debug" ) var ( diff --git a/go/arrow/type_traits_numeric.gen.go b/go/arrow/type_traits_numeric.gen.go index 57606c0fce6df..6edd75291155c 100644 --- a/go/arrow/type_traits_numeric.gen.go +++ b/go/arrow/type_traits_numeric.gen.go @@ -23,7 +23,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/endian" ) var ( diff --git a/go/arrow/type_traits_numeric.gen.go.tmpl b/go/arrow/type_traits_numeric.gen.go.tmpl index c491047b51429..ffae975c1aa15 100644 --- a/go/arrow/type_traits_numeric.gen.go.tmpl +++ b/go/arrow/type_traits_numeric.gen.go.tmpl @@ -21,7 +21,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/endian" ) var ( diff --git a/go/arrow/type_traits_numeric.gen_test.go b/go/arrow/type_traits_numeric.gen_test.go index ac2d0726ed6fa..74395485736ff 100644 --- a/go/arrow/type_traits_numeric.gen_test.go +++ b/go/arrow/type_traits_numeric.gen_test.go @@ -22,7 +22,7 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) func TestInt64Traits(t *testing.T) { diff --git a/go/arrow/type_traits_numeric.gen_test.go.tmpl b/go/arrow/type_traits_numeric.gen_test.go.tmpl index 4948f42cf542e..96685f31327bd 100644 --- a/go/arrow/type_traits_numeric.gen_test.go.tmpl +++ b/go/arrow/type_traits_numeric.gen_test.go.tmpl @@ -20,7 +20,7 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) {{- range .In}} diff --git a/go/arrow/type_traits_test.go b/go/arrow/type_traits_test.go index 0ae88b4bd82b6..85ad4da9d041d 100644 --- a/go/arrow/type_traits_test.go +++ b/go/arrow/type_traits_test.go @@ -22,10 +22,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/float16" ) func TestBooleanTraits(t *testing.T) { diff --git a/go/arrow/type_traits_timestamp.go b/go/arrow/type_traits_timestamp.go index c1a9aba3db386..7c393b3559f82 100644 --- a/go/arrow/type_traits_timestamp.go +++ b/go/arrow/type_traits_timestamp.go @@ -20,7 +20,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/endian" ) var TimestampTraits timestampTraits diff --git a/go/arrow/util/byte_size.go b/go/arrow/util/byte_size.go index 6d6fc021f8005..141b78c459119 100644 --- a/go/arrow/util/byte_size.go +++ b/go/arrow/util/byte_size.go @@ -17,9 +17,9 @@ package util import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" ) func isArrayDataNil(arrayData arrow.ArrayData) bool { diff --git a/go/arrow/util/byte_size_test.go b/go/arrow/util/byte_size_test.go index a218c69558fe9..052bf3adee158 100644 --- a/go/arrow/util/byte_size_test.go +++ b/go/arrow/util/byte_size_test.go @@ -20,10 +20,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/util" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/arrow/util" "github.com/stretchr/testify/assert" ) diff --git a/go/go.mod b/go/go.mod index a6c2af7025d32..de73c210d159d 100644 --- a/go/go.mod +++ b/go/go.mod @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -module github.com/apache/arrow/go/v15 +module github.com/apache/arrow/go/v14 go 1.20 diff --git a/go/internal/bitutils/bit_block_counter.go b/go/internal/bitutils/bit_block_counter.go index 99eece34cd068..86818bfd45ab7 100644 --- a/go/internal/bitutils/bit_block_counter.go +++ b/go/internal/bitutils/bit_block_counter.go @@ -21,8 +21,8 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/internal/utils" ) func loadWord(byt []byte) uint64 { diff --git a/go/internal/bitutils/bit_block_counter_test.go b/go/internal/bitutils/bit_block_counter_test.go index 790105c290182..ad5e989d1fa65 100644 --- a/go/internal/bitutils/bit_block_counter_test.go +++ b/go/internal/bitutils/bit_block_counter_test.go @@ -19,9 +19,9 @@ package bitutils_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" "github.com/stretchr/testify/assert" "golang.org/x/exp/rand" ) diff --git a/go/internal/bitutils/bit_run_reader.go b/go/internal/bitutils/bit_run_reader.go index f09149d7ec5df..a1686a4909b8b 100644 --- a/go/internal/bitutils/bit_run_reader.go +++ b/go/internal/bitutils/bit_run_reader.go @@ -22,9 +22,9 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/internal/utils" ) // BitRun represents a run of bits with the same value of length Len diff --git a/go/internal/bitutils/bit_run_reader_test.go b/go/internal/bitutils/bit_run_reader_test.go index 7db76768a9476..0e1500bff811a 100644 --- a/go/internal/bitutils/bit_run_reader_test.go +++ b/go/internal/bitutils/bit_run_reader_test.go @@ -21,9 +21,9 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/internal/bitutils" "github.com/stretchr/testify/assert" ) diff --git a/go/internal/bitutils/bit_set_run_reader.go b/go/internal/bitutils/bit_set_run_reader.go index 6764ca7912679..a2269ffecb870 100644 --- a/go/internal/bitutils/bit_set_run_reader.go +++ b/go/internal/bitutils/bit_set_run_reader.go @@ -20,8 +20,8 @@ import ( "encoding/binary" "math/bits" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/internal/utils" ) // IsMultipleOf64 returns whether v is a multiple of 64. diff --git a/go/internal/bitutils/bit_set_run_reader_test.go b/go/internal/bitutils/bit_set_run_reader_test.go index 832993671ef6d..205954b3800a8 100644 --- a/go/internal/bitutils/bit_set_run_reader_test.go +++ b/go/internal/bitutils/bit_set_run_reader_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/utils" "github.com/stretchr/testify/suite" ) diff --git a/go/internal/bitutils/bitmap_generate.go b/go/internal/bitutils/bitmap_generate.go index 08b5fceab57d3..78219d8120bcc 100644 --- a/go/internal/bitutils/bitmap_generate.go +++ b/go/internal/bitutils/bitmap_generate.go @@ -16,7 +16,7 @@ package bitutils -import "github.com/apache/arrow/go/v15/arrow/bitutil" +import "github.com/apache/arrow/go/v14/arrow/bitutil" // GenerateBits writes sequential bits to a bitmap. Bits preceding the // initial start offset are preserved, bits following the bitmap may diff --git a/go/internal/bitutils/bitmap_generate_test.go b/go/internal/bitutils/bitmap_generate_test.go index c9a6203864a20..788b2198c3a78 100644 --- a/go/internal/bitutils/bitmap_generate_test.go +++ b/go/internal/bitutils/bitmap_generate_test.go @@ -19,7 +19,7 @@ package bitutils_test import ( "testing" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/bitutils" "golang.org/x/exp/rand" ) diff --git a/go/internal/hashing/xxh3_memo_table.gen.go b/go/internal/hashing/xxh3_memo_table.gen.go index 39b82cdeff9a2..cc996552b49e5 100644 --- a/go/internal/hashing/xxh3_memo_table.gen.go +++ b/go/internal/hashing/xxh3_memo_table.gen.go @@ -21,9 +21,9 @@ package hashing import ( "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/internal/utils" ) type payloadInt8 struct { diff --git a/go/internal/hashing/xxh3_memo_table.gen.go.tmpl b/go/internal/hashing/xxh3_memo_table.gen.go.tmpl index 527008ad63c3c..25164341d227e 100644 --- a/go/internal/hashing/xxh3_memo_table.gen.go.tmpl +++ b/go/internal/hashing/xxh3_memo_table.gen.go.tmpl @@ -17,8 +17,8 @@ package hashing import ( - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/internal/utils" ) {{range .In}} diff --git a/go/internal/types/extension_types.go b/go/internal/types/extension_types.go index e24c89efc7b8b..1a31014874bbe 100644 --- a/go/internal/types/extension_types.go +++ b/go/internal/types/extension_types.go @@ -24,9 +24,9 @@ import ( "reflect" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/internal/json" "github.com/google/uuid" "golang.org/x/xerrors" ) diff --git a/go/internal/types/extension_types_test.go b/go/internal/types/extension_types_test.go index f93f1000c9e4a..16273890b4c9c 100644 --- a/go/internal/types/extension_types_test.go +++ b/go/internal/types/extension_types_test.go @@ -20,11 +20,11 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v14/internal/types" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/internal/utils/transpose_ints_def.go b/go/internal/utils/transpose_ints_def.go index c52598d7148ea..cc3b0abb59363 100644 --- a/go/internal/utils/transpose_ints_def.go +++ b/go/internal/utils/transpose_ints_def.go @@ -19,7 +19,7 @@ package utils import ( "errors" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) //go:generate go run ../../arrow/_tools/tmpl -i -data=transpose_ints.tmpldata -d arch=avx2 transpose_ints_simd.go.tmpl=transpose_ints_avx2_amd64.go diff --git a/go/internal/utils/transpose_ints_test.go b/go/internal/utils/transpose_ints_test.go index 73b2bbce3fc14..e41b61156abdc 100644 --- a/go/internal/utils/transpose_ints_test.go +++ b/go/internal/utils/transpose_ints_test.go @@ -22,7 +22,7 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/internal/utils" ) var ( diff --git a/go/parquet/cmd/parquet_reader/dumper.go b/go/parquet/cmd/parquet_reader/dumper.go index 4cb2ea4a96fee..a04dcf23b2d8d 100644 --- a/go/parquet/cmd/parquet_reader/dumper.go +++ b/go/parquet/cmd/parquet_reader/dumper.go @@ -22,9 +22,9 @@ import ( "reflect" "time" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/schema" ) const defaultBatchSize = 128 diff --git a/go/parquet/cmd/parquet_reader/main.go b/go/parquet/cmd/parquet_reader/main.go index 0d651d8c294c5..e7d1cd7fc5a7d 100644 --- a/go/parquet/cmd/parquet_reader/main.go +++ b/go/parquet/cmd/parquet_reader/main.go @@ -25,11 +25,11 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/docopt/docopt-go" ) diff --git a/go/parquet/cmd/parquet_schema/main.go b/go/parquet/cmd/parquet_schema/main.go index 01d541e8ac925..28d1288b54aeb 100644 --- a/go/parquet/cmd/parquet_schema/main.go +++ b/go/parquet/cmd/parquet_schema/main.go @@ -20,8 +20,8 @@ import ( "fmt" "os" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/docopt/docopt-go" ) diff --git a/go/parquet/compress/brotli.go b/go/parquet/compress/brotli.go index a1199d9711435..1c1ae1dffaca5 100644 --- a/go/parquet/compress/brotli.go +++ b/go/parquet/compress/brotli.go @@ -21,7 +21,7 @@ import ( "io" "github.com/andybalholm/brotli" - "github.com/apache/arrow/go/v15/parquet/internal/debug" + "github.com/apache/arrow/go/v14/parquet/internal/debug" ) type brotliCodec struct{} diff --git a/go/parquet/compress/compress.go b/go/parquet/compress/compress.go index f61147eb1ea10..4c4c792c2f614 100644 --- a/go/parquet/compress/compress.go +++ b/go/parquet/compress/compress.go @@ -23,7 +23,7 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" ) // Compression is an alias to the thrift compression codec enum type for easy use diff --git a/go/parquet/compress/compress_test.go b/go/parquet/compress/compress_test.go index d1c55b15bc3cc..e8041a64c0996 100644 --- a/go/parquet/compress/compress_test.go +++ b/go/parquet/compress/compress_test.go @@ -22,7 +22,7 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/compress" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/compress/zstd.go b/go/parquet/compress/zstd.go index fd8ec81953359..129af62807175 100644 --- a/go/parquet/compress/zstd.go +++ b/go/parquet/compress/zstd.go @@ -20,7 +20,7 @@ import ( "io" "sync" - "github.com/apache/arrow/go/v15/parquet/internal/debug" + "github.com/apache/arrow/go/v14/parquet/internal/debug" "github.com/klauspost/compress/zstd" ) diff --git a/go/parquet/doc.go b/go/parquet/doc.go index afeee00587ef5..99a605a284fae 100644 --- a/go/parquet/doc.go +++ b/go/parquet/doc.go @@ -29,9 +29,9 @@ // Install // // You can download the library and cli utilities via: -// go get -u github.com/apache/arrow/go/v15/parquet -// go install github.com/apache/arrow/go/v15/parquet/cmd/parquet_reader@latest -// go install github.com/apache/arrow/go/v15/parquet/cmd/parquet_schema@latest +// go get -u github.com/apache/arrow/go/v14/parquet +// go install github.com/apache/arrow/go/v14/parquet/cmd/parquet_reader@latest +// go install github.com/apache/arrow/go/v14/parquet/cmd/parquet_schema@latest // // Modules // diff --git a/go/parquet/encryption_properties.go b/go/parquet/encryption_properties.go index 6eb6cf1fe5680..dd85a856820fe 100644 --- a/go/parquet/encryption_properties.go +++ b/go/parquet/encryption_properties.go @@ -20,7 +20,7 @@ import ( "crypto/rand" "unicode/utf8" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" ) // Constants that will be used as the default values with encryption/decryption diff --git a/go/parquet/encryption_properties_test.go b/go/parquet/encryption_properties_test.go index ab028927c5ecb..0b1118a5cfa66 100644 --- a/go/parquet/encryption_properties_test.go +++ b/go/parquet/encryption_properties_test.go @@ -19,8 +19,8 @@ package parquet_test import ( "testing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/encryption_read_config_test.go b/go/parquet/encryption_read_config_test.go index 53b7ba3c621c0..8a84c19900dd0 100644 --- a/go/parquet/encryption_read_config_test.go +++ b/go/parquet/encryption_read_config_test.go @@ -23,10 +23,10 @@ import ( "path" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/encryption_write_config_test.go b/go/parquet/encryption_write_config_test.go index 558b89208c706..6b2d049e1390c 100644 --- a/go/parquet/encryption_write_config_test.go +++ b/go/parquet/encryption_write_config_test.go @@ -23,10 +23,10 @@ import ( "path/filepath" "testing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/file/column_reader.go b/go/parquet/file/column_reader.go index 766638d88f26c..b623bd5074d02 100644 --- a/go/parquet/file/column_reader.go +++ b/go/parquet/file/column_reader.go @@ -21,13 +21,13 @@ import ( "fmt" "sync" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/column_reader_test.go b/go/parquet/file/column_reader_test.go index 21ea52e2b7bbc..3ce05c6cfe47e 100755 --- a/go/parquet/file/column_reader_test.go +++ b/go/parquet/file/column_reader_test.go @@ -24,12 +24,12 @@ import ( "sync" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/testutils" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/file/column_reader_types.gen.go b/go/parquet/file/column_reader_types.gen.go index 3fb113780f811..31dccc157642f 100644 --- a/go/parquet/file/column_reader_types.gen.go +++ b/go/parquet/file/column_reader_types.gen.go @@ -21,9 +21,9 @@ package file import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" ) // Int32ColumnChunkReader is the Typed Column chunk reader instance for reading diff --git a/go/parquet/file/column_reader_types.gen.go.tmpl b/go/parquet/file/column_reader_types.gen.go.tmpl index 261b5f0bfacab..6557963e69f50 100644 --- a/go/parquet/file/column_reader_types.gen.go.tmpl +++ b/go/parquet/file/column_reader_types.gen.go.tmpl @@ -17,8 +17,8 @@ package file import ( - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" ) {{range .In}} diff --git a/go/parquet/file/column_writer.go b/go/parquet/file/column_writer.go index 0b0d1145d604e..9d9111042df28 100755 --- a/go/parquet/file/column_writer.go +++ b/go/parquet/file/column_writer.go @@ -21,14 +21,14 @@ import ( "encoding/binary" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/schema" ) //go:generate go run ../../arrow/_tools/tmpl/main.go -i -data=../internal/encoding/physical_types.tmpldata column_writer_types.gen.go.tmpl diff --git a/go/parquet/file/column_writer_test.go b/go/parquet/file/column_writer_test.go index 134c290fa6894..e2ae6b0f4a71e 100755 --- a/go/parquet/file/column_writer_test.go +++ b/go/parquet/file/column_writer_test.go @@ -24,19 +24,19 @@ import ( "sync" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - arrutils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + arrutils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/testutils" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" diff --git a/go/parquet/file/column_writer_types.gen.go b/go/parquet/file/column_writer_types.gen.go index 8b3be25f3ea29..5594f63249fb8 100644 --- a/go/parquet/file/column_writer_types.gen.go +++ b/go/parquet/file/column_writer_types.gen.go @@ -22,12 +22,12 @@ import ( "errors" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/column_writer_types.gen.go.tmpl b/go/parquet/file/column_writer_types.gen.go.tmpl index 7df69b4a219c6..c00e1dabb5fe6 100644 --- a/go/parquet/file/column_writer_types.gen.go.tmpl +++ b/go/parquet/file/column_writer_types.gen.go.tmpl @@ -19,10 +19,10 @@ package file import ( "fmt" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" ) {{range .In}} diff --git a/go/parquet/file/file_reader.go b/go/parquet/file/file_reader.go index afff579ded5b7..5481d5c7738b0 100644 --- a/go/parquet/file/file_reader.go +++ b/go/parquet/file/file_reader.go @@ -25,10 +25,10 @@ import ( "runtime" "sync" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + "github.com/apache/arrow/go/v14/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/file_reader_mmap.go b/go/parquet/file/file_reader_mmap.go index 03e12adf08c8c..89c5a84cee2e1 100644 --- a/go/parquet/file/file_reader_mmap.go +++ b/go/parquet/file/file_reader_mmap.go @@ -22,7 +22,7 @@ package file import ( "io" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/parquet" "golang.org/x/exp/mmap" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/file_reader_mmap_windows.go b/go/parquet/file/file_reader_mmap_windows.go index 06a9e97160fe0..6b08422163fd1 100644 --- a/go/parquet/file/file_reader_mmap_windows.go +++ b/go/parquet/file/file_reader_mmap_windows.go @@ -22,7 +22,7 @@ package file import ( "errors" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/parquet" ) func mmapOpen(filename string) (parquet.ReaderAtSeeker, error) { diff --git a/go/parquet/file/file_reader_test.go b/go/parquet/file/file_reader_test.go index f3248925cf5b9..2a9b097139062 100644 --- a/go/parquet/file/file_reader_test.go +++ b/go/parquet/file/file_reader_test.go @@ -25,16 +25,16 @@ import ( "path" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/thrift" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/schema" libthrift "github.com/apache/thrift/lib/go/thrift" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/parquet/file/file_writer.go b/go/parquet/file/file_writer.go index 48eb8f44d167b..c6289434bbe6e 100644 --- a/go/parquet/file/file_writer.go +++ b/go/parquet/file/file_writer.go @@ -21,11 +21,11 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/schema" ) // Writer is the primary interface for writing a parquet file diff --git a/go/parquet/file/file_writer_test.go b/go/parquet/file/file_writer_test.go index beceff9732584..af083ebe60e4f 100644 --- a/go/parquet/file/file_writer_test.go +++ b/go/parquet/file/file_writer_test.go @@ -22,13 +22,13 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/internal/testutils" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/parquet/file/level_conversion.go b/go/parquet/file/level_conversion.go index c23bdda445963..ac53dcd18abae 100755 --- a/go/parquet/file/level_conversion.go +++ b/go/parquet/file/level_conversion.go @@ -22,11 +22,11 @@ import ( "math/bits" "unsafe" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/bmi" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/schema" + shared_utils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/bmi" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/level_conversion_test.go b/go/parquet/file/level_conversion_test.go index 5d5bdde90dc7e..343edb687980d 100644 --- a/go/parquet/file/level_conversion_test.go +++ b/go/parquet/file/level_conversion_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/parquet/internal/bmi" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/parquet/internal/bmi" + "github.com/apache/arrow/go/v14/parquet/internal/utils" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/file/page_reader.go b/go/parquet/file/page_reader.go index 01f253aff8864..ab8679b127f1b 100644 --- a/go/parquet/file/page_reader.go +++ b/go/parquet/file/page_reader.go @@ -23,13 +23,13 @@ import ( "sync" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/thrift" + "github.com/apache/arrow/go/v14/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/page_writer.go b/go/parquet/file/page_writer.go index c16476fbb232c..b2b6d5061b0fc 100644 --- a/go/parquet/file/page_writer.go +++ b/go/parquet/file/page_writer.go @@ -20,15 +20,15 @@ import ( "bytes" "sync" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/thrift" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v14/parquet/metadata" libthrift "github.com/apache/thrift/lib/go/thrift" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/record_reader.go b/go/parquet/file/record_reader.go index ad836d29ef483..3311a6d77da39 100755 --- a/go/parquet/file/record_reader.go +++ b/go/parquet/file/record_reader.go @@ -23,14 +23,14 @@ import ( "unsafe" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/row_group_reader.go b/go/parquet/file/row_group_reader.go index 3c1c1edb0b484..f1aebf1085a23 100644 --- a/go/parquet/file/row_group_reader.go +++ b/go/parquet/file/row_group_reader.go @@ -20,10 +20,10 @@ import ( "fmt" "sync" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + "github.com/apache/arrow/go/v14/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/row_group_writer.go b/go/parquet/file/row_group_writer.go index 935c13d13a67f..75f0dede5246f 100644 --- a/go/parquet/file/row_group_writer.go +++ b/go/parquet/file/row_group_writer.go @@ -17,10 +17,10 @@ package file import ( - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v14/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/row_group_writer_test.go b/go/parquet/file/row_group_writer_test.go index 2ea317cb91001..bc95e99d91582 100644 --- a/go/parquet/file/row_group_writer_test.go +++ b/go/parquet/file/row_group_writer_test.go @@ -20,10 +20,10 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/apache/thrift/lib/go/thrift" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/internal/bmi/bmi_test.go b/go/parquet/internal/bmi/bmi_test.go index a5278dfef2211..460c6ec4e24c6 100644 --- a/go/parquet/internal/bmi/bmi_test.go +++ b/go/parquet/internal/bmi/bmi_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/parquet/internal/bmi" + "github.com/apache/arrow/go/v14/parquet/internal/bmi" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/internal/encoding/boolean_decoder.go b/go/parquet/internal/encoding/boolean_decoder.go index 3782dc85ea814..337a6db967a2e 100644 --- a/go/parquet/internal/encoding/boolean_decoder.go +++ b/go/parquet/internal/encoding/boolean_decoder.go @@ -23,10 +23,10 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow/bitutil" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + shared_utils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/utils" ) // PlainBooleanDecoder is for the Plain Encoding type, there is no diff --git a/go/parquet/internal/encoding/boolean_encoder.go b/go/parquet/internal/encoding/boolean_encoder.go index 3e01bde369d8b..3970e05fca289 100644 --- a/go/parquet/internal/encoding/boolean_encoder.go +++ b/go/parquet/internal/encoding/boolean_encoder.go @@ -19,10 +19,10 @@ package encoding import ( "encoding/binary" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/debug" + "github.com/apache/arrow/go/v14/parquet/internal/utils" ) const ( diff --git a/go/parquet/internal/encoding/byte_array_decoder.go b/go/parquet/internal/encoding/byte_array_decoder.go index 82ce9f84265c5..c3ca1de5888f7 100644 --- a/go/parquet/internal/encoding/byte_array_decoder.go +++ b/go/parquet/internal/encoding/byte_array_decoder.go @@ -19,12 +19,12 @@ package encoding import ( "encoding/binary" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - pqutils "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + pqutils "github.com/apache/arrow/go/v14/parquet/internal/utils" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/byte_array_encoder.go b/go/parquet/internal/encoding/byte_array_encoder.go index 9270b5531768c..cb49178435a6f 100644 --- a/go/parquet/internal/encoding/byte_array_encoder.go +++ b/go/parquet/internal/encoding/byte_array_encoder.go @@ -21,11 +21,11 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" ) // PlainByteArrayEncoder encodes byte arrays according to the spec for Plain encoding diff --git a/go/parquet/internal/encoding/decoder.go b/go/parquet/internal/encoding/decoder.go index cee624730e993..b441868e1df85 100644 --- a/go/parquet/internal/encoding/decoder.go +++ b/go/parquet/internal/encoding/decoder.go @@ -20,16 +20,16 @@ import ( "bytes" "reflect" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" + shared_utils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/debug" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/delta_bit_packing.go b/go/parquet/internal/encoding/delta_bit_packing.go index a00f3457cac7a..1fb91634e977b 100644 --- a/go/parquet/internal/encoding/delta_bit_packing.go +++ b/go/parquet/internal/encoding/delta_bit_packing.go @@ -23,11 +23,11 @@ import ( "math/bits" "reflect" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" + shared_utils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/utils" ) // see the deltaBitPack encoder for a description of the encoding format that is diff --git a/go/parquet/internal/encoding/delta_byte_array.go b/go/parquet/internal/encoding/delta_byte_array.go index b35d022fd5deb..60a20a3ff0d72 100644 --- a/go/parquet/internal/encoding/delta_byte_array.go +++ b/go/parquet/internal/encoding/delta_byte_array.go @@ -17,9 +17,9 @@ package encoding import ( - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/delta_byte_array_test.go b/go/parquet/internal/encoding/delta_byte_array_test.go index 0a206796f742a..69c632bf69971 100644 --- a/go/parquet/internal/encoding/delta_byte_array_test.go +++ b/go/parquet/internal/encoding/delta_byte_array_test.go @@ -18,8 +18,8 @@ package encoding import ( "fmt" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" "github.com/stretchr/testify/assert" "testing" ) diff --git a/go/parquet/internal/encoding/delta_length_byte_array.go b/go/parquet/internal/encoding/delta_length_byte_array.go index d5a99c187d11e..981a067b07e08 100644 --- a/go/parquet/internal/encoding/delta_length_byte_array.go +++ b/go/parquet/internal/encoding/delta_length_byte_array.go @@ -17,9 +17,9 @@ package encoding import ( - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/encoder.go b/go/parquet/internal/encoding/encoder.go index 7023309397a3b..f6b57fe63c433 100644 --- a/go/parquet/internal/encoding/encoder.go +++ b/go/parquet/internal/encoding/encoder.go @@ -21,14 +21,14 @@ import ( "math/bits" "reflect" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v14/parquet/schema" ) //go:generate go run ../../../arrow/_tools/tmpl/main.go -i -data=physical_types.tmpldata plain_encoder_types.gen.go.tmpl typed_encoder.gen.go.tmpl diff --git a/go/parquet/internal/encoding/encoding_benchmarks_test.go b/go/parquet/internal/encoding/encoding_benchmarks_test.go index e0645e9de54e4..0bb4e895f316d 100644 --- a/go/parquet/internal/encoding/encoding_benchmarks_test.go +++ b/go/parquet/internal/encoding/encoding_benchmarks_test.go @@ -21,14 +21,14 @@ import ( "math" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/hashing" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/internal/testutils" + "github.com/apache/arrow/go/v14/parquet/schema" ) const ( diff --git a/go/parquet/internal/encoding/encoding_test.go b/go/parquet/internal/encoding/encoding_test.go index 48e2316b0f897..b0d86321e09f1 100644 --- a/go/parquet/internal/encoding/encoding_test.go +++ b/go/parquet/internal/encoding/encoding_test.go @@ -26,13 +26,13 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/internal/testutils" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go b/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go index 1e589fc2e7be1..e4516bf4a1ac1 100644 --- a/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go +++ b/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go @@ -19,8 +19,8 @@ package encoding import ( "math" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go b/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go index 39202c8e25d9f..b5aee337d5bf0 100644 --- a/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go +++ b/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go @@ -19,9 +19,9 @@ package encoding import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/parquet" ) // PlainFixedLenByteArrayEncoder writes the raw bytes of the byte array diff --git a/go/parquet/internal/encoding/levels.go b/go/parquet/internal/encoding/levels.go index caf832059334b..e04ec19d5473d 100644 --- a/go/parquet/internal/encoding/levels.go +++ b/go/parquet/internal/encoding/levels.go @@ -23,11 +23,11 @@ import ( "math/bits" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + shared_utils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/utils" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/levels_test.go b/go/parquet/internal/encoding/levels_test.go index 304ce32b3106d..ae3a3a8c497fc 100644 --- a/go/parquet/internal/encoding/levels_test.go +++ b/go/parquet/internal/encoding/levels_test.go @@ -21,11 +21,11 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/internal/encoding/memo_table.go b/go/parquet/internal/encoding/memo_table.go index 810e8633b886e..170eab47d6160 100644 --- a/go/parquet/internal/encoding/memo_table.go +++ b/go/parquet/internal/encoding/memo_table.go @@ -20,11 +20,11 @@ import ( "math" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/hashing" + "github.com/apache/arrow/go/v14/parquet" ) //go:generate go run ../../../arrow/_tools/tmpl/main.go -i -data=physical_types.tmpldata memo_table_types.gen.go.tmpl diff --git a/go/parquet/internal/encoding/memo_table_test.go b/go/parquet/internal/encoding/memo_table_test.go index 1b9337010f855..1eb61cb2559a3 100644 --- a/go/parquet/internal/encoding/memo_table_test.go +++ b/go/parquet/internal/encoding/memo_table_test.go @@ -20,11 +20,11 @@ import ( "math" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/hashing" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/internal/encoding/memo_table_types.gen.go b/go/parquet/internal/encoding/memo_table_types.gen.go index 4da2721437814..c9af6178bbb98 100644 --- a/go/parquet/internal/encoding/memo_table_types.gen.go +++ b/go/parquet/internal/encoding/memo_table_types.gen.go @@ -19,8 +19,8 @@ package encoding import ( - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" ) // standard map based implementation of memo tables which can be more efficient diff --git a/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl b/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl index 75335f25ff1f7..7e88238d67b71 100644 --- a/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl +++ b/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl @@ -17,7 +17,7 @@ package encoding import ( - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/parquet" ) // standard map based implementation of memo tables which can be more efficient diff --git a/go/parquet/internal/encoding/plain_encoder_types.gen.go b/go/parquet/internal/encoding/plain_encoder_types.gen.go index 09403d74cb06f..23db13b7a4121 100644 --- a/go/parquet/internal/encoding/plain_encoder_types.gen.go +++ b/go/parquet/internal/encoding/plain_encoder_types.gen.go @@ -24,11 +24,11 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl b/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl index 2838c63a41857..e45419050302d 100644 --- a/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl +++ b/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl @@ -20,10 +20,10 @@ import ( "encoding/binary" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/internal/bitutils" ) var ( diff --git a/go/parquet/internal/encoding/typed_encoder.gen.go b/go/parquet/internal/encoding/typed_encoder.gen.go index 25fa309e0a38f..411e87c17e7d5 100644 --- a/go/parquet/internal/encoding/typed_encoder.gen.go +++ b/go/parquet/internal/encoding/typed_encoder.gen.go @@ -22,15 +22,15 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" + shared_utils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl b/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl index 66c2649d599c6..69415ccca4a26 100644 --- a/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl +++ b/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl @@ -17,13 +17,13 @@ package encoding import ( - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + shared_utils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/internal/bitutils" ) // fully typed encoder interfaces to enable writing against encoder/decoders diff --git a/go/parquet/internal/encoding/types.go b/go/parquet/internal/encoding/types.go index 4ab3ab1a1c954..2d13a9c8f877a 100644 --- a/go/parquet/internal/encoding/types.go +++ b/go/parquet/internal/encoding/types.go @@ -20,11 +20,11 @@ import ( "io" "sync" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encryption/aes.go b/go/parquet/internal/encryption/aes.go index b6e9130ef81cd..cc0cea0112fee 100644 --- a/go/parquet/internal/encryption/aes.go +++ b/go/parquet/internal/encryption/aes.go @@ -29,7 +29,7 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/parquet" ) // important constants for handling the aes encryption diff --git a/go/parquet/internal/encryption/decryptor.go b/go/parquet/internal/encryption/decryptor.go index 658559e6082c4..62a4f2ee9e56e 100644 --- a/go/parquet/internal/encryption/decryptor.go +++ b/go/parquet/internal/encryption/decryptor.go @@ -19,8 +19,8 @@ package encryption import ( "io" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" ) // FileDecryptor is an interface used by the filereader for decrypting an diff --git a/go/parquet/internal/encryption/encryptor.go b/go/parquet/internal/encryption/encryptor.go index bdbae4740a44f..2ef09e558a23f 100644 --- a/go/parquet/internal/encryption/encryptor.go +++ b/go/parquet/internal/encryption/encryptor.go @@ -19,8 +19,8 @@ package encryption import ( "io" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" ) // FileEncryptor is the interface for constructing encryptors for the different diff --git a/go/parquet/internal/testutils/pagebuilder.go b/go/parquet/internal/testutils/pagebuilder.go index 48ac331640087..9c1fd72a1bdf0 100644 --- a/go/parquet/internal/testutils/pagebuilder.go +++ b/go/parquet/internal/testutils/pagebuilder.go @@ -22,13 +22,13 @@ import ( "io" "reflect" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/mock" ) diff --git a/go/parquet/internal/testutils/primitive_typed.go b/go/parquet/internal/testutils/primitive_typed.go index daab0d427ba10..2bd0a29dd2970 100644 --- a/go/parquet/internal/testutils/primitive_typed.go +++ b/go/parquet/internal/testutils/primitive_typed.go @@ -20,11 +20,11 @@ import ( "fmt" "reflect" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/schema" ) type PrimitiveTypedTest struct { diff --git a/go/parquet/internal/testutils/random.go b/go/parquet/internal/testutils/random.go index bb101ebf9a92d..d9a06da43ba4e 100644 --- a/go/parquet/internal/testutils/random.go +++ b/go/parquet/internal/testutils/random.go @@ -24,13 +24,13 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/pqarrow" "golang.org/x/exp/rand" "gonum.org/v1/gonum/stat/distuv" diff --git a/go/parquet/internal/testutils/random_arrow.go b/go/parquet/internal/testutils/random_arrow.go index d886db0360b84..360b8e7476430 100644 --- a/go/parquet/internal/testutils/random_arrow.go +++ b/go/parquet/internal/testutils/random_arrow.go @@ -17,9 +17,9 @@ package testutils import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" "golang.org/x/exp/rand" ) diff --git a/go/parquet/internal/testutils/utils.go b/go/parquet/internal/testutils/utils.go index 3da76c17ddc32..1122f4286f386 100644 --- a/go/parquet/internal/testutils/utils.go +++ b/go/parquet/internal/testutils/utils.go @@ -19,7 +19,7 @@ package testutils import ( "reflect" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/parquet" ) var typeToParquetTypeMap = map[reflect.Type]parquet.Type{ diff --git a/go/parquet/internal/thrift/helpers.go b/go/parquet/internal/thrift/helpers.go index 3835830ac6c2d..357d9020989d3 100644 --- a/go/parquet/internal/thrift/helpers.go +++ b/go/parquet/internal/thrift/helpers.go @@ -23,7 +23,7 @@ import ( "context" "io" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" "github.com/apache/thrift/lib/go/thrift" ) diff --git a/go/parquet/internal/utils/bit_benchmark_test.go b/go/parquet/internal/utils/bit_benchmark_test.go index 14353380a5694..2518e48a4baa8 100644 --- a/go/parquet/internal/utils/bit_benchmark_test.go +++ b/go/parquet/internal/utils/bit_benchmark_test.go @@ -20,9 +20,9 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/parquet/internal/testutils" ) type linearBitRunReader struct { diff --git a/go/parquet/internal/utils/bit_packing_arm64.go b/go/parquet/internal/utils/bit_packing_arm64.go index e9fb2ef1d810b..92fe0f97b77c3 100644 --- a/go/parquet/internal/utils/bit_packing_arm64.go +++ b/go/parquet/internal/utils/bit_packing_arm64.go @@ -23,7 +23,7 @@ import ( "github.com/klauspost/cpuid/v2" // import for side effect of initializing feature flags // based on ARM_ENABLE_EXT env var - _ "github.com/apache/arrow/go/v15/parquet/internal/bmi" + _ "github.com/apache/arrow/go/v14/parquet/internal/bmi" ) func init() { diff --git a/go/parquet/internal/utils/bit_reader.go b/go/parquet/internal/utils/bit_reader.go index bf9741c79878b..cf61395c0b16c 100644 --- a/go/parquet/internal/utils/bit_reader.go +++ b/go/parquet/internal/utils/bit_reader.go @@ -24,10 +24,10 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" ) // masks for grabbing the trailing bits based on the number of trailing bits desired diff --git a/go/parquet/internal/utils/bit_reader_test.go b/go/parquet/internal/utils/bit_reader_test.go index 5ce1b799b463b..c285f5165cc0a 100644 --- a/go/parquet/internal/utils/bit_reader_test.go +++ b/go/parquet/internal/utils/bit_reader_test.go @@ -25,11 +25,11 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet/internal/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "golang.org/x/exp/rand" diff --git a/go/parquet/internal/utils/bit_writer.go b/go/parquet/internal/utils/bit_writer.go index 6cb255f5b0473..99a1db065590d 100644 --- a/go/parquet/internal/utils/bit_writer.go +++ b/go/parquet/internal/utils/bit_writer.go @@ -21,7 +21,7 @@ import ( "io" "log" - "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/bitutil" ) // WriterAtBuffer is a convenience struct for providing a WriteAt function diff --git a/go/parquet/internal/utils/bitmap_writer.go b/go/parquet/internal/utils/bitmap_writer.go index 3ef99291e3748..447222139b773 100644 --- a/go/parquet/internal/utils/bitmap_writer.go +++ b/go/parquet/internal/utils/bitmap_writer.go @@ -20,7 +20,7 @@ import ( "encoding/binary" "math/bits" - "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/bitutil" ) // BitmapWriter is an interface for bitmap writers so that we can use multiple diff --git a/go/parquet/internal/utils/bitmap_writer_test.go b/go/parquet/internal/utils/bitmap_writer_test.go index 3dddc7567903e..9f38f1ef291ea 100644 --- a/go/parquet/internal/utils/bitmap_writer_test.go +++ b/go/parquet/internal/utils/bitmap_writer_test.go @@ -22,8 +22,8 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/parquet/internal/utils" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/internal/utils/rle.go b/go/parquet/internal/utils/rle.go index 21ac2ce6b5db5..fef322c6fd560 100644 --- a/go/parquet/internal/utils/rle.go +++ b/go/parquet/internal/utils/rle.go @@ -24,10 +24,10 @@ import ( "encoding/binary" "math" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/utils/typed_rle_dict.gen.go b/go/parquet/internal/utils/typed_rle_dict.gen.go index 886d24564db4b..dff22711b120f 100644 --- a/go/parquet/internal/utils/typed_rle_dict.gen.go +++ b/go/parquet/internal/utils/typed_rle_dict.gen.go @@ -19,9 +19,9 @@ package utils import ( - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl b/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl index abcb419055a92..090275745ff59 100644 --- a/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl +++ b/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl @@ -17,9 +17,9 @@ package utils import ( - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/internal/utils" ) {{range .In}} diff --git a/go/parquet/metadata/app_version.go b/go/parquet/metadata/app_version.go index 9966827026106..85ace57365bd3 100644 --- a/go/parquet/metadata/app_version.go +++ b/go/parquet/metadata/app_version.go @@ -21,8 +21,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" ) var ( diff --git a/go/parquet/metadata/column_chunk.go b/go/parquet/metadata/column_chunk.go index 729f741e1b4f9..e6ea6ff1ea183 100644 --- a/go/parquet/metadata/column_chunk.go +++ b/go/parquet/metadata/column_chunk.go @@ -22,13 +22,13 @@ import ( "io" "reflect" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/thrift" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/metadata/file.go b/go/parquet/metadata/file.go index 3335140c2e1c8..dddd95c5df670 100644 --- a/go/parquet/metadata/file.go +++ b/go/parquet/metadata/file.go @@ -24,12 +24,12 @@ import ( "reflect" "unicode/utf8" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/thrift" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/metadata/metadata_test.go b/go/parquet/metadata/metadata_test.go index 8caa319f83e63..b685dd2223274 100644 --- a/go/parquet/metadata/metadata_test.go +++ b/go/parquet/metadata/metadata_test.go @@ -21,9 +21,9 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/metadata/row_group.go b/go/parquet/metadata/row_group.go index 2923720371abe..16af67241e959 100644 --- a/go/parquet/metadata/row_group.go +++ b/go/parquet/metadata/row_group.go @@ -20,10 +20,10 @@ import ( "fmt" "reflect" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encryption" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" ) // RowGroupMetaData is a proxy around the thrift RowGroup meta data object diff --git a/go/parquet/metadata/stat_compare_test.go b/go/parquet/metadata/stat_compare_test.go index 041696d84d17a..d02123cbef039 100644 --- a/go/parquet/metadata/stat_compare_test.go +++ b/go/parquet/metadata/stat_compare_test.go @@ -20,8 +20,8 @@ import ( "encoding/binary" "testing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/metadata/statistics.go b/go/parquet/metadata/statistics.go index 606e76ffd0454..245b06a7348c4 100644 --- a/go/parquet/metadata/statistics.go +++ b/go/parquet/metadata/statistics.go @@ -22,14 +22,14 @@ import ( "math" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/debug" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" ) //go:generate go run ../../arrow/_tools/tmpl/main.go -i -data=../internal/encoding/physical_types.tmpldata statistics_types.gen.go.tmpl diff --git a/go/parquet/metadata/statistics_test.go b/go/parquet/metadata/statistics_test.go index 47798d3c4dd3c..35d8b7821c51a 100644 --- a/go/parquet/metadata/statistics_test.go +++ b/go/parquet/metadata/statistics_test.go @@ -21,11 +21,11 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/metadata/statistics_types.gen.go b/go/parquet/metadata/statistics_types.gen.go index e8fb9877c8444..e6aa7f1801a0f 100644 --- a/go/parquet/metadata/statistics_types.gen.go +++ b/go/parquet/metadata/statistics_types.gen.go @@ -22,14 +22,14 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" + shared_utils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/metadata/statistics_types.gen.go.tmpl b/go/parquet/metadata/statistics_types.gen.go.tmpl index 4b6253a8574ea..35470f06046bf 100644 --- a/go/parquet/metadata/statistics_types.gen.go.tmpl +++ b/go/parquet/metadata/statistics_types.gen.go.tmpl @@ -19,13 +19,13 @@ package metadata import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v14/parquet/internal/utils" + shared_utils "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/internal/bitutils" ) {{range .In}} diff --git a/go/parquet/pqarrow/column_readers.go b/go/parquet/pqarrow/column_readers.go index 02f94c941c3d9..759a3d8675927 100644 --- a/go/parquet/pqarrow/column_readers.go +++ b/go/parquet/pqarrow/column_readers.go @@ -26,16 +26,16 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/sync/errgroup" ) diff --git a/go/parquet/pqarrow/encode_arrow.go b/go/parquet/pqarrow/encode_arrow.go index 81b4527b105cb..c3a0a50c43f45 100644 --- a/go/parquet/pqarrow/encode_arrow.go +++ b/go/parquet/pqarrow/encode_arrow.go @@ -25,16 +25,16 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/debug" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/debug" ) // get the count of the number of leaf arrays for the type diff --git a/go/parquet/pqarrow/encode_arrow_test.go b/go/parquet/pqarrow/encode_arrow_test.go index 281ca0d526476..3c20cf2d4757b 100644 --- a/go/parquet/pqarrow/encode_arrow_test.go +++ b/go/parquet/pqarrow/encode_arrow_test.go @@ -25,22 +25,22 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/pqarrow" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/parquet/internal/testutils" + "github.com/apache/arrow/go/v14/parquet/pqarrow" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/parquet/pqarrow/encode_dict_compute.go b/go/parquet/pqarrow/encode_dict_compute.go index b43b4002ed0af..59e25cec19132 100644 --- a/go/parquet/pqarrow/encode_dict_compute.go +++ b/go/parquet/pqarrow/encode_dict_compute.go @@ -21,14 +21,14 @@ package pqarrow import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/debug" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" ) func isDictEncoding(enc parquet.Encoding) bool { diff --git a/go/parquet/pqarrow/encode_dict_nocompute.go b/go/parquet/pqarrow/encode_dict_nocompute.go index 73ec5cfc46682..31d0f1bc1ce27 100644 --- a/go/parquet/pqarrow/encode_dict_nocompute.go +++ b/go/parquet/pqarrow/encode_dict_nocompute.go @@ -21,8 +21,8 @@ package pqarrow import ( "errors" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/parquet/file" ) func writeDictionaryArrow(*arrowWriteContext, file.ColumnChunkWriter, arrow.Array, []int16, []int16, bool) (err error) { diff --git a/go/parquet/pqarrow/encode_dictionary_test.go b/go/parquet/pqarrow/encode_dictionary_test.go index 28ebee53e1b83..995f3fb329534 100644 --- a/go/parquet/pqarrow/encode_dictionary_test.go +++ b/go/parquet/pqarrow/encode_dictionary_test.go @@ -26,14 +26,14 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/internal/testutils" + "github.com/apache/arrow/go/v14/parquet/pqarrow" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/parquet/pqarrow/file_reader.go b/go/parquet/pqarrow/file_reader.go index 3534cc87b78b5..d91010c62c19d 100755 --- a/go/parquet/pqarrow/file_reader.go +++ b/go/parquet/pqarrow/file_reader.go @@ -23,13 +23,13 @@ import ( "sync" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/arrio" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/file_reader_test.go b/go/parquet/pqarrow/file_reader_test.go index 9c1b4252f5fc6..d1f3ae1c984a2 100644 --- a/go/parquet/pqarrow/file_reader_test.go +++ b/go/parquet/pqarrow/file_reader_test.go @@ -26,13 +26,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/pqarrow" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/pqarrow/file_writer.go b/go/parquet/pqarrow/file_writer.go index 21f16c0b67938..aa0bae7b1fdfb 100644 --- a/go/parquet/pqarrow/file_writer.go +++ b/go/parquet/pqarrow/file_writer.go @@ -22,12 +22,12 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/helpers.go b/go/parquet/pqarrow/helpers.go index 39d17cf80ae68..e8b740d12ba62 100644 --- a/go/parquet/pqarrow/helpers.go +++ b/go/parquet/pqarrow/helpers.go @@ -17,7 +17,7 @@ package pqarrow import ( - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v14/arrow" ) func releaseArrays(arrays []arrow.Array) { diff --git a/go/parquet/pqarrow/path_builder.go b/go/parquet/pqarrow/path_builder.go index 0f1158bd1e9f0..c4915d58dd59f 100644 --- a/go/parquet/pqarrow/path_builder.go +++ b/go/parquet/pqarrow/path_builder.go @@ -21,11 +21,11 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/path_builder_test.go b/go/parquet/pqarrow/path_builder_test.go index e0a60262d3f4a..16b994582df02 100644 --- a/go/parquet/pqarrow/path_builder_test.go +++ b/go/parquet/pqarrow/path_builder_test.go @@ -20,10 +20,10 @@ import ( "context" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/parquet/pqarrow/properties.go b/go/parquet/pqarrow/properties.go index cc100fa80d87b..98427bbf9398c 100755 --- a/go/parquet/pqarrow/properties.go +++ b/go/parquet/pqarrow/properties.go @@ -19,9 +19,9 @@ package pqarrow import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet/internal/encoding" ) // ArrowWriterProperties are used to determine how to manipulate the arrow data diff --git a/go/parquet/pqarrow/reader_writer_test.go b/go/parquet/pqarrow/reader_writer_test.go index 9d09bcec15da6..e3f66834c2cee 100644 --- a/go/parquet/pqarrow/reader_writer_test.go +++ b/go/parquet/pqarrow/reader_writer_test.go @@ -22,12 +22,12 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/pqarrow" "golang.org/x/exp/rand" "gonum.org/v1/gonum/stat/distuv" ) diff --git a/go/parquet/pqarrow/schema.go b/go/parquet/pqarrow/schema.go index b23c37ea39102..9ba8554898986 100644 --- a/go/parquet/pqarrow/schema.go +++ b/go/parquet/pqarrow/schema.go @@ -22,15 +22,15 @@ import ( "math" "strconv" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/schema_test.go b/go/parquet/pqarrow/schema_test.go index b5e7dc8fad34a..e195358af6c98 100644 --- a/go/parquet/pqarrow/schema_test.go +++ b/go/parquet/pqarrow/schema_test.go @@ -20,15 +20,15 @@ import ( "encoding/base64" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/pqarrow" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v14/parquet/pqarrow" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/reader_properties.go b/go/parquet/reader_properties.go index be2377527d782..dd2032958a51c 100644 --- a/go/parquet/reader_properties.go +++ b/go/parquet/reader_properties.go @@ -21,8 +21,8 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/internal/utils" ) // ReaderProperties are used to define how the file reader will handle buffering and allocating buffers diff --git a/go/parquet/reader_writer_properties_test.go b/go/parquet/reader_writer_properties_test.go index 698129471adda..6bd93a02550e4 100644 --- a/go/parquet/reader_writer_properties_test.go +++ b/go/parquet/reader_writer_properties_test.go @@ -20,9 +20,9 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/column.go b/go/parquet/schema/column.go index 02d0d83e8465e..1575037d59d23 100644 --- a/go/parquet/schema/column.go +++ b/go/parquet/schema/column.go @@ -20,8 +20,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" ) // Column encapsulates the information necessary to interpret primitive diff --git a/go/parquet/schema/converted_types.go b/go/parquet/schema/converted_types.go index 58051dba7f19b..6a4702b819259 100644 --- a/go/parquet/schema/converted_types.go +++ b/go/parquet/schema/converted_types.go @@ -17,7 +17,7 @@ package schema import ( - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" ) // ConvertedType corresponds to the ConvertedType in the parquet.Thrift, diff --git a/go/parquet/schema/converted_types_test.go b/go/parquet/schema/converted_types_test.go index 8b8e061466474..a4f9ea0018196 100644 --- a/go/parquet/schema/converted_types_test.go +++ b/go/parquet/schema/converted_types_test.go @@ -19,7 +19,7 @@ package schema_test import ( "testing" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/helpers.go b/go/parquet/schema/helpers.go index 13075a065f073..23a5a0b667f26 100644 --- a/go/parquet/schema/helpers.go +++ b/go/parquet/schema/helpers.go @@ -17,7 +17,7 @@ package schema import ( - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v14/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/helpers_test.go b/go/parquet/schema/helpers_test.go index 98f3cab36d433..6d136e4ecd155 100644 --- a/go/parquet/schema/helpers_test.go +++ b/go/parquet/schema/helpers_test.go @@ -21,8 +21,8 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/logical_types.go b/go/parquet/schema/logical_types.go index 1ea44fc56c615..a0607a1a2f62d 100644 --- a/go/parquet/schema/logical_types.go +++ b/go/parquet/schema/logical_types.go @@ -20,10 +20,10 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/internal/debug" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" ) // DecimalMetadata is a struct for managing scale and precision information between diff --git a/go/parquet/schema/logical_types_test.go b/go/parquet/schema/logical_types_test.go index 49edf1748c52c..8531a061abb77 100644 --- a/go/parquet/schema/logical_types_test.go +++ b/go/parquet/schema/logical_types_test.go @@ -19,9 +19,9 @@ package schema_test import ( "testing" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/node.go b/go/parquet/schema/node.go index ff23624afa35d..28259c8d61f58 100644 --- a/go/parquet/schema/node.go +++ b/go/parquet/schema/node.go @@ -19,8 +19,8 @@ package schema import ( "fmt" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" "github.com/apache/thrift/lib/go/thrift" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/reflection.go b/go/parquet/schema/reflection.go index d79edb92408f8..bc863f1390030 100644 --- a/go/parquet/schema/reflection.go +++ b/go/parquet/schema/reflection.go @@ -22,8 +22,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/reflection_test.go b/go/parquet/schema/reflection_test.go index 06ad7191a5749..e18feada7a796 100644 --- a/go/parquet/schema/reflection_test.go +++ b/go/parquet/schema/reflection_test.go @@ -22,8 +22,8 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/schema.go b/go/parquet/schema/schema.go index ace2775763a06..1b3363f58d11f 100644 --- a/go/parquet/schema/schema.go +++ b/go/parquet/schema/schema.go @@ -35,8 +35,8 @@ import ( "io" "strings" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/schema_element_test.go b/go/parquet/schema/schema_element_test.go index dd1d293e5cfd7..e9a2ef5eb31f6 100644 --- a/go/parquet/schema/schema_element_test.go +++ b/go/parquet/schema/schema_element_test.go @@ -19,8 +19,8 @@ package schema import ( "testing" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/schema/schema_flatten_test.go b/go/parquet/schema/schema_flatten_test.go index 34f4ac8d3c450..90741ae9b43bb 100644 --- a/go/parquet/schema/schema_flatten_test.go +++ b/go/parquet/schema/schema_flatten_test.go @@ -19,8 +19,8 @@ package schema import ( "testing" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/schema/schema_test.go b/go/parquet/schema/schema_test.go index b60c7dfaafcd1..e69a13e07986e 100644 --- a/go/parquet/schema/schema_test.go +++ b/go/parquet/schema/schema_test.go @@ -20,9 +20,9 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v14/parquet" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/parquet/schema" "github.com/apache/thrift/lib/go/thrift" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" diff --git a/go/parquet/types.go b/go/parquet/types.go index 5b6f8fb0251d9..7b6df2ea9d16c 100644 --- a/go/parquet/types.go +++ b/go/parquet/types.go @@ -24,8 +24,8 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v14/arrow" + format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" ) const ( diff --git a/go/parquet/writer_properties.go b/go/parquet/writer_properties.go index 9e33bddf7faa7..acd43194c1be7 100644 --- a/go/parquet/writer_properties.go +++ b/go/parquet/writer_properties.go @@ -17,8 +17,8 @@ package parquet import ( - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v14/parquet/compress" ) // Constants for default property values used for the default reader, writer and column props. @@ -46,7 +46,7 @@ const ( DefaultStatsEnabled = true // If the stats are larger than 4K the writer will skip writing them out anyways. DefaultMaxStatsSize int64 = 4096 - DefaultCreatedBy = "parquet-go version 15.0.0-SNAPSHOT" + DefaultCreatedBy = "parquet-go version 14.0.1-SNAPSHOT" DefaultRootName = "schema" ) diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml index c0410ea4c2314..20602f3b80771 100644 --- a/java/adapter/avro/pom.xml +++ b/java/adapter/avro/pom.xml @@ -16,7 +16,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT ../../pom.xml diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 2490f708e6f24..f97bcaf74a27c 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -16,7 +16,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT ../../pom.xml diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index 72ba13ea81738..2213a2a7d0783 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -111,7 +111,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT ../../pom.xml diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index 8c6a9fb0151ef..a248389aeb406 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT arrow-algorithm Arrow Algorithms diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 1f6f854f60013..01f5e93dc0438 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-bom - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT pom Arrow Bill of Materials Arrow Bill of Materials diff --git a/java/c/pom.xml b/java/c/pom.xml index d66379d356624..7a4631704fa80 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -13,7 +13,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT 4.0.0 diff --git a/java/compression/pom.xml b/java/compression/pom.xml index 129268b9e6a07..283c2af2d7d76 100644 --- a/java/compression/pom.xml +++ b/java/compression/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT arrow-compression Arrow Compression diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index b533a1733521b..cdc1a411798c3 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT 4.0.0 diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 7b69179053d7f..172a8583d8c88 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -14,7 +14,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT ../pom.xml diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml index e7bb9508d2403..f113ba50759a9 100644 --- a/java/flight/flight-grpc/pom.xml +++ b/java/flight/flight-grpc/pom.xml @@ -13,7 +13,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/java/flight/flight-integration-tests/pom.xml b/java/flight/flight-integration-tests/pom.xml index 778cb7df11f61..6b7796641b607 100644 --- a/java/flight/flight-integration-tests/pom.xml +++ b/java/flight/flight-integration-tests/pom.xml @@ -15,7 +15,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT ../pom.xml diff --git a/java/flight/flight-sql-jdbc-core/pom.xml b/java/flight/flight-sql-jdbc-core/pom.xml index cbeaa88f1e2f7..34efb04fa8702 100644 --- a/java/flight/flight-sql-jdbc-core/pom.xml +++ b/java/flight/flight-sql-jdbc-core/pom.xml @@ -16,7 +16,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml b/java/flight/flight-sql-jdbc-driver/pom.xml index 263538ba48b37..69e3bfad8d883 100644 --- a/java/flight/flight-sql-jdbc-driver/pom.xml +++ b/java/flight/flight-sql-jdbc-driver/pom.xml @@ -16,7 +16,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index 09100d9731ad5..e49cb8fa4dd5c 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -14,7 +14,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT ../pom.xml diff --git a/java/flight/pom.xml b/java/flight/pom.xml index 7ddda94f77b49..fd65c054d971e 100644 --- a/java/flight/pom.xml +++ b/java/flight/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT 4.0.0 diff --git a/java/format/pom.xml b/java/format/pom.xml index 3f581311e20ea..58dd21e3288f2 100644 --- a/java/format/pom.xml +++ b/java/format/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT arrow-format diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index cfda0b0c527a9..e87ba2bb4eb88 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT org.apache.arrow.gandiva diff --git a/java/memory/memory-core/pom.xml b/java/memory/memory-core/pom.xml index 8f28699045383..5d1886032b3e8 100644 --- a/java/memory/memory-core/pom.xml +++ b/java/memory/memory-core/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT 4.0.0 diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml index 307f6ad81a0f1..56db6924bf746 100644 --- a/java/memory/memory-netty/pom.xml +++ b/java/memory/memory-netty/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT 4.0.0 diff --git a/java/memory/memory-unsafe/pom.xml b/java/memory/memory-unsafe/pom.xml index 9f8137308195d..68bd61e742741 100644 --- a/java/memory/memory-unsafe/pom.xml +++ b/java/memory/memory-unsafe/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT 4.0.0 diff --git a/java/memory/pom.xml b/java/memory/pom.xml index c10263b97f1bf..6fc9d11730eaf 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT arrow-memory Arrow Memory diff --git a/java/performance/pom.xml b/java/performance/pom.xml index 102832491ec05..6aa1599425ad8 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -14,7 +14,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT arrow-performance jar diff --git a/java/pom.xml b/java/pom.xml index 2a9997b7012b7..caf470ee2b2b2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT pom Apache Arrow Java Root POM diff --git a/java/tools/pom.xml b/java/tools/pom.xml index 128825c224369..ce6243e98dacf 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT arrow-tools Arrow Tools diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 4c8bf1e594aa4..0cfa421fbf7d4 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 14.0.1-SNAPSHOT arrow-vector Arrow Vectors diff --git a/js/package.json b/js/package.json index 33bc4849903ef..3bad73b6fdf76 100644 --- a/js/package.json +++ b/js/package.json @@ -124,5 +124,5 @@ "engines": { "node": ">=12.0" }, - "version": "15.0.0-SNAPSHOT" + "version": "14.0.1-SNAPSHOT" } diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index 47d2acd613f8b..df20ad9680707 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -94,7 +94,7 @@ endfunction() set(CMAKE_CXX_STANDARD 17) -set(MLARROW_VERSION "15.0.0-SNAPSHOT") +set(MLARROW_VERSION "14.0.1-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MLARROW_BASE_VERSION "${MLARROW_VERSION}") project(mlarrow VERSION "${MLARROW_BASE_VERSION}") diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 86285c3dc381b..95d23933a88b7 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -21,7 +21,7 @@ cmake_minimum_required(VERSION 3.16) project(pyarrow) -set(PYARROW_VERSION "15.0.0-SNAPSHOT") +set(PYARROW_VERSION "14.0.1-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" PYARROW_BASE_VERSION "${PYARROW_VERSION}") # Running from a Python sdist tarball diff --git a/python/setup.py b/python/setup.py index b1c825d84d5a9..4e45b8c68fe16 100755 --- a/python/setup.py +++ b/python/setup.py @@ -407,7 +407,7 @@ def get_outputs(self): # If the event of not running from a git clone (e.g. from a git archive # or a Python sdist), see if we can set the version number ourselves -default_version = '15.0.0-SNAPSHOT' +default_version = '14.0.1-SNAPSHOT' if (not os.path.exists('../.git') and not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')): os.environ['SETUPTOOLS_SCM_PRETEND_VERSION'] = \ diff --git a/r/NEWS.md b/r/NEWS.md index e00c6b51b597d..d2d5f08288cb8 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -21,6 +21,8 @@ # arrow 14.0.0 +# arrow 14.0.0 + ## New features * When reading partitioned CSV datasets and supplying a schema to diff --git a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb index 8551b647cb86f..86d8ffba80c90 100644 --- a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb +++ b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowCUDA - VERSION = "15.0.0-SNAPSHOT" + VERSION = "14.0.1-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb index acfdd675687be..b0a29b7f7b365 100644 --- a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb +++ b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowDataset - VERSION = "15.0.0-SNAPSHOT" + VERSION = "14.0.1-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb index 3354678e30032..6734ab6ae2536 100644 --- a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb +++ b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlightSQL - VERSION = "15.0.0-SNAPSHOT" + VERSION = "14.0.1-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight/lib/arrow-flight/version.rb b/ruby/red-arrow-flight/lib/arrow-flight/version.rb index f2141a68432e5..b3b778108199e 100644 --- a/ruby/red-arrow-flight/lib/arrow-flight/version.rb +++ b/ruby/red-arrow-flight/lib/arrow-flight/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlight - VERSION = "15.0.0-SNAPSHOT" + VERSION = "14.0.1-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow/lib/arrow/version.rb b/ruby/red-arrow/lib/arrow/version.rb index 235a7df75d672..222b72d10bb34 100644 --- a/ruby/red-arrow/lib/arrow/version.rb +++ b/ruby/red-arrow/lib/arrow/version.rb @@ -16,7 +16,7 @@ # under the License. module Arrow - VERSION = "15.0.0-SNAPSHOT" + VERSION = "14.0.1-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-gandiva/lib/gandiva/version.rb b/ruby/red-gandiva/lib/gandiva/version.rb index 6a1835f0e50e8..576ee097743db 100644 --- a/ruby/red-gandiva/lib/gandiva/version.rb +++ b/ruby/red-gandiva/lib/gandiva/version.rb @@ -16,7 +16,7 @@ # under the License. module Gandiva - VERSION = "15.0.0-SNAPSHOT" + VERSION = "14.0.1-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-parquet/lib/parquet/version.rb b/ruby/red-parquet/lib/parquet/version.rb index c5a945c4e4297..17a9157a36047 100644 --- a/ruby/red-parquet/lib/parquet/version.rb +++ b/ruby/red-parquet/lib/parquet/version.rb @@ -16,7 +16,7 @@ # under the License. module Parquet - VERSION = "15.0.0-SNAPSHOT" + VERSION = "14.0.1-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") From 452c420244ea45ee1376fc8f5707d0fc52c903c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 10 Nov 2023 16:21:57 +0100 Subject: [PATCH 33/57] MINOR: [Release] Update .deb/.rpm changelogs for 14.0.0 --- .../linux-packages/apache-arrow-apt-source/debian/changelog | 6 ++++++ .../apache-arrow-release/yum/apache-arrow-release.spec.in | 3 +++ dev/tasks/linux-packages/apache-arrow/debian/changelog | 6 ++++++ dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 3 +++ 4 files changed, 18 insertions(+) diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog index 221fb0caa8952..8e3cfb9d1c38f 100644 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog @@ -4,6 +4,12 @@ apache-arrow-apt-source (14.0.0-1) unstable; urgency=low -- Raúl Cumplido Thu, 19 Oct 2023 09:12:19 -0000 +apache-arrow-apt-source (14.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Raúl Cumplido Thu, 19 Oct 2023 09:12:19 -0000 + apache-arrow-apt-source (13.0.0-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in index 273bf32a2a8e4..680e7f75fe4b8 100644 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in +++ b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in @@ -105,6 +105,9 @@ fi * Thu Oct 19 2023 Raúl Cumplido - 14.0.0-1 - New upstream release. +* Thu Oct 19 2023 Raúl Cumplido - 14.0.0-1 +- New upstream release. + * Thu Aug 17 2023 Raúl Cumplido - 13.0.0-1 - New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/debian/changelog b/dev/tasks/linux-packages/apache-arrow/debian/changelog index 5e01d962c44d4..90a2ce8f21a91 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow/debian/changelog @@ -4,6 +4,12 @@ apache-arrow (14.0.0-1) unstable; urgency=low -- Raúl Cumplido Thu, 19 Oct 2023 09:12:19 -0000 +apache-arrow (14.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Raúl Cumplido Thu, 19 Oct 2023 09:12:19 -0000 + apache-arrow (13.0.0-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index f61d47db2edd7..95ea679d205ee 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -867,6 +867,9 @@ Documentation for Apache Parquet GLib. * Thu Oct 19 2023 Raúl Cumplido - 14.0.0-1 - New upstream release. +* Thu Oct 19 2023 Raúl Cumplido - 14.0.0-1 +- New upstream release. + * Thu Aug 17 2023 Raúl Cumplido - 13.0.0-1 - New upstream release. From 0b796f8e995e92265325cc453d702f12b46d35ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 10 Nov 2023 16:26:15 +0100 Subject: [PATCH 34/57] Revert "MINOR: [Release] Update .deb/.rpm changelogs for 14.0.0" This reverts commit 452c420244ea45ee1376fc8f5707d0fc52c903c5. --- .../linux-packages/apache-arrow-apt-source/debian/changelog | 6 ------ .../apache-arrow-release/yum/apache-arrow-release.spec.in | 3 --- dev/tasks/linux-packages/apache-arrow/debian/changelog | 6 ------ dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 3 --- 4 files changed, 18 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog index 8e3cfb9d1c38f..221fb0caa8952 100644 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog @@ -4,12 +4,6 @@ apache-arrow-apt-source (14.0.0-1) unstable; urgency=low -- Raúl Cumplido Thu, 19 Oct 2023 09:12:19 -0000 -apache-arrow-apt-source (14.0.0-1) unstable; urgency=low - - * New upstream release. - - -- Raúl Cumplido Thu, 19 Oct 2023 09:12:19 -0000 - apache-arrow-apt-source (13.0.0-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in index 680e7f75fe4b8..273bf32a2a8e4 100644 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in +++ b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in @@ -105,9 +105,6 @@ fi * Thu Oct 19 2023 Raúl Cumplido - 14.0.0-1 - New upstream release. -* Thu Oct 19 2023 Raúl Cumplido - 14.0.0-1 -- New upstream release. - * Thu Aug 17 2023 Raúl Cumplido - 13.0.0-1 - New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/debian/changelog b/dev/tasks/linux-packages/apache-arrow/debian/changelog index 90a2ce8f21a91..5e01d962c44d4 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow/debian/changelog @@ -4,12 +4,6 @@ apache-arrow (14.0.0-1) unstable; urgency=low -- Raúl Cumplido Thu, 19 Oct 2023 09:12:19 -0000 -apache-arrow (14.0.0-1) unstable; urgency=low - - * New upstream release. - - -- Raúl Cumplido Thu, 19 Oct 2023 09:12:19 -0000 - apache-arrow (13.0.0-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 95ea679d205ee..f61d47db2edd7 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -867,9 +867,6 @@ Documentation for Apache Parquet GLib. * Thu Oct 19 2023 Raúl Cumplido - 14.0.0-1 - New upstream release. -* Thu Oct 19 2023 Raúl Cumplido - 14.0.0-1 -- New upstream release. - * Thu Aug 17 2023 Raúl Cumplido - 13.0.0-1 - New upstream release. From 3c99fbb470c926e948b3b80d8ae9bac5b5c6eddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 10 Nov 2023 16:26:22 +0100 Subject: [PATCH 35/57] Revert "MINOR: [Release] Update versions for 14.0.1-SNAPSHOT" This reverts commit e5b2bba2229836f7b1b37a06d62081cb214d598a. --- c_glib/meson.build | 2 +- cpp/CMakeLists.txt | 2 +- cpp/vcpkg.json | 2 +- csharp/Directory.Build.props | 2 +- .../homebrew-formulae/apache-arrow-glib.rb | 2 +- dev/tasks/homebrew-formulae/apache-arrow.rb | 2 +- go/README.md | 2 +- go/arrow/_examples/helloworld/main.go | 8 ++--- go/arrow/_tools/tmpl/main.go | 2 +- go/arrow/array.go | 4 +-- go/arrow/array/array.go | 6 ++-- go/arrow/array/array_test.go | 10 +++--- go/arrow/array/binary.go | 4 +-- go/arrow/array/binary_test.go | 6 ++-- go/arrow/array/binarybuilder.go | 8 ++--- go/arrow/array/binarybuilder_test.go | 6 ++-- go/arrow/array/boolean.go | 8 ++--- go/arrow/array/boolean_test.go | 4 +-- go/arrow/array/booleanbuilder.go | 10 +++--- go/arrow/array/booleanbuilder_test.go | 6 ++-- go/arrow/array/bufferbuilder.go | 6 ++-- go/arrow/array/bufferbuilder_byte.go | 2 +- go/arrow/array/bufferbuilder_numeric.gen.go | 6 ++-- .../array/bufferbuilder_numeric.gen.go.tmpl | 6 ++-- go/arrow/array/bufferbuilder_numeric_test.go | 4 +-- go/arrow/array/builder.go | 8 ++--- go/arrow/array/builder_test.go | 4 +-- go/arrow/array/compare.go | 6 ++-- go/arrow/array/compare_test.go | 10 +++--- go/arrow/array/concat.go | 14 ++++---- go/arrow/array/concat_test.go | 10 +++--- go/arrow/array/data.go | 6 ++-- go/arrow/array/data_test.go | 4 +-- go/arrow/array/decimal128.go | 12 +++---- go/arrow/array/decimal128_test.go | 8 ++--- go/arrow/array/decimal256.go | 12 +++---- go/arrow/array/decimal256_test.go | 8 ++--- go/arrow/array/decimal_test.go | 12 +++---- go/arrow/array/dictionary.go | 20 ++++++------ go/arrow/array/dictionary_test.go | 14 ++++---- go/arrow/array/diff.go | 2 +- go/arrow/array/diff_test.go | 10 +++--- go/arrow/array/encoded.go | 12 +++---- go/arrow/array/encoded_test.go | 8 ++--- go/arrow/array/extension.go | 6 ++-- go/arrow/array/extension_test.go | 8 ++--- go/arrow/array/fixed_size_list.go | 10 +++--- go/arrow/array/fixed_size_list_test.go | 6 ++-- go/arrow/array/fixedsize_binary.go | 4 +-- go/arrow/array/fixedsize_binary_test.go | 6 ++-- go/arrow/array/fixedsize_binarybuilder.go | 8 ++--- .../array/fixedsize_binarybuilder_test.go | 4 +-- go/arrow/array/float16.go | 6 ++-- go/arrow/array/float16_builder.go | 12 +++---- go/arrow/array/float16_builder_test.go | 6 ++-- go/arrow/array/interval.go | 10 +++--- go/arrow/array/interval_test.go | 6 ++-- go/arrow/array/json_reader.go | 8 ++--- go/arrow/array/json_reader_test.go | 6 ++-- go/arrow/array/list.go | 10 +++--- go/arrow/array/list_test.go | 6 ++-- go/arrow/array/map.go | 6 ++-- go/arrow/array/map_test.go | 6 ++-- go/arrow/array/null.go | 8 ++--- go/arrow/array/null_test.go | 6 ++-- go/arrow/array/numeric.gen.go | 4 +-- go/arrow/array/numeric.gen.go.tmpl | 4 +-- go/arrow/array/numeric_test.go | 8 ++--- go/arrow/array/numericbuilder.gen.go | 10 +++--- go/arrow/array/numericbuilder.gen.go.tmpl | 10 +++--- go/arrow/array/numericbuilder.gen_test.go | 6 ++-- .../array/numericbuilder.gen_test.go.tmpl | 6 ++-- go/arrow/array/record.go | 8 ++--- go/arrow/array/record_test.go | 6 ++-- go/arrow/array/string.go | 6 ++-- go/arrow/array/string_test.go | 8 ++--- go/arrow/array/struct.go | 10 +++--- go/arrow/array/struct_test.go | 6 ++-- go/arrow/array/table.go | 4 +-- go/arrow/array/table_test.go | 6 ++-- go/arrow/array/timestamp.go | 10 +++--- go/arrow/array/timestamp_test.go | 6 ++-- go/arrow/array/union.go | 12 +++---- go/arrow/array/union_test.go | 6 ++-- go/arrow/array/util.go | 10 +++--- go/arrow/array/util_test.go | 14 ++++---- go/arrow/arrio/arrio.go | 2 +- go/arrow/arrio/arrio_test.go | 10 +++--- go/arrow/bitutil/bitmaps.go | 6 ++-- go/arrow/bitutil/bitmaps_test.go | 4 +-- go/arrow/bitutil/bitutil.go | 2 +- go/arrow/bitutil/bitutil_test.go | 4 +-- go/arrow/cdata/cdata.go | 8 ++--- go/arrow/cdata/cdata_exports.go | 10 +++--- go/arrow/cdata/cdata_test.go | 12 +++---- go/arrow/cdata/cdata_test_framework.go | 8 ++--- go/arrow/cdata/exports.go | 4 +-- go/arrow/cdata/import_allocator.go | 2 +- go/arrow/cdata/interface.go | 8 ++--- go/arrow/cdata/test/test_cimport.go | 8 ++--- go/arrow/compute/arithmetic.go | 12 +++---- go/arrow/compute/arithmetic_test.go | 20 ++++++------ go/arrow/compute/cast.go | 10 +++--- go/arrow/compute/cast_test.go | 20 ++++++------ go/arrow/compute/datum.go | 6 ++-- go/arrow/compute/example_test.go | 10 +++--- go/arrow/compute/exec.go | 6 ++-- go/arrow/compute/exec/kernel.go | 8 ++--- go/arrow/compute/exec/kernel_test.go | 12 +++---- go/arrow/compute/exec/span.go | 10 +++--- go/arrow/compute/exec/span_test.go | 16 +++++----- go/arrow/compute/exec/utils.go | 14 ++++---- go/arrow/compute/exec/utils_test.go | 8 ++--- go/arrow/compute/exec_internals_test.go | 14 ++++---- go/arrow/compute/exec_test.go | 12 +++---- go/arrow/compute/executor.go | 16 +++++----- go/arrow/compute/expression.go | 16 +++++----- go/arrow/compute/expression_test.go | 10 +++--- go/arrow/compute/exprs/builders.go | 4 +-- go/arrow/compute/exprs/builders_test.go | 4 +-- go/arrow/compute/exprs/exec.go | 18 +++++------ go/arrow/compute/exprs/exec_internal_test.go | 8 ++--- go/arrow/compute/exprs/exec_test.go | 12 +++---- go/arrow/compute/exprs/extension_types.go | 4 +-- go/arrow/compute/exprs/field_refs.go | 10 +++--- go/arrow/compute/exprs/types.go | 4 +-- go/arrow/compute/fieldref.go | 4 +-- go/arrow/compute/fieldref_test.go | 8 ++--- go/arrow/compute/functions.go | 4 +-- go/arrow/compute/functions_test.go | 4 +-- .../internal/kernels/base_arithmetic.go | 10 +++--- .../internal/kernels/base_arithmetic_amd64.go | 4 +-- .../kernels/base_arithmetic_avx2_amd64.go | 2 +- .../kernels/base_arithmetic_sse4_amd64.go | 2 +- .../kernels/basic_arithmetic_noasm.go | 2 +- .../compute/internal/kernels/boolean_cast.go | 6 ++-- go/arrow/compute/internal/kernels/cast.go | 6 ++-- .../compute/internal/kernels/cast_numeric.go | 2 +- .../kernels/cast_numeric_avx2_amd64.go | 2 +- .../kernels/cast_numeric_neon_arm64.go | 2 +- .../kernels/cast_numeric_sse4_amd64.go | 2 +- .../compute/internal/kernels/cast_temporal.go | 8 ++--- go/arrow/compute/internal/kernels/helpers.go | 14 ++++---- .../compute/internal/kernels/numeric_cast.go | 14 ++++---- go/arrow/compute/internal/kernels/rounding.go | 10 +++--- .../internal/kernels/scalar_arithmetic.go | 14 ++++---- .../internal/kernels/scalar_boolean.go | 6 ++-- .../kernels/scalar_comparison_amd64.go | 4 +-- .../kernels/scalar_comparison_avx2_amd64.go | 2 +- .../kernels/scalar_comparison_noasm.go | 2 +- .../kernels/scalar_comparison_sse4_amd64.go | 2 +- .../internal/kernels/scalar_comparisons.go | 16 +++++----- .../compute/internal/kernels/string_casts.go | 12 +++---- go/arrow/compute/internal/kernels/types.go | 8 ++--- .../compute/internal/kernels/vector_hash.go | 14 ++++---- .../internal/kernels/vector_run_end_encode.go | 16 +++++----- .../internal/kernels/vector_selection.go | 14 ++++---- go/arrow/compute/registry.go | 2 +- go/arrow/compute/registry_test.go | 6 ++-- go/arrow/compute/scalar_bool.go | 6 ++-- go/arrow/compute/scalar_bool_test.go | 10 +++--- go/arrow/compute/scalar_compare.go | 6 ++-- go/arrow/compute/scalar_compare_test.go | 18 +++++------ go/arrow/compute/selection.go | 8 ++--- go/arrow/compute/utils.go | 12 +++---- go/arrow/compute/vector_hash.go | 4 +-- go/arrow/compute/vector_hash_test.go | 14 ++++---- go/arrow/compute/vector_run_end_test.go | 14 ++++---- go/arrow/compute/vector_run_ends.go | 4 +-- go/arrow/compute/vector_selection_test.go | 18 +++++------ go/arrow/csv/common.go | 4 +-- go/arrow/csv/reader.go | 14 ++++---- go/arrow/csv/reader_test.go | 14 ++++---- go/arrow/csv/transformer.go | 4 +-- go/arrow/csv/writer.go | 2 +- go/arrow/csv/writer_test.go | 16 +++++----- go/arrow/datatype.go | 2 +- go/arrow/datatype_binary_test.go | 2 +- go/arrow/datatype_extension_test.go | 4 +-- go/arrow/datatype_fixedwidth.go | 2 +- go/arrow/datatype_fixedwidth_test.go | 2 +- go/arrow/datatype_nested.go | 2 +- go/arrow/datatype_null_test.go | 2 +- go/arrow/decimal128/decimal128.go | 2 +- go/arrow/decimal128/decimal128_test.go | 2 +- go/arrow/decimal256/decimal256.go | 4 +-- go/arrow/decimal256/decimal256_test.go | 2 +- go/arrow/doc.go | 2 +- go/arrow/encoded/ree_utils.go | 2 +- go/arrow/encoded/ree_utils_test.go | 8 ++--- go/arrow/endian/endian.go | 4 +-- go/arrow/example_test.go | 8 ++--- go/arrow/flight/basic_auth_flight_test.go | 2 +- go/arrow/flight/client.go | 2 +- go/arrow/flight/cookie_middleware_test.go | 2 +- go/arrow/flight/example_flight_server_test.go | 2 +- go/arrow/flight/flight_middleware_test.go | 4 +-- go/arrow/flight/flight_test.go | 10 +++--- go/arrow/flight/flightsql/client.go | 12 +++---- go/arrow/flight/flightsql/client_test.go | 12 +++---- go/arrow/flight/flightsql/column_metadata.go | 2 +- go/arrow/flight/flightsql/driver/README.md | 6 ++-- .../flight/flightsql/driver/config_test.go | 2 +- go/arrow/flight/flightsql/driver/driver.go | 10 +++--- .../flight/flightsql/driver/driver_test.go | 14 ++++---- go/arrow/flight/flightsql/driver/utils.go | 4 +-- .../flight/flightsql/driver/utils_test.go | 12 +++---- .../cmd/sqlite_flightsql_server/main.go | 6 ++-- .../flightsql/example/sql_batch_reader.go | 10 +++--- .../flight/flightsql/example/sqlite_info.go | 4 +-- .../flight/flightsql/example/sqlite_server.go | 14 ++++---- .../sqlite_tables_schema_batch_reader.go | 12 +++---- .../flight/flightsql/example/type_info.go | 8 ++--- .../flightsql/schema_ref/reference_schemas.go | 2 +- go/arrow/flight/flightsql/server.go | 16 +++++----- go/arrow/flight/flightsql/server_test.go | 12 +++---- go/arrow/flight/flightsql/sql_info.go | 4 +-- .../flight/flightsql/sqlite_server_test.go | 16 +++++----- go/arrow/flight/flightsql/types.go | 2 +- go/arrow/flight/record_batch_reader.go | 12 +++---- go/arrow/flight/record_batch_writer.go | 6 ++-- go/arrow/flight/server.go | 2 +- go/arrow/flight/server_example_test.go | 2 +- go/arrow/internal/arrdata/arrdata.go | 16 +++++----- go/arrow/internal/arrdata/ioutil.go | 10 +++--- go/arrow/internal/arrjson/arrjson.go | 20 ++++++------ go/arrow/internal/arrjson/arrjson_test.go | 6 ++-- go/arrow/internal/arrjson/option.go | 4 +-- go/arrow/internal/arrjson/reader.go | 10 +++--- go/arrow/internal/arrjson/writer.go | 10 +++--- .../internal/cdata_integration/entrypoints.go | 8 ++--- go/arrow/internal/dictutils/dict.go | 6 ++-- go/arrow/internal/dictutils/dict_test.go | 8 ++--- .../arrow-flight-integration-client/main.go | 2 +- .../arrow-flight-integration-server/main.go | 2 +- .../internal/flight_integration/scenario.go | 18 +++++------ .../internal/testing/gen/random_array_gen.go | 10 +++--- go/arrow/internal/testing/tools/bits_test.go | 2 +- go/arrow/internal/testing/tools/data_types.go | 4 +-- go/arrow/internal/utils.go | 4 +-- go/arrow/ipc/cmd/arrow-cat/main.go | 4 +-- go/arrow/ipc/cmd/arrow-cat/main_test.go | 8 ++--- go/arrow/ipc/cmd/arrow-file-to-stream/main.go | 6 ++-- .../ipc/cmd/arrow-file-to-stream/main_test.go | 4 +-- .../cmd/arrow-json-integration-test/main.go | 12 +++---- .../arrow-json-integration-test/main_test.go | 4 +-- go/arrow/ipc/cmd/arrow-ls/main.go | 4 +-- go/arrow/ipc/cmd/arrow-ls/main_test.go | 8 ++--- go/arrow/ipc/cmd/arrow-stream-to-file/main.go | 6 ++-- .../ipc/cmd/arrow-stream-to-file/main_test.go | 4 +-- go/arrow/ipc/compression.go | 6 ++-- go/arrow/ipc/endian_swap.go | 6 ++-- go/arrow/ipc/endian_swap_test.go | 10 +++--- go/arrow/ipc/file_reader.go | 16 +++++----- go/arrow/ipc/file_test.go | 6 ++-- go/arrow/ipc/file_writer.go | 10 +++--- go/arrow/ipc/ipc.go | 8 ++--- go/arrow/ipc/ipc_test.go | 8 ++--- go/arrow/ipc/message.go | 6 ++-- go/arrow/ipc/message_test.go | 6 ++-- go/arrow/ipc/metadata.go | 10 +++--- go/arrow/ipc/metadata_test.go | 12 +++---- go/arrow/ipc/reader.go | 14 ++++---- go/arrow/ipc/reader_test.go | 6 ++-- go/arrow/ipc/stream_test.go | 6 ++-- go/arrow/ipc/writer.go | 16 +++++----- go/arrow/ipc/writer_test.go | 10 +++--- go/arrow/math/float64.go | 2 +- go/arrow/math/float64_avx2_amd64.go | 2 +- go/arrow/math/float64_neon_arm64.go | 2 +- go/arrow/math/float64_sse4_amd64.go | 2 +- go/arrow/math/float64_test.go | 6 ++-- go/arrow/math/int64.go | 2 +- go/arrow/math/int64_avx2_amd64.go | 2 +- go/arrow/math/int64_neon_arm64.go | 2 +- go/arrow/math/int64_sse4_amd64.go | 2 +- go/arrow/math/int64_test.go | 6 ++-- go/arrow/math/type.go.tmpl | 2 +- go/arrow/math/type_simd_amd64.go.tmpl | 2 +- go/arrow/math/type_simd_arm64.go.tmpl | 2 +- go/arrow/math/type_test.go.tmpl | 6 ++-- go/arrow/math/uint64.go | 2 +- go/arrow/math/uint64_avx2_amd64.go | 2 +- go/arrow/math/uint64_neon_arm64.go | 2 +- go/arrow/math/uint64_sse4_amd64.go | 2 +- go/arrow/math/uint64_test.go | 6 ++-- go/arrow/memory/buffer.go | 2 +- go/arrow/memory/buffer_test.go | 2 +- go/arrow/memory/cgo_allocator.go | 2 +- go/arrow/memory/default_mallocator.go | 2 +- go/arrow/memory/default_mallocator_test.go | 4 +-- go/arrow/memory/mallocator/mallocator_test.go | 2 +- go/arrow/memory/memory_test.go | 2 +- go/arrow/record.go | 2 +- go/arrow/scalar/append.go | 10 +++--- go/arrow/scalar/append_test.go | 10 +++--- go/arrow/scalar/binary.go | 4 +-- go/arrow/scalar/compare.go | 2 +- go/arrow/scalar/nested.go | 8 ++--- go/arrow/scalar/numeric.gen.go | 6 ++-- go/arrow/scalar/numeric.gen_test.go | 4 +-- go/arrow/scalar/numeric.gen_test.go.tmpl | 4 +-- go/arrow/scalar/parse.go | 12 +++---- go/arrow/scalar/scalar.go | 20 ++++++------ go/arrow/scalar/scalar_test.go | 12 +++---- go/arrow/scalar/temporal.go | 2 +- go/arrow/schema.go | 2 +- go/arrow/schema_test.go | 2 +- go/arrow/table.go | 2 +- go/arrow/tensor/numeric.gen.go | 2 +- go/arrow/tensor/numeric.gen.go.tmpl | 4 +-- go/arrow/tensor/numeric.gen_test.go | 8 ++--- go/arrow/tensor/numeric.gen_test.go.tmpl | 8 ++--- go/arrow/tensor/tensor.go | 4 +-- go/arrow/tensor/tensor_test.go | 8 ++--- go/arrow/type_traits_boolean.go | 2 +- go/arrow/type_traits_decimal128.go | 4 +-- go/arrow/type_traits_decimal256.go | 4 +-- go/arrow/type_traits_float16.go | 4 +-- go/arrow/type_traits_interval.go | 4 +-- go/arrow/type_traits_numeric.gen.go | 2 +- go/arrow/type_traits_numeric.gen.go.tmpl | 2 +- go/arrow/type_traits_numeric.gen_test.go | 2 +- go/arrow/type_traits_numeric.gen_test.go.tmpl | 2 +- go/arrow/type_traits_test.go | 8 ++--- go/arrow/type_traits_timestamp.go | 2 +- go/arrow/util/byte_size.go | 6 ++-- go/arrow/util/byte_size_test.go | 8 ++--- go/go.mod | 2 +- go/internal/bitutils/bit_block_counter.go | 4 +-- .../bitutils/bit_block_counter_test.go | 6 ++-- go/internal/bitutils/bit_run_reader.go | 6 ++-- go/internal/bitutils/bit_run_reader_test.go | 6 ++-- go/internal/bitutils/bit_set_run_reader.go | 4 +-- .../bitutils/bit_set_run_reader_test.go | 6 ++-- go/internal/bitutils/bitmap_generate.go | 2 +- go/internal/bitutils/bitmap_generate_test.go | 2 +- go/internal/hashing/xxh3_memo_table.gen.go | 6 ++-- .../hashing/xxh3_memo_table.gen.go.tmpl | 4 +-- go/internal/types/extension_types.go | 6 ++-- go/internal/types/extension_types_test.go | 10 +++--- go/internal/utils/transpose_ints_def.go | 2 +- go/internal/utils/transpose_ints_test.go | 2 +- go/parquet/cmd/parquet_reader/dumper.go | 6 ++-- go/parquet/cmd/parquet_reader/main.go | 10 +++--- go/parquet/cmd/parquet_schema/main.go | 4 +-- go/parquet/compress/brotli.go | 2 +- go/parquet/compress/compress.go | 2 +- go/parquet/compress/compress_test.go | 2 +- go/parquet/compress/zstd.go | 2 +- go/parquet/doc.go | 6 ++-- go/parquet/encryption_properties.go | 2 +- go/parquet/encryption_properties_test.go | 4 +-- go/parquet/encryption_read_config_test.go | 8 ++--- go/parquet/encryption_write_config_test.go | 8 ++--- go/parquet/file/column_reader.go | 14 ++++---- go/parquet/file/column_reader_test.go | 12 +++---- go/parquet/file/column_reader_types.gen.go | 6 ++-- .../file/column_reader_types.gen.go.tmpl | 4 +-- go/parquet/file/column_writer.go | 16 +++++----- go/parquet/file/column_writer_test.go | 26 +++++++-------- go/parquet/file/column_writer_types.gen.go | 12 +++---- .../file/column_writer_types.gen.go.tmpl | 8 ++--- go/parquet/file/file_reader.go | 8 ++--- go/parquet/file/file_reader_mmap.go | 2 +- go/parquet/file/file_reader_mmap_windows.go | 2 +- go/parquet/file/file_reader_test.go | 20 ++++++------ go/parquet/file/file_writer.go | 10 +++--- go/parquet/file/file_writer_test.go | 14 ++++---- go/parquet/file/level_conversion.go | 10 +++--- go/parquet/file/level_conversion_test.go | 6 ++-- go/parquet/file/page_reader.go | 14 ++++---- go/parquet/file/page_writer.go | 18 +++++------ go/parquet/file/record_reader.go | 16 +++++----- go/parquet/file/row_group_reader.go | 8 ++--- go/parquet/file/row_group_writer.go | 8 ++--- go/parquet/file/row_group_writer_test.go | 8 ++--- go/parquet/internal/bmi/bmi_test.go | 2 +- .../internal/encoding/boolean_decoder.go | 8 ++--- .../internal/encoding/boolean_encoder.go | 8 ++--- .../internal/encoding/byte_array_decoder.go | 12 +++---- .../internal/encoding/byte_array_encoder.go | 10 +++--- go/parquet/internal/encoding/decoder.go | 20 ++++++------ .../internal/encoding/delta_bit_packing.go | 10 +++--- .../internal/encoding/delta_byte_array.go | 6 ++-- .../encoding/delta_byte_array_test.go | 4 +-- .../encoding/delta_length_byte_array.go | 6 ++-- go/parquet/internal/encoding/encoder.go | 16 +++++----- .../encoding/encoding_benchmarks_test.go | 16 +++++----- go/parquet/internal/encoding/encoding_test.go | 14 ++++---- .../encoding/fixed_len_byte_array_decoder.go | 4 +-- .../encoding/fixed_len_byte_array_encoder.go | 6 ++-- go/parquet/internal/encoding/levels.go | 10 +++--- go/parquet/internal/encoding/levels_test.go | 10 +++--- go/parquet/internal/encoding/memo_table.go | 10 +++--- .../internal/encoding/memo_table_test.go | 10 +++--- .../internal/encoding/memo_table_types.gen.go | 4 +-- .../encoding/memo_table_types.gen.go.tmpl | 2 +- .../encoding/plain_encoder_types.gen.go | 10 +++--- .../encoding/plain_encoder_types.gen.go.tmpl | 8 ++--- .../internal/encoding/typed_encoder.gen.go | 18 +++++------ .../encoding/typed_encoder.gen.go.tmpl | 14 ++++---- go/parquet/internal/encoding/types.go | 10 +++--- go/parquet/internal/encryption/aes.go | 2 +- go/parquet/internal/encryption/decryptor.go | 4 +-- go/parquet/internal/encryption/encryptor.go | 4 +-- go/parquet/internal/testutils/pagebuilder.go | 14 ++++---- .../internal/testutils/primitive_typed.go | 10 +++--- go/parquet/internal/testutils/random.go | 14 ++++---- go/parquet/internal/testutils/random_arrow.go | 6 ++-- go/parquet/internal/testutils/utils.go | 2 +- go/parquet/internal/thrift/helpers.go | 2 +- .../internal/utils/bit_benchmark_test.go | 6 ++-- .../internal/utils/bit_packing_arm64.go | 2 +- go/parquet/internal/utils/bit_reader.go | 8 ++--- go/parquet/internal/utils/bit_reader_test.go | 10 +++--- go/parquet/internal/utils/bit_writer.go | 2 +- go/parquet/internal/utils/bitmap_writer.go | 2 +- .../internal/utils/bitmap_writer_test.go | 4 +-- go/parquet/internal/utils/rle.go | 8 ++--- .../internal/utils/typed_rle_dict.gen.go | 6 ++-- .../internal/utils/typed_rle_dict.gen.go.tmpl | 6 ++-- go/parquet/metadata/app_version.go | 4 +-- go/parquet/metadata/column_chunk.go | 14 ++++---- go/parquet/metadata/file.go | 12 +++---- go/parquet/metadata/metadata_test.go | 6 ++-- go/parquet/metadata/row_group.go | 8 ++--- go/parquet/metadata/stat_compare_test.go | 4 +-- go/parquet/metadata/statistics.go | 16 +++++----- go/parquet/metadata/statistics_test.go | 10 +++--- go/parquet/metadata/statistics_types.gen.go | 16 +++++----- .../metadata/statistics_types.gen.go.tmpl | 14 ++++---- go/parquet/pqarrow/column_readers.go | 20 ++++++------ go/parquet/pqarrow/encode_arrow.go | 20 ++++++------ go/parquet/pqarrow/encode_arrow_test.go | 32 +++++++++---------- go/parquet/pqarrow/encode_dict_compute.go | 16 +++++----- go/parquet/pqarrow/encode_dict_nocompute.go | 4 +-- go/parquet/pqarrow/encode_dictionary_test.go | 16 +++++----- go/parquet/pqarrow/file_reader.go | 14 ++++---- go/parquet/pqarrow/file_reader_test.go | 14 ++++---- go/parquet/pqarrow/file_writer.go | 12 +++---- go/parquet/pqarrow/helpers.go | 2 +- go/parquet/pqarrow/path_builder.go | 10 +++--- go/parquet/pqarrow/path_builder_test.go | 8 ++--- go/parquet/pqarrow/properties.go | 6 ++-- go/parquet/pqarrow/reader_writer_test.go | 12 +++---- go/parquet/pqarrow/schema.go | 18 +++++------ go/parquet/pqarrow/schema_test.go | 18 +++++------ go/parquet/reader_properties.go | 4 +-- go/parquet/reader_writer_properties_test.go | 6 ++-- go/parquet/schema/column.go | 4 +-- go/parquet/schema/converted_types.go | 2 +- go/parquet/schema/converted_types_test.go | 2 +- go/parquet/schema/helpers.go | 2 +- go/parquet/schema/helpers_test.go | 4 +-- go/parquet/schema/logical_types.go | 8 ++--- go/parquet/schema/logical_types_test.go | 6 ++-- go/parquet/schema/node.go | 4 +-- go/parquet/schema/reflection.go | 4 +-- go/parquet/schema/reflection_test.go | 4 +-- go/parquet/schema/schema.go | 4 +-- go/parquet/schema/schema_element_test.go | 4 +-- go/parquet/schema/schema_flatten_test.go | 4 +-- go/parquet/schema/schema_test.go | 6 ++-- go/parquet/types.go | 4 +-- go/parquet/writer_properties.go | 6 ++-- java/adapter/avro/pom.xml | 2 +- java/adapter/jdbc/pom.xml | 2 +- java/adapter/orc/pom.xml | 2 +- java/algorithm/pom.xml | 2 +- java/bom/pom.xml | 2 +- java/c/pom.xml | 2 +- java/compression/pom.xml | 2 +- java/dataset/pom.xml | 2 +- java/flight/flight-core/pom.xml | 2 +- java/flight/flight-grpc/pom.xml | 2 +- java/flight/flight-integration-tests/pom.xml | 2 +- java/flight/flight-sql-jdbc-core/pom.xml | 2 +- java/flight/flight-sql-jdbc-driver/pom.xml | 2 +- java/flight/flight-sql/pom.xml | 2 +- java/flight/pom.xml | 2 +- java/format/pom.xml | 2 +- java/gandiva/pom.xml | 2 +- java/memory/memory-core/pom.xml | 2 +- java/memory/memory-netty/pom.xml | 2 +- java/memory/memory-unsafe/pom.xml | 2 +- java/memory/pom.xml | 2 +- java/performance/pom.xml | 2 +- java/pom.xml | 2 +- java/tools/pom.xml | 2 +- java/vector/pom.xml | 2 +- js/package.json | 2 +- matlab/CMakeLists.txt | 2 +- python/CMakeLists.txt | 2 +- python/setup.py | 2 +- r/NEWS.md | 2 -- ruby/red-arrow-cuda/lib/arrow-cuda/version.rb | 2 +- .../lib/arrow-dataset/version.rb | 2 +- .../lib/arrow-flight-sql/version.rb | 2 +- .../lib/arrow-flight/version.rb | 2 +- ruby/red-arrow/lib/arrow/version.rb | 2 +- ruby/red-gandiva/lib/gandiva/version.rb | 2 +- ruby/red-parquet/lib/parquet/version.rb | 2 +- 503 files changed, 1730 insertions(+), 1732 deletions(-) diff --git a/c_glib/meson.build b/c_glib/meson.build index 1aba855703828..7c495d2567d72 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -24,7 +24,7 @@ project('arrow-glib', 'c', 'cpp', 'cpp_std=c++17', ]) -version = '14.0.1-SNAPSHOT' +version = '15.0.0-SNAPSHOT' if version.endswith('-SNAPSHOT') version_numbers = version.split('-')[0].split('.') version_tag = version.split('-')[1] diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d57fcebb498dc..bcb298407bd8b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -71,7 +71,7 @@ if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() -set(ARROW_VERSION "14.0.1-SNAPSHOT") +set(ARROW_VERSION "15.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}") diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index d1508658b8165..c0bf5dce50e32 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -1,6 +1,6 @@ { "name": "arrow", - "version-string": "14.0.1-SNAPSHOT", + "version-string": "15.0.0-SNAPSHOT", "dependencies": [ "abseil", { diff --git a/csharp/Directory.Build.props b/csharp/Directory.Build.props index 1047272ffd448..ae6edda0e2f0e 100644 --- a/csharp/Directory.Build.props +++ b/csharp/Directory.Build.props @@ -29,7 +29,7 @@ Apache Arrow library Copyright 2016-2019 The Apache Software Foundation The Apache Software Foundation - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT diff --git a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb index e9fca5bc74216..e29354def4c1c 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb @@ -29,7 +29,7 @@ class ApacheArrowGlib < Formula desc "GLib bindings for Apache Arrow" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-14.0.1-SNAPSHOT/apache-arrow-14.0.1-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-15.0.0-SNAPSHOT/apache-arrow-15.0.0-SNAPSHOT.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index ff7e54ba64bc0..14d229b477dc8 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -29,7 +29,7 @@ class ApacheArrow < Formula desc "Columnar in-memory analytics layer designed to accelerate big data" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-14.0.1-SNAPSHOT/apache-arrow-14.0.1-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-15.0.0-SNAPSHOT/apache-arrow-15.0.0-SNAPSHOT.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/go/README.md b/go/README.md index 660549cb1b366..c45bcd756f81c 100644 --- a/go/README.md +++ b/go/README.md @@ -20,7 +20,7 @@ Apache Arrow for Go =================== -[![Go Reference](https://pkg.go.dev/badge/github.com/apache/arrow/go/v14.svg)](https://pkg.go.dev/github.com/apache/arrow/go/v14) +[![Go Reference](https://pkg.go.dev/badge/github.com/apache/arrow/go/v15.svg)](https://pkg.go.dev/github.com/apache/arrow/go/v15) [Apache Arrow][arrow] is a cross-language development platform for in-memory data. It specifies a standardized language-independent columnar memory format diff --git a/go/arrow/_examples/helloworld/main.go b/go/arrow/_examples/helloworld/main.go index 02601be694400..156a52b86da1c 100644 --- a/go/arrow/_examples/helloworld/main.go +++ b/go/arrow/_examples/helloworld/main.go @@ -19,10 +19,10 @@ package main import ( "os" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/math" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/math" + "github.com/apache/arrow/go/v15/arrow/memory" ) func main() { diff --git a/go/arrow/_tools/tmpl/main.go b/go/arrow/_tools/tmpl/main.go index 87c4b6ed02c12..c591c3b96803c 100644 --- a/go/arrow/_tools/tmpl/main.go +++ b/go/arrow/_tools/tmpl/main.go @@ -28,7 +28,7 @@ import ( "strings" "text/template" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/internal/json" ) const Ext = ".tmpl" diff --git a/go/arrow/array.go b/go/arrow/array.go index 7622e7503e0b6..e07fa478aae57 100644 --- a/go/arrow/array.go +++ b/go/arrow/array.go @@ -19,8 +19,8 @@ package arrow import ( "fmt" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // ArrayData is the underlying memory and metadata of an Arrow array, corresponding diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go index 1ee04c7aa2bcc..bbe301ee661f3 100644 --- a/go/arrow/array/array.go +++ b/go/arrow/array/array.go @@ -19,9 +19,9 @@ package array import ( "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) const ( diff --git a/go/arrow/array/array_test.go b/go/arrow/array/array_test.go index 6139548ddec7d..bbfbee83585da 100644 --- a/go/arrow/array/array_test.go +++ b/go/arrow/array/array_test.go @@ -19,11 +19,11 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/binary.go b/go/arrow/array/binary.go index e9e6e66e7e813..bf27139fddbaa 100644 --- a/go/arrow/array/binary.go +++ b/go/arrow/array/binary.go @@ -23,8 +23,8 @@ import ( "strings" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/internal/json" ) type BinaryLike interface { diff --git a/go/arrow/array/binary_test.go b/go/arrow/array/binary_test.go index 5febb55691d7d..9c1770950a8b5 100644 --- a/go/arrow/array/binary_test.go +++ b/go/arrow/array/binary_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/binarybuilder.go b/go/arrow/array/binarybuilder.go index 3cb709b45b7a1..a51bc799e4965 100644 --- a/go/arrow/array/binarybuilder.go +++ b/go/arrow/array/binarybuilder.go @@ -24,10 +24,10 @@ import ( "reflect" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // A BinaryBuilder is used to build a Binary array using the Append methods. diff --git a/go/arrow/array/binarybuilder_test.go b/go/arrow/array/binarybuilder_test.go index c63307433bcca..96be73da6516d 100644 --- a/go/arrow/array/binarybuilder_test.go +++ b/go/arrow/array/binarybuilder_test.go @@ -20,9 +20,9 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go index 464cef48bdf32..43bac64a4c990 100644 --- a/go/arrow/array/boolean.go +++ b/go/arrow/array/boolean.go @@ -21,10 +21,10 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // A type which represents an immutable sequence of boolean values. diff --git a/go/arrow/array/boolean_test.go b/go/arrow/array/boolean_test.go index c49ec08624cc6..bcd17ee5967d6 100644 --- a/go/arrow/array/boolean_test.go +++ b/go/arrow/array/boolean_test.go @@ -22,8 +22,8 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/booleanbuilder.go b/go/arrow/array/booleanbuilder.go index 10b7405aa5b2a..cd0cffd5e43e2 100644 --- a/go/arrow/array/booleanbuilder.go +++ b/go/arrow/array/booleanbuilder.go @@ -23,11 +23,11 @@ import ( "strconv" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) type BooleanBuilder struct { diff --git a/go/arrow/array/booleanbuilder_test.go b/go/arrow/array/booleanbuilder_test.go index e270636a87b1c..f6f6c80dab37f 100644 --- a/go/arrow/array/booleanbuilder_test.go +++ b/go/arrow/array/booleanbuilder_test.go @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/bufferbuilder.go b/go/arrow/array/bufferbuilder.go index e023b0d907421..cb381e25b32a2 100644 --- a/go/arrow/array/bufferbuilder.go +++ b/go/arrow/array/bufferbuilder.go @@ -19,9 +19,9 @@ package array import ( "sync/atomic" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" ) type bufBuilder interface { diff --git a/go/arrow/array/bufferbuilder_byte.go b/go/arrow/array/bufferbuilder_byte.go index 00a0d1c21b3ef..9b2b559ba9f68 100644 --- a/go/arrow/array/bufferbuilder_byte.go +++ b/go/arrow/array/bufferbuilder_byte.go @@ -16,7 +16,7 @@ package array -import "github.com/apache/arrow/go/v14/arrow/memory" +import "github.com/apache/arrow/go/v15/arrow/memory" type byteBufferBuilder struct { bufferBuilder diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go b/go/arrow/array/bufferbuilder_numeric.gen.go index 879bc9f571152..a7961166c0edd 100644 --- a/go/arrow/array/bufferbuilder_numeric.gen.go +++ b/go/arrow/array/bufferbuilder_numeric.gen.go @@ -19,9 +19,9 @@ package array import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" ) type int64BufferBuilder struct { diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl b/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl index e859b5bff02a1..845d7ef01c89a 100644 --- a/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl +++ b/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl @@ -17,9 +17,9 @@ package array import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" ) {{range .In}} diff --git a/go/arrow/array/bufferbuilder_numeric_test.go b/go/arrow/array/bufferbuilder_numeric_test.go index ba0a60e5b703b..b51e9ae9207ea 100644 --- a/go/arrow/array/bufferbuilder_numeric_test.go +++ b/go/arrow/array/bufferbuilder_numeric_test.go @@ -20,8 +20,8 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/endian" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/builder.go b/go/arrow/array/builder.go index 2f15ac965e07c..bb15298e03ccf 100644 --- a/go/arrow/array/builder.go +++ b/go/arrow/array/builder.go @@ -20,10 +20,10 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) const ( diff --git a/go/arrow/array/builder_test.go b/go/arrow/array/builder_test.go index 3cacb54f725e7..7bec86d86cc8b 100644 --- a/go/arrow/array/builder_test.go +++ b/go/arrow/array/builder_test.go @@ -19,8 +19,8 @@ package array import ( "testing" - "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/compare.go b/go/arrow/array/compare.go index e70716bee91a7..778de41e32c67 100644 --- a/go/arrow/array/compare.go +++ b/go/arrow/array/compare.go @@ -20,9 +20,9 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/internal/bitutils" ) // RecordEqual reports whether the two provided records are equal. diff --git a/go/arrow/array/compare_test.go b/go/arrow/array/compare_test.go index 51421aa04fe9a..4fc9cf50e8643 100644 --- a/go/arrow/array/compare_test.go +++ b/go/arrow/array/compare_test.go @@ -22,11 +22,11 @@ import ( "sort" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/concat.go b/go/arrow/array/concat.go index 9d815023c4b76..53c5be06895b9 100644 --- a/go/arrow/array/concat.go +++ b/go/arrow/array/concat.go @@ -23,13 +23,13 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/encoded" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/encoded" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/utils" ) // Concatenate creates a new arrow.Array which is the concatenation of the diff --git a/go/arrow/array/concat_test.go b/go/arrow/array/concat_test.go index c80844f05bacd..1cc484ad1a923 100644 --- a/go/arrow/array/concat_test.go +++ b/go/arrow/array/concat_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/array/data.go b/go/arrow/array/data.go index 49df06fb1174a..8cce49182b879 100644 --- a/go/arrow/array/data.go +++ b/go/arrow/array/data.go @@ -22,9 +22,9 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" ) // Data represents the memory and metadata of an Arrow array. diff --git a/go/arrow/array/data_test.go b/go/arrow/array/data_test.go index 2773096995fb6..b7b0f396470d7 100644 --- a/go/arrow/array/data_test.go +++ b/go/arrow/array/data_test.go @@ -19,8 +19,8 @@ package array import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/decimal128.go b/go/arrow/array/decimal128.go index 3317531687ca4..16a492db09c67 100644 --- a/go/arrow/array/decimal128.go +++ b/go/arrow/array/decimal128.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // A type which represents an immutable sequence of 128-bit decimal values. diff --git a/go/arrow/array/decimal128_test.go b/go/arrow/array/decimal128_test.go index 8ef09b68c176a..8c26d00cdc18e 100644 --- a/go/arrow/array/decimal128_test.go +++ b/go/arrow/array/decimal128_test.go @@ -19,10 +19,10 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/decimal256.go b/go/arrow/array/decimal256.go index d63544f784a09..8f72e414d1959 100644 --- a/go/arrow/array/decimal256.go +++ b/go/arrow/array/decimal256.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // Decimal256 is a type that represents an immutable sequence of 256-bit decimal values. diff --git a/go/arrow/array/decimal256_test.go b/go/arrow/array/decimal256_test.go index a02098e79103f..6085d6b5a6a59 100644 --- a/go/arrow/array/decimal256_test.go +++ b/go/arrow/array/decimal256_test.go @@ -19,10 +19,10 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/decimal_test.go b/go/arrow/array/decimal_test.go index 2203b2702c09a..67900447be1cf 100644 --- a/go/arrow/array/decimal_test.go +++ b/go/arrow/array/decimal_test.go @@ -21,12 +21,12 @@ import ( "math/big" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/array/dictionary.go b/go/arrow/array/dictionary.go index d0a1c4dc97e1d..856f91605ff53 100644 --- a/go/arrow/array/dictionary.go +++ b/go/arrow/array/dictionary.go @@ -25,16 +25,16 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/hashing" - "github.com/apache/arrow/go/v14/internal/json" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/hashing" + "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v15/internal/utils" ) // Dictionary represents the type for dictionary-encoded data with a data diff --git a/go/arrow/array/dictionary_test.go b/go/arrow/array/dictionary_test.go index d0878fa3b0329..5a3e0e10c23f3 100644 --- a/go/arrow/array/dictionary_test.go +++ b/go/arrow/array/dictionary_test.go @@ -24,13 +24,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/array/diff.go b/go/arrow/array/diff.go index 026a27b983f59..6bf6372531fd7 100644 --- a/go/arrow/array/diff.go +++ b/go/arrow/array/diff.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) // Edit represents one entry in the edit script to compare two arrays. diff --git a/go/arrow/array/diff_test.go b/go/arrow/array/diff_test.go index 0eff8dc4f0577..17539c38d282f 100644 --- a/go/arrow/array/diff_test.go +++ b/go/arrow/array/diff_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v15/internal/types" ) type diffTestCase struct { diff --git a/go/arrow/array/encoded.go b/go/arrow/array/encoded.go index bf4a942cf1c35..fa5fa7addf34c 100644 --- a/go/arrow/array/encoded.go +++ b/go/arrow/array/encoded.go @@ -23,12 +23,12 @@ import ( "reflect" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/encoded" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/encoded" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v15/internal/utils" ) // RunEndEncoded represents an array containing two children: diff --git a/go/arrow/array/encoded_test.go b/go/arrow/array/encoded_test.go index 57c01cf22d2d6..5bfac7a1a96e6 100644 --- a/go/arrow/array/encoded_test.go +++ b/go/arrow/array/encoded_test.go @@ -20,10 +20,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/arrow/array/extension.go b/go/arrow/array/extension.go index 03e8c1734b9d6..021b8e7bc451b 100644 --- a/go/arrow/array/extension.go +++ b/go/arrow/array/extension.go @@ -20,9 +20,9 @@ import ( "fmt" "reflect" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // ExtensionArray is the interface that needs to be implemented to handle diff --git a/go/arrow/array/extension_test.go b/go/arrow/array/extension_test.go index 5b473fed428ef..a8e2b0dfd59bb 100644 --- a/go/arrow/array/extension_test.go +++ b/go/arrow/array/extension_test.go @@ -19,10 +19,10 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/array/fixed_size_list.go b/go/arrow/array/fixed_size_list.go index 62c3213861a7d..5923d68590b15 100644 --- a/go/arrow/array/fixed_size_list.go +++ b/go/arrow/array/fixed_size_list.go @@ -22,11 +22,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // FixedSizeList represents an immutable sequence of N array values. diff --git a/go/arrow/array/fixed_size_list_test.go b/go/arrow/array/fixed_size_list_test.go index 83dff923fe120..5c01199ddf987 100644 --- a/go/arrow/array/fixed_size_list_test.go +++ b/go/arrow/array/fixed_size_list_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/fixedsize_binary.go b/go/arrow/array/fixedsize_binary.go index 5466156d5a256..6cdaeace939fd 100644 --- a/go/arrow/array/fixedsize_binary.go +++ b/go/arrow/array/fixedsize_binary.go @@ -22,8 +22,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/internal/json" ) // A type which represents an immutable sequence of fixed-length binary strings. diff --git a/go/arrow/array/fixedsize_binary_test.go b/go/arrow/array/fixedsize_binary_test.go index b65c7051a9b3e..785725537cbdd 100644 --- a/go/arrow/array/fixedsize_binary_test.go +++ b/go/arrow/array/fixedsize_binary_test.go @@ -21,9 +21,9 @@ import ( "github.com/stretchr/testify/assert" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestFixedSizeBinary(t *testing.T) { diff --git a/go/arrow/array/fixedsize_binarybuilder.go b/go/arrow/array/fixedsize_binarybuilder.go index ba4b474a89fd0..230a65fd2d352 100644 --- a/go/arrow/array/fixedsize_binarybuilder.go +++ b/go/arrow/array/fixedsize_binarybuilder.go @@ -23,10 +23,10 @@ import ( "reflect" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // A FixedSizeBinaryBuilder is used to build a FixedSizeBinary array using the Append methods. diff --git a/go/arrow/array/fixedsize_binarybuilder_test.go b/go/arrow/array/fixedsize_binarybuilder_test.go index e3962ad3b9372..8e4a0ac1e46a7 100644 --- a/go/arrow/array/fixedsize_binarybuilder_test.go +++ b/go/arrow/array/fixedsize_binarybuilder_test.go @@ -19,8 +19,8 @@ package array import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/float16.go b/go/arrow/array/float16.go index de499e26706cc..4260f8e3774b4 100644 --- a/go/arrow/array/float16.go +++ b/go/arrow/array/float16.go @@ -20,9 +20,9 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/internal/json" ) // A type which represents an immutable sequence of Float16 values. diff --git a/go/arrow/array/float16_builder.go b/go/arrow/array/float16_builder.go index f96ab6037e002..033b9fa2d8028 100644 --- a/go/arrow/array/float16_builder.go +++ b/go/arrow/array/float16_builder.go @@ -23,12 +23,12 @@ import ( "strconv" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) type Float16Builder struct { diff --git a/go/arrow/array/float16_builder_test.go b/go/arrow/array/float16_builder_test.go index 6ee028d978fb0..f8c5890179869 100644 --- a/go/arrow/array/float16_builder_test.go +++ b/go/arrow/array/float16_builder_test.go @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/interval.go b/go/arrow/array/interval.go index ff059c92c8583..2a5529f1c30f7 100644 --- a/go/arrow/array/interval.go +++ b/go/arrow/array/interval.go @@ -23,11 +23,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) func NewIntervalData(data arrow.ArrayData) arrow.Array { diff --git a/go/arrow/array/interval_test.go b/go/arrow/array/interval_test.go index 50a96e4779f03..f83fc52dfa34e 100644 --- a/go/arrow/array/interval_test.go +++ b/go/arrow/array/interval_test.go @@ -20,9 +20,9 @@ import ( "math" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/json_reader.go b/go/arrow/array/json_reader.go index e09717c4199b7..2f05d4b70dd76 100644 --- a/go/arrow/array/json_reader.go +++ b/go/arrow/array/json_reader.go @@ -22,10 +22,10 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) type Option func(config) diff --git a/go/arrow/array/json_reader_test.go b/go/arrow/array/json_reader_test.go index 3a095a0d33189..7f12bf211dd04 100644 --- a/go/arrow/array/json_reader_test.go +++ b/go/arrow/array/json_reader_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/list.go b/go/arrow/array/list.go index d8d8b8c76165a..f10e2072c43a2 100644 --- a/go/arrow/array/list.go +++ b/go/arrow/array/list.go @@ -23,11 +23,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) type ListLike interface { diff --git a/go/arrow/array/list_test.go b/go/arrow/array/list_test.go index bf3555b3f6603..11404b2d8bb95 100644 --- a/go/arrow/array/list_test.go +++ b/go/arrow/array/list_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/map.go b/go/arrow/array/map.go index 9945a90ce495e..fe07a68785067 100644 --- a/go/arrow/array/map.go +++ b/go/arrow/array/map.go @@ -20,9 +20,9 @@ import ( "bytes" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // Map represents an immutable sequence of Key/Value structs. It is a diff --git a/go/arrow/array/map_test.go b/go/arrow/array/map_test.go index 3fe78549ec803..a7ecc032682bc 100644 --- a/go/arrow/array/map_test.go +++ b/go/arrow/array/map_test.go @@ -20,9 +20,9 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/null.go b/go/arrow/array/null.go index 150a1030eb49d..2735a88a92cb3 100644 --- a/go/arrow/array/null.go +++ b/go/arrow/array/null.go @@ -23,10 +23,10 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // Null represents an immutable, degenerate array with no physical storage. diff --git a/go/arrow/array/null_test.go b/go/arrow/array/null_test.go index e1bf1e0345b84..5d230ec5cec71 100644 --- a/go/arrow/array/null_test.go +++ b/go/arrow/array/null_test.go @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go index a3e1101516aa6..59c9a979768d5 100644 --- a/go/arrow/array/numeric.gen.go +++ b/go/arrow/array/numeric.gen.go @@ -23,8 +23,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/internal/json" ) // A type which represents an immutable sequence of int64 values. diff --git a/go/arrow/array/numeric.gen.go.tmpl b/go/arrow/array/numeric.gen.go.tmpl index 34d17fbfc8854..027456009daad 100644 --- a/go/arrow/array/numeric.gen.go.tmpl +++ b/go/arrow/array/numeric.gen.go.tmpl @@ -21,8 +21,8 @@ import ( "strings" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/internal/json" ) {{range .In}} diff --git a/go/arrow/array/numeric_test.go b/go/arrow/array/numeric_test.go index 34f59c238cbef..91dd724c8da50 100644 --- a/go/arrow/array/numeric_test.go +++ b/go/arrow/array/numeric_test.go @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go index 7f01180f55957..52b189d6ed453 100644 --- a/go/arrow/array/numericbuilder.gen.go +++ b/go/arrow/array/numericbuilder.gen.go @@ -27,11 +27,11 @@ import ( "sync/atomic" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) type Int64Builder struct { diff --git a/go/arrow/array/numericbuilder.gen.go.tmpl b/go/arrow/array/numericbuilder.gen.go.tmpl index cf663c031f616..82ac35465d424 100644 --- a/go/arrow/array/numericbuilder.gen.go.tmpl +++ b/go/arrow/array/numericbuilder.gen.go.tmpl @@ -17,11 +17,11 @@ package array import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) {{range .In}} diff --git a/go/arrow/array/numericbuilder.gen_test.go b/go/arrow/array/numericbuilder.gen_test.go index b5986775cd995..e1f72773403d8 100644 --- a/go/arrow/array/numericbuilder.gen_test.go +++ b/go/arrow/array/numericbuilder.gen_test.go @@ -21,9 +21,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numericbuilder.gen_test.go.tmpl b/go/arrow/array/numericbuilder.gen_test.go.tmpl index bc8c993374671..eddd884e2eddf 100644 --- a/go/arrow/array/numericbuilder.gen_test.go.tmpl +++ b/go/arrow/array/numericbuilder.gen_test.go.tmpl @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/record.go b/go/arrow/array/record.go index 0b0fe4c3847fc..d080f726e472d 100644 --- a/go/arrow/array/record.go +++ b/go/arrow/array/record.go @@ -22,10 +22,10 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // RecordReader reads a stream of records. diff --git a/go/arrow/array/record_test.go b/go/arrow/array/record_test.go index da50a30b646f1..7d438d1f1f81e 100644 --- a/go/arrow/array/record_test.go +++ b/go/arrow/array/record_test.go @@ -21,9 +21,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/string.go b/go/arrow/array/string.go index 86e27c970cbe9..9ab7c938ef5d8 100644 --- a/go/arrow/array/string.go +++ b/go/arrow/array/string.go @@ -23,9 +23,9 @@ import ( "strings" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // String represents an immutable sequence of variable-length UTF-8 strings. diff --git a/go/arrow/array/string_test.go b/go/arrow/array/string_test.go index fbc106b098332..d743a3ec7f37f 100644 --- a/go/arrow/array/string_test.go +++ b/go/arrow/array/string_test.go @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go index 248a25bf6044d..94052953852c2 100644 --- a/go/arrow/array/struct.go +++ b/go/arrow/array/struct.go @@ -23,11 +23,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // Struct represents an ordered sequence of relative types. diff --git a/go/arrow/array/struct_test.go b/go/arrow/array/struct_test.go index 86a5b311fcc21..1b0dc5a3e4b19 100644 --- a/go/arrow/array/struct_test.go +++ b/go/arrow/array/struct_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/table.go b/go/arrow/array/table.go index 6456992e34973..a987dd057f82c 100644 --- a/go/arrow/array/table.go +++ b/go/arrow/array/table.go @@ -23,8 +23,8 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) // NewColumnSlice returns a new zero-copy slice of the column with the indicated diff --git a/go/arrow/array/table_test.go b/go/arrow/array/table_test.go index 8e22d53963ba4..9535ae6b089bb 100644 --- a/go/arrow/array/table_test.go +++ b/go/arrow/array/table_test.go @@ -22,9 +22,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestChunked(t *testing.T) { diff --git a/go/arrow/array/timestamp.go b/go/arrow/array/timestamp.go index 2928b1fc77ff6..6ffb43e067af0 100644 --- a/go/arrow/array/timestamp.go +++ b/go/arrow/array/timestamp.go @@ -24,11 +24,11 @@ import ( "sync/atomic" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) // Timestamp represents an immutable sequence of arrow.Timestamp values. diff --git a/go/arrow/array/timestamp_test.go b/go/arrow/array/timestamp_test.go index 1e55364e8da0d..acbad8b586dd4 100644 --- a/go/arrow/array/timestamp_test.go +++ b/go/arrow/array/timestamp_test.go @@ -20,9 +20,9 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/union.go b/go/arrow/array/union.go index 869355ac7108a..c0a5050560634 100644 --- a/go/arrow/array/union.go +++ b/go/arrow/array/union.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/json" ) // Union is a convenience interface to encompass both Sparse and Dense diff --git a/go/arrow/array/union_test.go b/go/arrow/array/union_test.go index 2f2bf3b84e9f7..e876f5def26d3 100644 --- a/go/arrow/array/union_test.go +++ b/go/arrow/array/union_test.go @@ -21,9 +21,9 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/array/util.go b/go/arrow/array/util.go index 54d15a8095c80..a1b3cc7d4e5f7 100644 --- a/go/arrow/array/util.go +++ b/go/arrow/array/util.go @@ -22,11 +22,11 @@ import ( "io" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/hashing" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/hashing" + "github.com/apache/arrow/go/v15/internal/json" ) func min(a, b int) int { diff --git a/go/arrow/array/util_test.go b/go/arrow/array/util_test.go index d3d2b37365513..84a6debdf3946 100644 --- a/go/arrow/array/util_test.go +++ b/go/arrow/array/util_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/arrio/arrio.go b/go/arrow/arrio/arrio.go index 466a93a68400a..51cf6dc46d44a 100644 --- a/go/arrow/arrio/arrio.go +++ b/go/arrow/arrio/arrio.go @@ -22,7 +22,7 @@ import ( "errors" "io" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) // Reader is the interface that wraps the Read method. diff --git a/go/arrow/arrio/arrio_test.go b/go/arrow/arrio/arrio_test.go index 7e9aac8fbe520..c80d5d2569d67 100644 --- a/go/arrow/arrio/arrio_test.go +++ b/go/arrow/arrio/arrio_test.go @@ -22,11 +22,11 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) type copyKind int diff --git a/go/arrow/bitutil/bitmaps.go b/go/arrow/bitutil/bitmaps.go index 2e9c0601c3c45..d7516771def7f 100644 --- a/go/arrow/bitutil/bitmaps.go +++ b/go/arrow/bitutil/bitmaps.go @@ -22,9 +22,9 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" ) // BitmapReader is a simple bitmap reader for a byte slice. diff --git a/go/arrow/bitutil/bitmaps_test.go b/go/arrow/bitutil/bitmaps_test.go index 60022a179d1da..c926bff39e09a 100644 --- a/go/arrow/bitutil/bitmaps_test.go +++ b/go/arrow/bitutil/bitmaps_test.go @@ -22,8 +22,8 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/bitutil/bitutil.go b/go/arrow/bitutil/bitutil.go index a4a1519b8f940..dc510a8b374c4 100644 --- a/go/arrow/bitutil/bitutil.go +++ b/go/arrow/bitutil/bitutil.go @@ -22,7 +22,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory" ) var ( diff --git a/go/arrow/bitutil/bitutil_test.go b/go/arrow/bitutil/bitutil_test.go index 9e29e9d04d1db..189c8541f4925 100644 --- a/go/arrow/bitutil/bitutil_test.go +++ b/go/arrow/bitutil/bitutil_test.go @@ -21,8 +21,8 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/cdata/cdata.go b/go/arrow/cdata/cdata.go index 1d454c57f722e..f9693851d7483 100644 --- a/go/arrow/cdata/cdata.go +++ b/go/arrow/cdata/cdata.go @@ -46,10 +46,10 @@ import ( "syscall" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/cdata/cdata_exports.go b/go/arrow/cdata/cdata_exports.go index 187c2deb9755f..91f1b352e0327 100644 --- a/go/arrow/cdata/cdata_exports.go +++ b/go/arrow/cdata/cdata_exports.go @@ -45,11 +45,11 @@ import ( "strings" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/internal" - "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/internal" + "github.com/apache/arrow/go/v15/arrow/ipc" ) func encodeCMetadata(keys, values []string) []byte { diff --git a/go/arrow/cdata/cdata_test.go b/go/arrow/cdata/cdata_test.go index f09fa3ff2f6fa..607cfe53217a6 100644 --- a/go/arrow/cdata/cdata_test.go +++ b/go/arrow/cdata/cdata_test.go @@ -35,12 +35,12 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/memory/mallocator" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory/mallocator" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/cdata/cdata_test_framework.go b/go/arrow/cdata/cdata_test_framework.go index 2df52dcb3cd40..1251b20201e41 100644 --- a/go/arrow/cdata/cdata_test_framework.go +++ b/go/arrow/cdata/cdata_test_framework.go @@ -69,10 +69,10 @@ import ( "runtime/cgo" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/internal" - "github.com/apache/arrow/go/v14/arrow/memory/mallocator" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/internal" + "github.com/apache/arrow/go/v15/arrow/memory/mallocator" ) const ( diff --git a/go/arrow/cdata/exports.go b/go/arrow/cdata/exports.go index 9d2576818e31c..7353df62d113a 100644 --- a/go/arrow/cdata/exports.go +++ b/go/arrow/cdata/exports.go @@ -20,8 +20,8 @@ import ( "runtime/cgo" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" ) // #include diff --git a/go/arrow/cdata/import_allocator.go b/go/arrow/cdata/import_allocator.go index eff8c7517caef..cf1c6a961ff37 100644 --- a/go/arrow/cdata/import_allocator.go +++ b/go/arrow/cdata/import_allocator.go @@ -20,7 +20,7 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) // #include "arrow/c/helpers.h" diff --git a/go/arrow/cdata/interface.go b/go/arrow/cdata/interface.go index bf5c5270bae2f..8ce06280a0bf5 100644 --- a/go/arrow/cdata/interface.go +++ b/go/arrow/cdata/interface.go @@ -22,10 +22,10 @@ package cdata import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/cdata/test/test_cimport.go b/go/arrow/cdata/test/test_cimport.go index a7c5dc034496a..147c3691f0c71 100644 --- a/go/arrow/cdata/test/test_cimport.go +++ b/go/arrow/cdata/test/test_cimport.go @@ -23,10 +23,10 @@ import ( "fmt" "runtime" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/cdata" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/cdata" + "github.com/apache/arrow/go/v15/arrow/memory" ) // #include diff --git a/go/arrow/compute/arithmetic.go b/go/arrow/compute/arithmetic.go index 2fb95f06ff84a..1ee1959b2ddc8 100644 --- a/go/arrow/compute/arithmetic.go +++ b/go/arrow/compute/arithmetic.go @@ -22,12 +22,12 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/scalar" ) type ( diff --git a/go/arrow/compute/arithmetic_test.go b/go/arrow/compute/arithmetic_test.go index a99b2074eb69f..821ffd9e068d4 100644 --- a/go/arrow/compute/arithmetic_test.go +++ b/go/arrow/compute/arithmetic_test.go @@ -26,16 +26,16 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/klauspost/cpuid/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/arrow/compute/cast.go b/go/arrow/compute/cast.go index 8b720a2bad5d8..133a983038ce1 100644 --- a/go/arrow/compute/cast.go +++ b/go/arrow/compute/cast.go @@ -23,11 +23,11 @@ import ( "fmt" "sync" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/cast_test.go b/go/arrow/compute/cast_test.go index cc301cc35e6e5..10957a45167aa 100644 --- a/go/arrow/compute/cast_test.go +++ b/go/arrow/compute/cast_test.go @@ -26,16 +26,16 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v15/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/compute/datum.go b/go/arrow/compute/datum.go index 1d3c1b4dab700..98bd1f1a0a326 100644 --- a/go/arrow/compute/datum.go +++ b/go/arrow/compute/datum.go @@ -21,9 +21,9 @@ package compute import ( "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/scalar" ) //go:generate go run golang.org/x/tools/cmd/stringer -type=DatumKind -linecomment diff --git a/go/arrow/compute/example_test.go b/go/arrow/compute/example_test.go index f0db9f377868e..e2b07b7e191ae 100644 --- a/go/arrow/compute/example_test.go +++ b/go/arrow/compute/example_test.go @@ -23,11 +23,11 @@ import ( "fmt" "log" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/memory" ) // This example demonstrates how to register a custom scalar function. diff --git a/go/arrow/compute/exec.go b/go/arrow/compute/exec.go index 84e3310cc2df8..eba47e64bd509 100644 --- a/go/arrow/compute/exec.go +++ b/go/arrow/compute/exec.go @@ -22,9 +22,9 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) func haveChunkedArray(values []Datum) bool { diff --git a/go/arrow/compute/exec/kernel.go b/go/arrow/compute/exec/kernel.go index 327426da68eb9..766857f63e565 100644 --- a/go/arrow/compute/exec/kernel.go +++ b/go/arrow/compute/exec/kernel.go @@ -24,10 +24,10 @@ import ( "hash/maphash" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/exec/kernel_test.go b/go/arrow/compute/exec/kernel_test.go index 16a29ea2deda6..4df6b42ff9408 100644 --- a/go/arrow/compute/exec/kernel_test.go +++ b/go/arrow/compute/exec/kernel_test.go @@ -22,12 +22,12 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exec/span.go b/go/arrow/compute/exec/span.go index b6d240fa4a0be..0b5f6208227f5 100644 --- a/go/arrow/compute/exec/span.go +++ b/go/arrow/compute/exec/span.go @@ -23,11 +23,11 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" ) // BufferSpan is a lightweight Buffer holder for ArraySpans that does not diff --git a/go/arrow/compute/exec/span_test.go b/go/arrow/compute/exec/span_test.go index c642be1c30c23..474c005b44642 100644 --- a/go/arrow/compute/exec/span_test.go +++ b/go/arrow/compute/exec/span_test.go @@ -24,14 +24,14 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v15/internal/types" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exec/utils.go b/go/arrow/compute/exec/utils.go index 6d83b75da1313..17bc30ef2adb0 100644 --- a/go/arrow/compute/exec/utils.go +++ b/go/arrow/compute/exec/utils.go @@ -25,13 +25,13 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/exp/constraints" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/exec/utils_test.go b/go/arrow/compute/exec/utils_test.go index e9bb07177f2ac..b26e4ff41e79f 100644 --- a/go/arrow/compute/exec/utils_test.go +++ b/go/arrow/compute/exec/utils_test.go @@ -21,10 +21,10 @@ package exec_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exec_internals_test.go b/go/arrow/compute/exec_internals_test.go index eb9273d23389f..bae32268862ff 100644 --- a/go/arrow/compute/exec_internals_test.go +++ b/go/arrow/compute/exec_internals_test.go @@ -24,13 +24,13 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/compute/exec_test.go b/go/arrow/compute/exec_test.go index 30bfbc0246a1c..a37f67c03e8ce 100644 --- a/go/arrow/compute/exec_test.go +++ b/go/arrow/compute/exec_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/compute/executor.go b/go/arrow/compute/executor.go index 6da7ed1293065..1cba0b1e19f69 100644 --- a/go/arrow/compute/executor.go +++ b/go/arrow/compute/executor.go @@ -25,14 +25,14 @@ import ( "runtime" "sync" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" ) // ExecCtx holds simple contextual information for execution diff --git a/go/arrow/compute/expression.go b/go/arrow/compute/expression.go index 9f20c9704dca7..fbb6c502d98ab 100644 --- a/go/arrow/compute/expression.go +++ b/go/arrow/compute/expression.go @@ -28,14 +28,14 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" ) var hashSeed = maphash.MakeSeed() diff --git a/go/arrow/compute/expression_test.go b/go/arrow/compute/expression_test.go index 71b1b64c4b0c1..b3b44ae1faa61 100644 --- a/go/arrow/compute/expression_test.go +++ b/go/arrow/compute/expression_test.go @@ -22,11 +22,11 @@ package compute_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exprs/builders.go b/go/arrow/compute/exprs/builders.go index 39606c27e714a..0b694525d1b47 100644 --- a/go/arrow/compute/exprs/builders.go +++ b/go/arrow/compute/exprs/builders.go @@ -25,8 +25,8 @@ import ( "strings" "unicode" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute" "github.com/substrait-io/substrait-go/expr" "github.com/substrait-io/substrait-go/extensions" "github.com/substrait-io/substrait-go/types" diff --git a/go/arrow/compute/exprs/builders_test.go b/go/arrow/compute/exprs/builders_test.go index e42d7569a8f03..69501622359d5 100644 --- a/go/arrow/compute/exprs/builders_test.go +++ b/go/arrow/compute/exprs/builders_test.go @@ -21,8 +21,8 @@ package exprs_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exprs" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exprs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/substrait-io/substrait-go/expr" diff --git a/go/arrow/compute/exprs/exec.go b/go/arrow/compute/exprs/exec.go index d63752696082b..800ffe62f2559 100644 --- a/go/arrow/compute/exprs/exec.go +++ b/go/arrow/compute/exprs/exec.go @@ -23,15 +23,15 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/substrait-io/substrait-go/expr" "github.com/substrait-io/substrait-go/extensions" "github.com/substrait-io/substrait-go/types" diff --git a/go/arrow/compute/exprs/exec_internal_test.go b/go/arrow/compute/exprs/exec_internal_test.go index 9475a9a8cbca9..680bf36f11958 100644 --- a/go/arrow/compute/exprs/exec_internal_test.go +++ b/go/arrow/compute/exprs/exec_internal_test.go @@ -23,10 +23,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/compute/exprs/exec_test.go b/go/arrow/compute/exprs/exec_test.go index e30d2afdb73f8..65cafc8e62ddb 100644 --- a/go/arrow/compute/exprs/exec_test.go +++ b/go/arrow/compute/exprs/exec_test.go @@ -23,12 +23,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exprs" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exprs" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/substrait-io/substrait-go/expr" diff --git a/go/arrow/compute/exprs/extension_types.go b/go/arrow/compute/exprs/extension_types.go index ce375492d67f1..db7992b8f089f 100644 --- a/go/arrow/compute/exprs/extension_types.go +++ b/go/arrow/compute/exprs/extension_types.go @@ -24,8 +24,8 @@ import ( "reflect" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" ) type simpleExtensionTypeFactory[P comparable] struct { diff --git a/go/arrow/compute/exprs/field_refs.go b/go/arrow/compute/exprs/field_refs.go index fd4313818195e..e95e3c8c9abb9 100644 --- a/go/arrow/compute/exprs/field_refs.go +++ b/go/arrow/compute/exprs/field_refs.go @@ -21,11 +21,11 @@ package exprs import ( "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/substrait-io/substrait-go/expr" ) diff --git a/go/arrow/compute/exprs/types.go b/go/arrow/compute/exprs/types.go index 9f807e1e1ecbc..6a5b81d11b3eb 100644 --- a/go/arrow/compute/exprs/types.go +++ b/go/arrow/compute/exprs/types.go @@ -24,8 +24,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute" "github.com/substrait-io/substrait-go/expr" "github.com/substrait-io/substrait-go/extensions" "github.com/substrait-io/substrait-go/types" diff --git a/go/arrow/compute/fieldref.go b/go/arrow/compute/fieldref.go index ee6f39948ebf1..565ae3bfadbd0 100644 --- a/go/arrow/compute/fieldref.go +++ b/go/arrow/compute/fieldref.go @@ -27,8 +27,8 @@ import ( "unicode" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" ) var ( diff --git a/go/arrow/compute/fieldref_test.go b/go/arrow/compute/fieldref_test.go index c38ad8ac1fe30..c4fa72182835f 100644 --- a/go/arrow/compute/fieldref_test.go +++ b/go/arrow/compute/fieldref_test.go @@ -19,10 +19,10 @@ package compute_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/functions.go b/go/arrow/compute/functions.go index a1905f91850f0..b85062721d9b8 100644 --- a/go/arrow/compute/functions.go +++ b/go/arrow/compute/functions.go @@ -23,8 +23,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exec" ) type Function interface { diff --git a/go/arrow/compute/functions_test.go b/go/arrow/compute/functions_test.go index b37da285badd0..1e6bbd598e0c9 100644 --- a/go/arrow/compute/functions_test.go +++ b/go/arrow/compute/functions_test.go @@ -21,8 +21,8 @@ package compute_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/compute/internal/kernels/base_arithmetic.go b/go/arrow/compute/internal/kernels/base_arithmetic.go index 67e80af74f511..c7950877264df 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic.go @@ -24,11 +24,11 @@ import ( "math/bits" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/internal/debug" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go b/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go index 0e78e6c9183ff..942b8e4ff5600 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go @@ -21,8 +21,8 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/debug" "golang.org/x/exp/constraints" "golang.org/x/sys/cpu" ) diff --git a/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go b/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go index 29cce7830e0f4..6814e834b4bf3 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go b/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go index e9b03551f81b1..633ec5f4f18e5 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go b/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go index 4f160a1411091..8e46ca030c8b7 100644 --- a/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go +++ b/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go @@ -19,7 +19,7 @@ package kernels import ( - "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/exec" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/boolean_cast.go b/go/arrow/compute/internal/kernels/boolean_cast.go index 18d04c845c41d..923c5b3f54512 100644 --- a/go/arrow/compute/internal/kernels/boolean_cast.go +++ b/go/arrow/compute/internal/kernels/boolean_cast.go @@ -22,9 +22,9 @@ import ( "strconv" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" ) func isNonZero[T exec.FixedWidthTypes](ctx *exec.KernelCtx, in []T, out []byte) error { diff --git a/go/arrow/compute/internal/kernels/cast.go b/go/arrow/compute/internal/kernels/cast.go index 5a71206b2476c..bc4ee3abd128c 100644 --- a/go/arrow/compute/internal/kernels/cast.go +++ b/go/arrow/compute/internal/kernels/cast.go @@ -19,9 +19,9 @@ package kernels import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute/exec" ) type CastOptions struct { diff --git a/go/arrow/compute/internal/kernels/cast_numeric.go b/go/arrow/compute/internal/kernels/cast_numeric.go index 4e5c5c1d9ea77..2e893c7205f6a 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric.go +++ b/go/arrow/compute/internal/kernels/cast_numeric.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) var castNumericUnsafe func(itype, otype arrow.Type, in, out []byte, len int) = castNumericGo diff --git a/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go b/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go index 6b28441ec083e..eafa4b41001ae 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go +++ b/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go b/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go index d53a4486f6983..75ad79fd0e7f3 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go +++ b/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" "golang.org/x/sys/cpu" ) diff --git a/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go b/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go index 1cbea033dec27..d04f393c4c099 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go +++ b/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/cast_temporal.go b/go/arrow/compute/internal/kernels/cast_temporal.go index 82fce1e33c959..542a8a4590b28 100644 --- a/go/arrow/compute/internal/kernels/cast_temporal.go +++ b/go/arrow/compute/internal/kernels/cast_temporal.go @@ -24,10 +24,10 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) const millisecondsInDay = 86400000 diff --git a/go/arrow/compute/internal/kernels/helpers.go b/go/arrow/compute/internal/kernels/helpers.go index ed25071c9d87f..686c4b3e0c29a 100644 --- a/go/arrow/compute/internal/kernels/helpers.go +++ b/go/arrow/compute/internal/kernels/helpers.go @@ -22,13 +22,13 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v15/internal/bitutils" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/numeric_cast.go b/go/arrow/compute/internal/kernels/numeric_cast.go index 8e5350754723d..c055552bf7ff5 100644 --- a/go/arrow/compute/internal/kernels/numeric_cast.go +++ b/go/arrow/compute/internal/kernels/numeric_cast.go @@ -23,13 +23,13 @@ import ( "strconv" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/internal/bitutils" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/rounding.go b/go/arrow/compute/internal/kernels/rounding.go index 2f58a9faea8a2..8a1bec1180ac8 100644 --- a/go/arrow/compute/internal/kernels/rounding.go +++ b/go/arrow/compute/internal/kernels/rounding.go @@ -22,11 +22,11 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/scalar" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/scalar_arithmetic.go b/go/arrow/compute/internal/kernels/scalar_arithmetic.go index 9cb32ae64a4a9..cf17e9fd9548b 100644 --- a/go/arrow/compute/internal/kernels/scalar_arithmetic.go +++ b/go/arrow/compute/internal/kernels/scalar_arithmetic.go @@ -22,13 +22,13 @@ import ( "fmt" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/scalar" ) // scalar kernel that ignores (assumed all-null inputs) and returns null diff --git a/go/arrow/compute/internal/kernels/scalar_boolean.go b/go/arrow/compute/internal/kernels/scalar_boolean.go index 812f4ad1bd363..0707c92e6a198 100644 --- a/go/arrow/compute/internal/kernels/scalar_boolean.go +++ b/go/arrow/compute/internal/kernels/scalar_boolean.go @@ -19,9 +19,9 @@ package kernels import ( - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/scalar" ) type computeWordFN func(leftTrue, leftFalse, rightTrue, rightFalse uint64) (outValid, outData uint64) diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go b/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go index 585d1bff01ab6..8e5ce1ab7c1ad 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go @@ -21,8 +21,8 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exec" "golang.org/x/sys/cpu" ) diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go b/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go index 868179057ccf4..cf9fc1eeedbb8 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go b/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go index 56abad42b1ffc..c0aef5a04e9b8 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go @@ -18,7 +18,7 @@ package kernels -import "github.com/apache/arrow/go/v14/arrow/compute/exec" +import "github.com/apache/arrow/go/v15/arrow/compute/exec" func genCompareKernel[T exec.NumericTypes](op CompareOperator) *CompareData { return genGoCompareKernel(getCmpOp[T](op)) diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go b/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go index 7cf96a4171d6c..f8b36a1e4be76 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/scalar_comparisons.go b/go/arrow/compute/internal/kernels/scalar_comparisons.go index 8a957eafd85da..9a7640a8d8a39 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparisons.go +++ b/go/arrow/compute/internal/kernels/scalar_comparisons.go @@ -23,14 +23,14 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/scalar" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v15/internal/bitutils" ) type binaryKernel func(left, right, out []byte, offset int) diff --git a/go/arrow/compute/internal/kernels/string_casts.go b/go/arrow/compute/internal/kernels/string_casts.go index 3070514665d7d..76da901e33f8d 100644 --- a/go/arrow/compute/internal/kernels/string_casts.go +++ b/go/arrow/compute/internal/kernels/string_casts.go @@ -23,12 +23,12 @@ import ( "strconv" "unicode/utf8" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/internal/bitutils" ) func validateUtf8Fsb(input *exec.ArraySpan) error { diff --git a/go/arrow/compute/internal/kernels/types.go b/go/arrow/compute/internal/kernels/types.go index 2788fb7054c53..481eab36059f8 100644 --- a/go/arrow/compute/internal/kernels/types.go +++ b/go/arrow/compute/internal/kernels/types.go @@ -21,10 +21,10 @@ package kernels import ( "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/scalar" ) var ( diff --git a/go/arrow/compute/internal/kernels/vector_hash.go b/go/arrow/compute/internal/kernels/vector_hash.go index e0ede826e876e..9401e31cc5b09 100644 --- a/go/arrow/compute/internal/kernels/vector_hash.go +++ b/go/arrow/compute/internal/kernels/vector_hash.go @@ -21,13 +21,13 @@ package kernels import ( "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/internal/hashing" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/hashing" ) type HashState interface { diff --git a/go/arrow/compute/internal/kernels/vector_run_end_encode.go b/go/arrow/compute/internal/kernels/vector_run_end_encode.go index e073ff1ff5308..a147bf7d50170 100644 --- a/go/arrow/compute/internal/kernels/vector_run_end_encode.go +++ b/go/arrow/compute/internal/kernels/vector_run_end_encode.go @@ -24,14 +24,14 @@ import ( "sort" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" ) type RunEndEncodeState struct { diff --git a/go/arrow/compute/internal/kernels/vector_selection.go b/go/arrow/compute/internal/kernels/vector_selection.go index c7a902bd008d0..714e452325bfd 100644 --- a/go/arrow/compute/internal/kernels/vector_selection.go +++ b/go/arrow/compute/internal/kernels/vector_selection.go @@ -22,13 +22,13 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" ) type NullSelectionBehavior int8 diff --git a/go/arrow/compute/registry.go b/go/arrow/compute/registry.go index 4f1c435fb680a..379e0ccbe86a9 100644 --- a/go/arrow/compute/registry.go +++ b/go/arrow/compute/registry.go @@ -21,7 +21,7 @@ package compute import ( "sync" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/debug" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/registry_test.go b/go/arrow/compute/registry_test.go index 428f7cf1d8fda..4e4f44f1d39b6 100644 --- a/go/arrow/compute/registry_test.go +++ b/go/arrow/compute/registry_test.go @@ -23,9 +23,9 @@ import ( "errors" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exec" "github.com/stretchr/testify/assert" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/scalar_bool.go b/go/arrow/compute/scalar_bool.go index 49c745688284e..1f28a6e2bfcb2 100644 --- a/go/arrow/compute/scalar_bool.go +++ b/go/arrow/compute/scalar_bool.go @@ -21,9 +21,9 @@ package compute import ( "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/scalar_bool_test.go b/go/arrow/compute/scalar_bool_test.go index 6eadc973d6965..bd4f3c5c0df2b 100644 --- a/go/arrow/compute/scalar_bool_test.go +++ b/go/arrow/compute/scalar_bool_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/compute/scalar_compare.go b/go/arrow/compute/scalar_compare.go index 476f37711e289..24a4191a10999 100644 --- a/go/arrow/compute/scalar_compare.go +++ b/go/arrow/compute/scalar_compare.go @@ -21,9 +21,9 @@ package compute import ( "context" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" ) type compareFunction struct { diff --git a/go/arrow/compute/scalar_compare_test.go b/go/arrow/compute/scalar_compare_test.go index 13fdd24fdd6b3..d209f72c800b0 100644 --- a/go/arrow/compute/scalar_compare_test.go +++ b/go/arrow/compute/scalar_compare_test.go @@ -24,15 +24,15 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/compute/selection.go b/go/arrow/compute/selection.go index ed6d80418d3f6..90bc5280ef2a7 100644 --- a/go/arrow/compute/selection.go +++ b/go/arrow/compute/selection.go @@ -22,10 +22,10 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" "golang.org/x/sync/errgroup" ) diff --git a/go/arrow/compute/utils.go b/go/arrow/compute/utils.go index cc4d6edc48ec8..9de6523fd0b5f 100644 --- a/go/arrow/compute/utils.go +++ b/go/arrow/compute/utils.go @@ -24,12 +24,12 @@ import ( "math" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/compute/vector_hash.go b/go/arrow/compute/vector_hash.go index 5f9aec55c55bc..144c123380fe2 100644 --- a/go/arrow/compute/vector_hash.go +++ b/go/arrow/compute/vector_hash.go @@ -21,8 +21,8 @@ package compute import ( "context" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/vector_hash_test.go b/go/arrow/compute/vector_hash_test.go index 50c96ddc04f6c..9410720de7941 100644 --- a/go/arrow/compute/vector_hash_test.go +++ b/go/arrow/compute/vector_hash_test.go @@ -23,13 +23,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/compute/vector_run_end_test.go b/go/arrow/compute/vector_run_end_test.go index 8d894d96b8f0a..51c0f834ceb6a 100644 --- a/go/arrow/compute/vector_run_end_test.go +++ b/go/arrow/compute/vector_run_end_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/compute/vector_run_ends.go b/go/arrow/compute/vector_run_ends.go index 48f3dcba8c481..3e47c67de3c35 100644 --- a/go/arrow/compute/vector_run_ends.go +++ b/go/arrow/compute/vector_run_ends.go @@ -21,8 +21,8 @@ package compute import ( "context" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/vector_selection_test.go b/go/arrow/compute/vector_selection_test.go index 10788d60e10d2..f44840ba72034 100644 --- a/go/arrow/compute/vector_selection_test.go +++ b/go/arrow/compute/vector_selection_test.go @@ -24,15 +24,15 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/compute/exec" - "github.com/apache/arrow/go/v14/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v14/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v15/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/csv/common.go b/go/arrow/csv/common.go index 91f42897a24a6..2a1f7300a986f 100644 --- a/go/arrow/csv/common.go +++ b/go/arrow/csv/common.go @@ -22,8 +22,8 @@ import ( "errors" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" ) var ( diff --git a/go/arrow/csv/reader.go b/go/arrow/csv/reader.go index 80af520a12b7f..e58b426d837fb 100644 --- a/go/arrow/csv/reader.go +++ b/go/arrow/csv/reader.go @@ -29,13 +29,13 @@ import ( "time" "unicode/utf8" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" ) // Reader wraps encoding/csv.Reader and creates array.Records from a schema. diff --git a/go/arrow/csv/reader_test.go b/go/arrow/csv/reader_test.go index f481cd2e961d3..dfcb6625bd7cc 100644 --- a/go/arrow/csv/reader_test.go +++ b/go/arrow/csv/reader_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/csv" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/csv" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/csv/transformer.go b/go/arrow/csv/transformer.go index 7c8eafcee1a59..0f0181520b847 100644 --- a/go/arrow/csv/transformer.go +++ b/go/arrow/csv/transformer.go @@ -25,8 +25,8 @@ import ( "math/big" "strconv" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" ) func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array) []string { diff --git a/go/arrow/csv/writer.go b/go/arrow/csv/writer.go index e558364ecf16d..a672008b58a07 100644 --- a/go/arrow/csv/writer.go +++ b/go/arrow/csv/writer.go @@ -22,7 +22,7 @@ import ( "strconv" "sync" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) // Writer wraps encoding/csv.Writer and writes arrow.Record based on a schema. diff --git a/go/arrow/csv/writer_test.go b/go/arrow/csv/writer_test.go index f945023790426..1918e2e492dff 100644 --- a/go/arrow/csv/writer_test.go +++ b/go/arrow/csv/writer_test.go @@ -25,14 +25,14 @@ import ( "log" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/csv" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/csv" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" "github.com/google/uuid" ) diff --git a/go/arrow/datatype.go b/go/arrow/datatype.go index f0fb24ec873c5..24113b55899dc 100644 --- a/go/arrow/datatype.go +++ b/go/arrow/datatype.go @@ -21,7 +21,7 @@ import ( "hash/maphash" "strings" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) // Type is a logical type. They can be expressed as diff --git a/go/arrow/datatype_binary_test.go b/go/arrow/datatype_binary_test.go index c47df3da1db9c..25ba6e8db4ba4 100644 --- a/go/arrow/datatype_binary_test.go +++ b/go/arrow/datatype_binary_test.go @@ -19,7 +19,7 @@ package arrow_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) func TestBinaryType(t *testing.T) { diff --git a/go/arrow/datatype_extension_test.go b/go/arrow/datatype_extension_test.go index c542b22b769e6..9811800400506 100644 --- a/go/arrow/datatype_extension_test.go +++ b/go/arrow/datatype_extension_test.go @@ -20,8 +20,8 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go index fc0b3aea56e70..bcbc8ef6aec87 100644 --- a/go/arrow/datatype_fixedwidth.go +++ b/go/arrow/datatype_fixedwidth.go @@ -21,7 +21,7 @@ import ( "strconv" "time" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/internal/json" "golang.org/x/xerrors" ) diff --git a/go/arrow/datatype_fixedwidth_test.go b/go/arrow/datatype_fixedwidth_test.go index 1d3a07de09c1d..918572d40b8f4 100644 --- a/go/arrow/datatype_fixedwidth_test.go +++ b/go/arrow/datatype_fixedwidth_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/datatype_nested.go b/go/arrow/datatype_nested.go index 4ae4880334620..e381cd7047e45 100644 --- a/go/arrow/datatype_nested.go +++ b/go/arrow/datatype_nested.go @@ -22,7 +22,7 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) type ( diff --git a/go/arrow/datatype_null_test.go b/go/arrow/datatype_null_test.go index 286a72fb7fea3..57cddfadb8ef8 100644 --- a/go/arrow/datatype_null_test.go +++ b/go/arrow/datatype_null_test.go @@ -19,7 +19,7 @@ package arrow_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) func TestNullType(t *testing.T) { diff --git a/go/arrow/decimal128/decimal128.go b/go/arrow/decimal128/decimal128.go index ff9c088b9d638..7bde39d327417 100644 --- a/go/arrow/decimal128/decimal128.go +++ b/go/arrow/decimal128/decimal128.go @@ -23,7 +23,7 @@ import ( "math/big" "math/bits" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) const ( diff --git a/go/arrow/decimal128/decimal128_test.go b/go/arrow/decimal128/decimal128_test.go index 102eede6d3b28..879f2849028f8 100644 --- a/go/arrow/decimal128/decimal128_test.go +++ b/go/arrow/decimal128/decimal128_test.go @@ -22,7 +22,7 @@ import ( "math/big" "testing" - "github.com/apache/arrow/go/v14/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal128" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/decimal256/decimal256.go b/go/arrow/decimal256/decimal256.go index cf2157fdd48ed..5f2ad5f32165c 100644 --- a/go/arrow/decimal256/decimal256.go +++ b/go/arrow/decimal256/decimal256.go @@ -23,8 +23,8 @@ import ( "math/big" "math/bits" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) const ( diff --git a/go/arrow/decimal256/decimal256_test.go b/go/arrow/decimal256/decimal256_test.go index 9c734e23aa0d6..3be6a1944406f 100644 --- a/go/arrow/decimal256/decimal256_test.go +++ b/go/arrow/decimal256/decimal256_test.go @@ -23,7 +23,7 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/decimal256" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/doc.go b/go/arrow/doc.go index 87aea1d41035c..c8da330e4f8a1 100644 --- a/go/arrow/doc.go +++ b/go/arrow/doc.go @@ -36,7 +36,7 @@ To build with tinygo include the noasm build tag. */ package arrow -const PkgVersion = "14.0.1-SNAPSHOT" +const PkgVersion = "15.0.0-SNAPSHOT" //go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl //go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl diff --git a/go/arrow/encoded/ree_utils.go b/go/arrow/encoded/ree_utils.go index 1f71e7b5210cf..2ff4e7cadd38b 100644 --- a/go/arrow/encoded/ree_utils.go +++ b/go/arrow/encoded/ree_utils.go @@ -20,7 +20,7 @@ import ( "math" "sort" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) // FindPhysicalIndex performs a binary search on the run-ends to return diff --git a/go/arrow/encoded/ree_utils_test.go b/go/arrow/encoded/ree_utils_test.go index 785a3b105a47d..9470331002b80 100644 --- a/go/arrow/encoded/ree_utils_test.go +++ b/go/arrow/encoded/ree_utils_test.go @@ -21,10 +21,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/encoded" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/encoded" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/endian/endian.go b/go/arrow/endian/endian.go index 3ecda7b3600b7..d4865025b1fb0 100644 --- a/go/arrow/endian/endian.go +++ b/go/arrow/endian/endian.go @@ -17,8 +17,8 @@ package endian import ( - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" ) type Endianness flatbuf.Endianness diff --git a/go/arrow/example_test.go b/go/arrow/example_test.go index 10b3633f65536..b28cc9093739f 100644 --- a/go/arrow/example_test.go +++ b/go/arrow/example_test.go @@ -20,10 +20,10 @@ import ( "fmt" "log" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/tensor" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/tensor" ) // This example demonstrates how to build an array of int64 values using a builder and Append. diff --git a/go/arrow/flight/basic_auth_flight_test.go b/go/arrow/flight/basic_auth_flight_test.go index 0f63a400353d2..4097bf02edeff 100755 --- a/go/arrow/flight/basic_auth_flight_test.go +++ b/go/arrow/flight/basic_auth_flight_test.go @@ -22,7 +22,7 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/go/arrow/flight/client.go b/go/arrow/flight/client.go index c9a1417accab4..8c400eb66b64d 100644 --- a/go/arrow/flight/client.go +++ b/go/arrow/flight/client.go @@ -26,7 +26,7 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" diff --git a/go/arrow/flight/cookie_middleware_test.go b/go/arrow/flight/cookie_middleware_test.go index 2bde37fc0bb85..bdfe4fb32c1a5 100644 --- a/go/arrow/flight/cookie_middleware_test.go +++ b/go/arrow/flight/cookie_middleware_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/go/arrow/flight/example_flight_server_test.go b/go/arrow/flight/example_flight_server_test.go index 09f01361d13a0..d3a71b3b67ade 100755 --- a/go/arrow/flight/example_flight_server_test.go +++ b/go/arrow/flight/example_flight_server_test.go @@ -23,7 +23,7 @@ import ( "io" "log" - "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/go/arrow/flight/flight_middleware_test.go b/go/arrow/flight/flight_middleware_test.go index f2b240d1f2e45..e6faa1b7df1d2 100755 --- a/go/arrow/flight/flight_middleware_test.go +++ b/go/arrow/flight/flight_middleware_test.go @@ -23,8 +23,8 @@ import ( sync "sync" "testing" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/go/arrow/flight/flight_test.go b/go/arrow/flight/flight_test.go index 7cd724b76b8ce..cd37658603d5b 100755 --- a/go/arrow/flight/flight_test.go +++ b/go/arrow/flight/flight_test.go @@ -23,11 +23,11 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/go/arrow/flight/flightsql/client.go b/go/arrow/flight/flightsql/client.go index 13d046661e96d..c0c7e2cf20a28 100644 --- a/go/arrow/flight/flightsql/client.go +++ b/go/arrow/flight/flightsql/client.go @@ -22,12 +22,12 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" diff --git a/go/arrow/flight/flightsql/client_test.go b/go/arrow/flight/flightsql/client_test.go index bb4ae29477997..3efe4ba4049b8 100644 --- a/go/arrow/flight/flightsql/client_test.go +++ b/go/arrow/flight/flightsql/client_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "google.golang.org/grpc" diff --git a/go/arrow/flight/flightsql/column_metadata.go b/go/arrow/flight/flightsql/column_metadata.go index d9785892888b2..8e0a69b99d055 100644 --- a/go/arrow/flight/flightsql/column_metadata.go +++ b/go/arrow/flight/flightsql/column_metadata.go @@ -19,7 +19,7 @@ package flightsql import ( "strconv" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) const ( diff --git a/go/arrow/flight/flightsql/driver/README.md b/go/arrow/flight/flightsql/driver/README.md index b8850527c19c1..f1447a7d24256 100644 --- a/go/arrow/flight/flightsql/driver/README.md +++ b/go/arrow/flight/flightsql/driver/README.md @@ -36,7 +36,7 @@ connection pooling, transactions combined with ease of use (see (#usage)). ## Prerequisites * Go 1.17+ -* Installation via `go get -u github.com/apache/arrow/go/v14/arrow/flight/flightsql` +* Installation via `go get -u github.com/apache/arrow/go/v15/arrow/flight/flightsql` * Backend speaking FlightSQL --------------------------------------- @@ -55,7 +55,7 @@ import ( "database/sql" "time" - _ "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + _ "github.com/apache/arrow/go/v15/arrow/flight/flightsql" ) // Open the connection to an SQLite backend @@ -141,7 +141,7 @@ import ( "log" "time" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" ) func main() { diff --git a/go/arrow/flight/flightsql/driver/config_test.go b/go/arrow/flight/flightsql/driver/config_test.go index abcececeaf214..6b24b535585d9 100644 --- a/go/arrow/flight/flightsql/driver/config_test.go +++ b/go/arrow/flight/flightsql/driver/config_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/driver" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/driver" ) func TestConfigTLSRegistry(t *testing.T) { diff --git a/go/arrow/flight/flightsql/driver/driver.go b/go/arrow/flight/flightsql/driver/driver.go index b4cb0d2dbda2e..e325489236c6d 100644 --- a/go/arrow/flight/flightsql/driver/driver.go +++ b/go/arrow/flight/flightsql/driver/driver.go @@ -25,11 +25,11 @@ import ( "sort" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/memory" "google.golang.org/grpc" "google.golang.org/grpc/credentials" diff --git a/go/arrow/flight/flightsql/driver/driver_test.go b/go/arrow/flight/flightsql/driver/driver_test.go index 4449b390974a7..a388bf155ec99 100644 --- a/go/arrow/flight/flightsql/driver/driver_test.go +++ b/go/arrow/flight/flightsql/driver/driver_test.go @@ -33,13 +33,13 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/driver" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/example" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/driver" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v15/arrow/memory" ) const defaultTableName = "drivertest" diff --git a/go/arrow/flight/flightsql/driver/utils.go b/go/arrow/flight/flightsql/driver/utils.go index 99352e7251c53..022e9da4925f9 100644 --- a/go/arrow/flight/flightsql/driver/utils.go +++ b/go/arrow/flight/flightsql/driver/utils.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" ) // *** GRPC helpers *** diff --git a/go/arrow/flight/flightsql/driver/utils_test.go b/go/arrow/flight/flightsql/driver/utils_test.go index 355c888f56cc4..8f5ea031adf28 100644 --- a/go/arrow/flight/flightsql/driver/utils_test.go +++ b/go/arrow/flight/flightsql/driver/utils_test.go @@ -22,12 +22,12 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go b/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go index 6508833876fa5..1a50422f826f3 100644 --- a/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go +++ b/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go @@ -27,9 +27,9 @@ import ( "os" "strconv" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" ) func main() { diff --git a/go/arrow/flight/flightsql/example/sql_batch_reader.go b/go/arrow/flight/flightsql/example/sql_batch_reader.go index 7101b8f830a6f..36a0d7b424544 100644 --- a/go/arrow/flight/flightsql/example/sql_batch_reader.go +++ b/go/arrow/flight/flightsql/example/sql_batch_reader.go @@ -26,11 +26,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/wrapperspb" diff --git a/go/arrow/flight/flightsql/example/sqlite_info.go b/go/arrow/flight/flightsql/example/sqlite_info.go index bf29265853c65..c993af97f7449 100644 --- a/go/arrow/flight/flightsql/example/sqlite_info.go +++ b/go/arrow/flight/flightsql/example/sqlite_info.go @@ -20,8 +20,8 @@ package example import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" ) func SqlInfoResultMap() flightsql.SqlInfoResultMap { diff --git a/go/arrow/flight/flightsql/example/sqlite_server.go b/go/arrow/flight/flightsql/example/sqlite_server.go index 63b41ee5e0a4a..f06dd0210655f 100644 --- a/go/arrow/flight/flightsql/example/sqlite_server.go +++ b/go/arrow/flight/flightsql/example/sqlite_server.go @@ -45,13 +45,13 @@ import ( "strings" "sync" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" diff --git a/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go b/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go index 507abf622d890..e07a98fffdbf7 100644 --- a/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go +++ b/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" sqlite3 "modernc.org/sqlite/lib" ) diff --git a/go/arrow/flight/flightsql/example/type_info.go b/go/arrow/flight/flightsql/example/type_info.go index 6d6ceb339e4b0..0180fc1f23af5 100644 --- a/go/arrow/flight/flightsql/example/type_info.go +++ b/go/arrow/flight/flightsql/example/type_info.go @@ -22,10 +22,10 @@ package example import ( "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v15/arrow/memory" ) func GetTypeInfoResult(mem memory.Allocator) arrow.Record { diff --git a/go/arrow/flight/flightsql/schema_ref/reference_schemas.go b/go/arrow/flight/flightsql/schema_ref/reference_schemas.go index 98d61f8110d45..b65688306473b 100644 --- a/go/arrow/flight/flightsql/schema_ref/reference_schemas.go +++ b/go/arrow/flight/flightsql/schema_ref/reference_schemas.go @@ -18,7 +18,7 @@ // by FlightSQL servers and clients. package schema_ref -import "github.com/apache/arrow/go/v14/arrow" +import "github.com/apache/arrow/go/v15/arrow" var ( Catalogs = arrow.NewSchema( diff --git a/go/arrow/flight/flightsql/server.go b/go/arrow/flight/flightsql/server.go index e9cbc177da2c4..a086610433eae 100644 --- a/go/arrow/flight/flightsql/server.go +++ b/go/arrow/flight/flightsql/server.go @@ -20,14 +20,14 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" - pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" + pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" diff --git a/go/arrow/flight/flightsql/server_test.go b/go/arrow/flight/flightsql/server_test.go index 33e9b8236a261..e444da4aaf4a2 100644 --- a/go/arrow/flight/flightsql/server_test.go +++ b/go/arrow/flight/flightsql/server_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/go/arrow/flight/flightsql/sql_info.go b/go/arrow/flight/flightsql/sql_info.go index a12c4fbdaeda9..5f78e1facd581 100644 --- a/go/arrow/flight/flightsql/sql_info.go +++ b/go/arrow/flight/flightsql/sql_info.go @@ -17,8 +17,8 @@ package flightsql import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" ) const ( diff --git a/go/arrow/flight/flightsql/sqlite_server_test.go b/go/arrow/flight/flightsql/sqlite_server_test.go index 03ae3d7a8313a..e6fa798c55cf1 100644 --- a/go/arrow/flight/flightsql/sqlite_server_test.go +++ b/go/arrow/flight/flightsql/sqlite_server_test.go @@ -26,14 +26,14 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/example" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "google.golang.org/grpc/codes" diff --git a/go/arrow/flight/flightsql/types.go b/go/arrow/flight/flightsql/types.go index 98086a61fe067..2b7419482e3e3 100644 --- a/go/arrow/flight/flightsql/types.go +++ b/go/arrow/flight/flightsql/types.go @@ -17,7 +17,7 @@ package flightsql import ( - pb "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" + pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) diff --git a/go/arrow/flight/record_batch_reader.go b/go/arrow/flight/record_batch_reader.go index b726180627e6f..f137c75172368 100644 --- a/go/arrow/flight/record_batch_reader.go +++ b/go/arrow/flight/record_batch_reader.go @@ -21,12 +21,12 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) // DataStreamReader is an interface for receiving flight data messages on a stream diff --git a/go/arrow/flight/record_batch_writer.go b/go/arrow/flight/record_batch_writer.go index 3ae17bcf48042..426dca6c3e907 100644 --- a/go/arrow/flight/record_batch_writer.go +++ b/go/arrow/flight/record_batch_writer.go @@ -19,9 +19,9 @@ package flight import ( "bytes" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) // DataStreamWriter is an interface that represents an Arrow Flight stream diff --git a/go/arrow/flight/server.go b/go/arrow/flight/server.go index cf471f9b3a2b6..8676b15644e47 100644 --- a/go/arrow/flight/server.go +++ b/go/arrow/flight/server.go @@ -22,7 +22,7 @@ import ( "os" "os/signal" - "github.com/apache/arrow/go/v14/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" "google.golang.org/grpc" ) diff --git a/go/arrow/flight/server_example_test.go b/go/arrow/flight/server_example_test.go index b82e2cb7f7164..4e2e4f4c6b587 100644 --- a/go/arrow/flight/server_example_test.go +++ b/go/arrow/flight/server_example_test.go @@ -21,7 +21,7 @@ import ( "fmt" "net" - "github.com/apache/arrow/go/v14/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/health" diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go index 0851bff0fe0da..6631e4245c19d 100644 --- a/go/arrow/internal/arrdata/arrdata.go +++ b/go/arrow/internal/arrdata/arrdata.go @@ -21,14 +21,14 @@ import ( "fmt" "sort" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" ) var ( diff --git a/go/arrow/internal/arrdata/ioutil.go b/go/arrow/internal/arrdata/ioutil.go index 1a2ce0e5b5eca..a6becc2151ef3 100644 --- a/go/arrow/internal/arrdata/ioutil.go +++ b/go/arrow/internal/arrdata/ioutil.go @@ -23,11 +23,11 @@ import ( "sync" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) // CheckArrowFile checks whether a given ARROW file contains the expected list of records. diff --git a/go/arrow/internal/arrjson/arrjson.go b/go/arrow/internal/arrjson/arrjson.go index ad87b73fc4ddb..87bdc1f44d875 100644 --- a/go/arrow/internal/arrjson/arrjson.go +++ b/go/arrow/internal/arrjson/arrjson.go @@ -26,16 +26,16 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" ) type Schema struct { diff --git a/go/arrow/internal/arrjson/arrjson_test.go b/go/arrow/internal/arrjson/arrjson_test.go index ee85d431805ab..7beadee370edb 100644 --- a/go/arrow/internal/arrjson/arrjson_test.go +++ b/go/arrow/internal/arrjson/arrjson_test.go @@ -22,9 +22,9 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/internal/arrjson/option.go b/go/arrow/internal/arrjson/option.go index 7ae2058d471a7..47d40f857c54e 100644 --- a/go/arrow/internal/arrjson/option.go +++ b/go/arrow/internal/arrjson/option.go @@ -17,8 +17,8 @@ package arrjson import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" ) type config struct { diff --git a/go/arrow/internal/arrjson/reader.go b/go/arrow/internal/arrjson/reader.go index c8056ef1dc744..226fa1b1919f9 100644 --- a/go/arrow/internal/arrjson/reader.go +++ b/go/arrow/internal/arrjson/reader.go @@ -20,11 +20,11 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/internal/json" ) type Reader struct { diff --git a/go/arrow/internal/arrjson/writer.go b/go/arrow/internal/arrjson/writer.go index ddac32609b604..af7032d581f4d 100644 --- a/go/arrow/internal/arrjson/writer.go +++ b/go/arrow/internal/arrjson/writer.go @@ -20,11 +20,11 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/internal/json" ) const ( diff --git a/go/arrow/internal/cdata_integration/entrypoints.go b/go/arrow/internal/cdata_integration/entrypoints.go index 629b8a762a689..a40db8316f848 100644 --- a/go/arrow/internal/cdata_integration/entrypoints.go +++ b/go/arrow/internal/cdata_integration/entrypoints.go @@ -25,10 +25,10 @@ import ( "runtime" "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/cdata" - "github.com/apache/arrow/go/v14/arrow/internal/arrjson" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/cdata" + "github.com/apache/arrow/go/v15/arrow/internal/arrjson" + "github.com/apache/arrow/go/v15/arrow/memory" ) // #include diff --git a/go/arrow/internal/dictutils/dict.go b/go/arrow/internal/dictutils/dict.go index e09a2f4a0ecd1..d31369f7d25b3 100644 --- a/go/arrow/internal/dictutils/dict.go +++ b/go/arrow/internal/dictutils/dict.go @@ -21,9 +21,9 @@ import ( "fmt" "hash/maphash" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" ) type Kind int8 diff --git a/go/arrow/internal/dictutils/dict_test.go b/go/arrow/internal/dictutils/dict_test.go index 6cafacbd1df00..9d2f7ae4782c7 100644 --- a/go/arrow/internal/dictutils/dict_test.go +++ b/go/arrow/internal/dictutils/dict_test.go @@ -20,10 +20,10 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestDictMemo(t *testing.T) { diff --git a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go index 8da39858fabab..1afb53c5294f4 100755 --- a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go +++ b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - "github.com/apache/arrow/go/v14/arrow/internal/flight_integration" + "github.com/apache/arrow/go/v15/arrow/internal/flight_integration" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) diff --git a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go index 9cebb2c695ece..d382ff9e88670 100644 --- a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go +++ b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go @@ -23,7 +23,7 @@ import ( "os" "syscall" - "github.com/apache/arrow/go/v14/arrow/internal/flight_integration" + "github.com/apache/arrow/go/v15/arrow/internal/flight_integration" ) var ( diff --git a/go/arrow/internal/flight_integration/scenario.go b/go/arrow/internal/flight_integration/scenario.go index 4f47d7fd506ee..3ec905e2d659c 100644 --- a/go/arrow/internal/flight_integration/scenario.go +++ b/go/arrow/internal/flight_integration/scenario.go @@ -31,15 +31,15 @@ import ( "strings" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v14/arrow/internal/arrjson" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v15/arrow/internal/arrjson" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" "golang.org/x/xerrors" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/go/arrow/internal/testing/gen/random_array_gen.go b/go/arrow/internal/testing/gen/random_array_gen.go index 41f2578209a7f..b42273ff93fac 100644 --- a/go/arrow/internal/testing/gen/random_array_gen.go +++ b/go/arrow/internal/testing/gen/random_array_gen.go @@ -19,11 +19,11 @@ package gen import ( "math" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/exp/rand" "gonum.org/v1/gonum/stat/distuv" ) diff --git a/go/arrow/internal/testing/tools/bits_test.go b/go/arrow/internal/testing/tools/bits_test.go index 7d0fccb5f999e..6ad8ac5b12b68 100644 --- a/go/arrow/internal/testing/tools/bits_test.go +++ b/go/arrow/internal/testing/tools/bits_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/internal/testing/tools/data_types.go b/go/arrow/internal/testing/tools/data_types.go index 654780bc3e853..161cfa431491d 100644 --- a/go/arrow/internal/testing/tools/data_types.go +++ b/go/arrow/internal/testing/tools/data_types.go @@ -21,8 +21,8 @@ package tools import ( "reflect" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/float16" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/internal/utils.go b/go/arrow/internal/utils.go index 265f030dfd19a..619eebd97dc78 100644 --- a/go/arrow/internal/utils.go +++ b/go/arrow/internal/utils.go @@ -17,8 +17,8 @@ package internal import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" ) const CurMetadataVersion = flatbuf.MetadataVersionV5 diff --git a/go/arrow/ipc/cmd/arrow-cat/main.go b/go/arrow/ipc/cmd/arrow-cat/main.go index af53c9b4e5f44..0251b08c09bff 100644 --- a/go/arrow/ipc/cmd/arrow-cat/main.go +++ b/go/arrow/ipc/cmd/arrow-cat/main.go @@ -63,8 +63,8 @@ import ( "log" "os" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-cat/main_test.go b/go/arrow/ipc/cmd/arrow-cat/main_test.go index 22e4f6ffe0778..b6528b85b416d 100644 --- a/go/arrow/ipc/cmd/arrow-cat/main_test.go +++ b/go/arrow/ipc/cmd/arrow-cat/main_test.go @@ -23,10 +23,10 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestCatStream(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-file-to-stream/main.go b/go/arrow/ipc/cmd/arrow-file-to-stream/main.go index 7e86263d6d5b8..6187b94465211 100644 --- a/go/arrow/ipc/cmd/arrow-file-to-stream/main.go +++ b/go/arrow/ipc/cmd/arrow-file-to-stream/main.go @@ -24,9 +24,9 @@ import ( "log" "os" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go b/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go index 30db24715386f..2f66a8b48c59d 100644 --- a/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go +++ b/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go @@ -21,8 +21,8 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestFileToStream(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-json-integration-test/main.go b/go/arrow/ipc/cmd/arrow-json-integration-test/main.go index 91ba6a4ad9b61..7db2fa1d1c5db 100644 --- a/go/arrow/ipc/cmd/arrow-json-integration-test/main.go +++ b/go/arrow/ipc/cmd/arrow-json-integration-test/main.go @@ -22,12 +22,12 @@ import ( "log" "os" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/internal/arrjson" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/internal/arrjson" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/internal/types" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go b/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go index 140f9f62478c0..eb702f0aa2bc5 100644 --- a/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go +++ b/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go @@ -20,8 +20,8 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestIntegration(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-ls/main.go b/go/arrow/ipc/cmd/arrow-ls/main.go index f739c9ae7da8e..4230ae2449985 100644 --- a/go/arrow/ipc/cmd/arrow-ls/main.go +++ b/go/arrow/ipc/cmd/arrow-ls/main.go @@ -61,8 +61,8 @@ import ( "log" "os" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-ls/main_test.go b/go/arrow/ipc/cmd/arrow-ls/main_test.go index 2c038dae97b52..bda1c2dc4d3dc 100644 --- a/go/arrow/ipc/cmd/arrow-ls/main_test.go +++ b/go/arrow/ipc/cmd/arrow-ls/main_test.go @@ -23,10 +23,10 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestLsStream(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-stream-to-file/main.go b/go/arrow/ipc/cmd/arrow-stream-to-file/main.go index 758231ea21cf5..7ed3f6a281d9f 100644 --- a/go/arrow/ipc/cmd/arrow-stream-to-file/main.go +++ b/go/arrow/ipc/cmd/arrow-stream-to-file/main.go @@ -24,9 +24,9 @@ import ( "log" "os" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go b/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go index 4354b46e8964b..73196551054ac 100644 --- a/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go +++ b/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go @@ -21,8 +21,8 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestStreamToFile(t *testing.T) { diff --git a/go/arrow/ipc/compression.go b/go/arrow/ipc/compression.go index 73fb916500804..8856b732f9c5d 100644 --- a/go/arrow/ipc/compression.go +++ b/go/arrow/ipc/compression.go @@ -19,9 +19,9 @@ package ipc import ( "io" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/klauspost/compress/zstd" "github.com/pierrec/lz4/v4" ) diff --git a/go/arrow/ipc/endian_swap.go b/go/arrow/ipc/endian_swap.go index d98fec1089f04..d2e0948434abc 100644 --- a/go/arrow/ipc/endian_swap.go +++ b/go/arrow/ipc/endian_swap.go @@ -20,9 +20,9 @@ import ( "errors" "math/bits" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" ) // swap the endianness of the array's buffers as needed in-place to save diff --git a/go/arrow/ipc/endian_swap_test.go b/go/arrow/ipc/endian_swap_test.go index 4f02bb53d61a4..1c724103f4441 100644 --- a/go/arrow/ipc/endian_swap_test.go +++ b/go/arrow/ipc/endian_swap_test.go @@ -20,11 +20,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go index 10cb2cae764e6..330355d3a60c3 100644 --- a/go/arrow/ipc/file_reader.go +++ b/go/arrow/ipc/file_reader.go @@ -23,14 +23,14 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/internal" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/internal" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" ) // FileReader is an Arrow file reader. diff --git a/go/arrow/ipc/file_test.go b/go/arrow/ipc/file_test.go index 1b22719dc8097..5f4dac1f899bb 100644 --- a/go/arrow/ipc/file_test.go +++ b/go/arrow/ipc/file_test.go @@ -21,9 +21,9 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestFile(t *testing.T) { diff --git a/go/arrow/ipc/file_writer.go b/go/arrow/ipc/file_writer.go index 12384225b70a1..55e4d7c2dc22c 100644 --- a/go/arrow/ipc/file_writer.go +++ b/go/arrow/ipc/file_writer.go @@ -21,11 +21,11 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" ) // PayloadWriter is an interface for injecting a different payloadwriter diff --git a/go/arrow/ipc/ipc.go b/go/arrow/ipc/ipc.go index 6c04b6f5ad2d6..8cf56e8331fcb 100644 --- a/go/arrow/ipc/ipc.go +++ b/go/arrow/ipc/ipc.go @@ -19,10 +19,10 @@ package ipc import ( "io" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" ) const ( diff --git a/go/arrow/ipc/ipc_test.go b/go/arrow/ipc/ipc_test.go index 551d325dc1212..d02d8734b56d4 100644 --- a/go/arrow/ipc/ipc_test.go +++ b/go/arrow/ipc/ipc_test.go @@ -29,10 +29,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestArrow12072(t *testing.T) { diff --git a/go/arrow/ipc/message.go b/go/arrow/ipc/message.go index c5d0ec68d19c4..709aa5aa2dba4 100644 --- a/go/arrow/ipc/message.go +++ b/go/arrow/ipc/message.go @@ -22,9 +22,9 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" ) // MetadataVersion represents the Arrow metadata version. diff --git a/go/arrow/ipc/message_test.go b/go/arrow/ipc/message_test.go index cc41fdd3fe13d..912d112229f0c 100644 --- a/go/arrow/ipc/message_test.go +++ b/go/arrow/ipc/message_test.go @@ -22,9 +22,9 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestMessageReaderBodyInAllocator(t *testing.T) { diff --git a/go/arrow/ipc/metadata.go b/go/arrow/ipc/metadata.go index 9bab47d6fa0cd..bd437834c3d06 100644 --- a/go/arrow/ipc/metadata.go +++ b/go/arrow/ipc/metadata.go @@ -23,11 +23,11 @@ import ( "io" "sort" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" flatbuffers "github.com/google/flatbuffers/go" ) diff --git a/go/arrow/ipc/metadata_test.go b/go/arrow/ipc/metadata_test.go index 3038abdf5f791..a35068813ba56 100644 --- a/go/arrow/ipc/metadata_test.go +++ b/go/arrow/ipc/metadata_test.go @@ -21,12 +21,12 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" flatbuffers "github.com/google/flatbuffers/go" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/ipc/reader.go b/go/arrow/ipc/reader.go index 1f684c1f6c034..826062fafdb46 100644 --- a/go/arrow/ipc/reader.go +++ b/go/arrow/ipc/reader.go @@ -23,13 +23,13 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" ) // Reader reads records from an io.Reader. diff --git a/go/arrow/ipc/reader_test.go b/go/arrow/ipc/reader_test.go index 229374e94ec74..f00f3bb3da476 100644 --- a/go/arrow/ipc/reader_test.go +++ b/go/arrow/ipc/reader_test.go @@ -20,9 +20,9 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/ipc/stream_test.go b/go/arrow/ipc/stream_test.go index d8bbd8f04b14d..201bfb82a5677 100644 --- a/go/arrow/ipc/stream_test.go +++ b/go/arrow/ipc/stream_test.go @@ -22,9 +22,9 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v14/arrow/internal/arrdata" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" ) func TestStream(t *testing.T) { diff --git a/go/arrow/ipc/writer.go b/go/arrow/ipc/writer.go index a97f47ef4aa43..58c56d2d16ccf 100644 --- a/go/arrow/ipc/writer.go +++ b/go/arrow/ipc/writer.go @@ -26,14 +26,14 @@ import ( "sync" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/internal/dictutils" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/dictutils" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" ) type swriter struct { diff --git a/go/arrow/ipc/writer_test.go b/go/arrow/ipc/writer_test.go index da461c3d52272..ea7592554c888 100644 --- a/go/arrow/ipc/writer_test.go +++ b/go/arrow/ipc/writer_test.go @@ -24,11 +24,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/math/float64.go b/go/arrow/math/float64.go index 81de594df6ce6..8d72ecf1b2668 100644 --- a/go/arrow/math/float64.go +++ b/go/arrow/math/float64.go @@ -19,7 +19,7 @@ package math import ( - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) type Float64Funcs struct { diff --git a/go/arrow/math/float64_avx2_amd64.go b/go/arrow/math/float64_avx2_amd64.go index 8a131a2bc8fc5..e78d4affe9f6f 100644 --- a/go/arrow/math/float64_avx2_amd64.go +++ b/go/arrow/math/float64_avx2_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) //go:noescape diff --git a/go/arrow/math/float64_neon_arm64.go b/go/arrow/math/float64_neon_arm64.go index ad9b6435b26fc..a5013960d9ec7 100755 --- a/go/arrow/math/float64_neon_arm64.go +++ b/go/arrow/math/float64_neon_arm64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) //go:noescape diff --git a/go/arrow/math/float64_sse4_amd64.go b/go/arrow/math/float64_sse4_amd64.go index a2be5de43f5bb..cdd88f91c2c2b 100644 --- a/go/arrow/math/float64_sse4_amd64.go +++ b/go/arrow/math/float64_sse4_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) //go:noescape diff --git a/go/arrow/math/float64_test.go b/go/arrow/math/float64_test.go index b10214f6d892f..3a7c247c5df8c 100644 --- a/go/arrow/math/float64_test.go +++ b/go/arrow/math/float64_test.go @@ -21,9 +21,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/math" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/math" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/math/int64.go b/go/arrow/math/int64.go index 2adff970c2233..b8236dfbb6552 100644 --- a/go/arrow/math/int64.go +++ b/go/arrow/math/int64.go @@ -19,7 +19,7 @@ package math import ( - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) type Int64Funcs struct { diff --git a/go/arrow/math/int64_avx2_amd64.go b/go/arrow/math/int64_avx2_amd64.go index ac0b2e1ff662a..5c16e790c1e55 100644 --- a/go/arrow/math/int64_avx2_amd64.go +++ b/go/arrow/math/int64_avx2_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) //go:noescape diff --git a/go/arrow/math/int64_neon_arm64.go b/go/arrow/math/int64_neon_arm64.go index 4ab5fb771a526..00ec48b6e4735 100755 --- a/go/arrow/math/int64_neon_arm64.go +++ b/go/arrow/math/int64_neon_arm64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) //go:noescape diff --git a/go/arrow/math/int64_sse4_amd64.go b/go/arrow/math/int64_sse4_amd64.go index 85304210eb448..a894bad2ef6a3 100644 --- a/go/arrow/math/int64_sse4_amd64.go +++ b/go/arrow/math/int64_sse4_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) //go:noescape diff --git a/go/arrow/math/int64_test.go b/go/arrow/math/int64_test.go index 06b42ffe9351b..3781fd3570785 100644 --- a/go/arrow/math/int64_test.go +++ b/go/arrow/math/int64_test.go @@ -21,9 +21,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/math" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/math" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/math/type.go.tmpl b/go/arrow/math/type.go.tmpl index 7ae142c46f12b..f1a4dfa1362f8 100644 --- a/go/arrow/math/type.go.tmpl +++ b/go/arrow/math/type.go.tmpl @@ -17,7 +17,7 @@ package math import ( - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) {{$def := .D}} diff --git a/go/arrow/math/type_simd_amd64.go.tmpl b/go/arrow/math/type_simd_amd64.go.tmpl index 72d40ea386a8f..77dee758b05cd 100644 --- a/go/arrow/math/type_simd_amd64.go.tmpl +++ b/go/arrow/math/type_simd_amd64.go.tmpl @@ -21,7 +21,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) {{$name := printf "%s_%s" .In.Type .D.arch}} diff --git a/go/arrow/math/type_simd_arm64.go.tmpl b/go/arrow/math/type_simd_arm64.go.tmpl index 72d40ea386a8f..77dee758b05cd 100755 --- a/go/arrow/math/type_simd_arm64.go.tmpl +++ b/go/arrow/math/type_simd_arm64.go.tmpl @@ -21,7 +21,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) {{$name := printf "%s_%s" .In.Type .D.arch}} diff --git a/go/arrow/math/type_test.go.tmpl b/go/arrow/math/type_test.go.tmpl index 160fc34e4d4d4..969377e454e52 100644 --- a/go/arrow/math/type_test.go.tmpl +++ b/go/arrow/math/type_test.go.tmpl @@ -19,9 +19,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/math" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/math" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/math/uint64.go b/go/arrow/math/uint64.go index 64ac63a3d1853..b9a70360224ea 100644 --- a/go/arrow/math/uint64.go +++ b/go/arrow/math/uint64.go @@ -19,7 +19,7 @@ package math import ( - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) type Uint64Funcs struct { diff --git a/go/arrow/math/uint64_avx2_amd64.go b/go/arrow/math/uint64_avx2_amd64.go index 2a35bebc980b3..c1f48efefd8eb 100644 --- a/go/arrow/math/uint64_avx2_amd64.go +++ b/go/arrow/math/uint64_avx2_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) //go:noescape diff --git a/go/arrow/math/uint64_neon_arm64.go b/go/arrow/math/uint64_neon_arm64.go index f6588694bb1fd..37ca4ecad04e7 100755 --- a/go/arrow/math/uint64_neon_arm64.go +++ b/go/arrow/math/uint64_neon_arm64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) //go:noescape diff --git a/go/arrow/math/uint64_sse4_amd64.go b/go/arrow/math/uint64_sse4_amd64.go index 8c6621464f118..7163c1de171de 100644 --- a/go/arrow/math/uint64_sse4_amd64.go +++ b/go/arrow/math/uint64_sse4_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow/array" ) //go:noescape diff --git a/go/arrow/math/uint64_test.go b/go/arrow/math/uint64_test.go index aac768e599cf8..e8ba42b59aa10 100644 --- a/go/arrow/math/uint64_test.go +++ b/go/arrow/math/uint64_test.go @@ -21,9 +21,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/math" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/math" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/buffer.go b/go/arrow/memory/buffer.go index 5a2b4297031f0..ffdb41e3dbe9e 100644 --- a/go/arrow/memory/buffer.go +++ b/go/arrow/memory/buffer.go @@ -19,7 +19,7 @@ package memory import ( "sync/atomic" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) // Buffer is a wrapper type for a buffer of bytes. diff --git a/go/arrow/memory/buffer_test.go b/go/arrow/memory/buffer_test.go index 25a0b39848519..92bb071d0abde 100644 --- a/go/arrow/memory/buffer_test.go +++ b/go/arrow/memory/buffer_test.go @@ -19,7 +19,7 @@ package memory_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/cgo_allocator.go b/go/arrow/memory/cgo_allocator.go index 85ee445216f6a..af25d1899a6fe 100644 --- a/go/arrow/memory/cgo_allocator.go +++ b/go/arrow/memory/cgo_allocator.go @@ -22,7 +22,7 @@ package memory import ( "runtime" - cga "github.com/apache/arrow/go/v14/arrow/memory/internal/cgoalloc" + cga "github.com/apache/arrow/go/v15/arrow/memory/internal/cgoalloc" ) // CgoArrowAllocator is an allocator which exposes the C++ memory pool class diff --git a/go/arrow/memory/default_mallocator.go b/go/arrow/memory/default_mallocator.go index 12ad0846677b1..4a9ef942fd08d 100644 --- a/go/arrow/memory/default_mallocator.go +++ b/go/arrow/memory/default_mallocator.go @@ -19,7 +19,7 @@ package memory import ( - "github.com/apache/arrow/go/v14/arrow/memory/mallocator" + "github.com/apache/arrow/go/v15/arrow/memory/mallocator" ) // DefaultAllocator is a default implementation of Allocator and can be used anywhere diff --git a/go/arrow/memory/default_mallocator_test.go b/go/arrow/memory/default_mallocator_test.go index b209ced68a067..5a38e8b4e843c 100644 --- a/go/arrow/memory/default_mallocator_test.go +++ b/go/arrow/memory/default_mallocator_test.go @@ -21,8 +21,8 @@ package memory_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/memory/mallocator" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory/mallocator" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/mallocator/mallocator_test.go b/go/arrow/memory/mallocator/mallocator_test.go index 5b09355033f70..91b74383ed494 100644 --- a/go/arrow/memory/mallocator/mallocator_test.go +++ b/go/arrow/memory/mallocator/mallocator_test.go @@ -23,7 +23,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v14/arrow/memory/mallocator" + "github.com/apache/arrow/go/v15/arrow/memory/mallocator" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/memory_test.go b/go/arrow/memory/memory_test.go index dff2e0bfd0712..adaa8359369be 100644 --- a/go/arrow/memory/memory_test.go +++ b/go/arrow/memory/memory_test.go @@ -19,7 +19,7 @@ package memory_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/record.go b/go/arrow/record.go index d98c7732ef798..885af38034938 100644 --- a/go/arrow/record.go +++ b/go/arrow/record.go @@ -16,7 +16,7 @@ package arrow -import "github.com/apache/arrow/go/v14/internal/json" +import "github.com/apache/arrow/go/v15/internal/json" // Record is a collection of equal-length arrays matching a particular Schema. // Also known as a RecordBatch in the spec and in some implementations. diff --git a/go/arrow/scalar/append.go b/go/arrow/scalar/append.go index 9bcfe3e22dc63..fe8dd32e9eae5 100644 --- a/go/arrow/scalar/append.go +++ b/go/arrow/scalar/append.go @@ -21,11 +21,11 @@ package scalar import ( "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/scalar/append_test.go b/go/arrow/scalar/append_test.go index 497a6a2d02933..e509bca1fb7b0 100644 --- a/go/arrow/scalar/append_test.go +++ b/go/arrow/scalar/append_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/scalar/binary.go b/go/arrow/scalar/binary.go index 3c041210fde9c..b6abe9cba7b1c 100644 --- a/go/arrow/scalar/binary.go +++ b/go/arrow/scalar/binary.go @@ -21,8 +21,8 @@ import ( "fmt" "unicode/utf8" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" ) type BinaryScalar interface { diff --git a/go/arrow/scalar/compare.go b/go/arrow/scalar/compare.go index be7fa4d01cc19..b4a3fe08d26fe 100644 --- a/go/arrow/scalar/compare.go +++ b/go/arrow/scalar/compare.go @@ -16,7 +16,7 @@ package scalar -import "github.com/apache/arrow/go/v14/arrow" +import "github.com/apache/arrow/go/v15/arrow" // Equals returns true if two scalars are equal, which means they have the same // datatype, validity and value. diff --git a/go/arrow/scalar/nested.go b/go/arrow/scalar/nested.go index 87e84210f58a1..8250beb5ed90d 100644 --- a/go/arrow/scalar/nested.go +++ b/go/arrow/scalar/nested.go @@ -21,10 +21,10 @@ import ( "errors" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/scalar/numeric.gen.go b/go/arrow/scalar/numeric.gen.go index 90ae2a7004ce3..da088162f4017 100644 --- a/go/arrow/scalar/numeric.gen.go +++ b/go/arrow/scalar/numeric.gen.go @@ -24,9 +24,9 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" ) type Int8 struct { diff --git a/go/arrow/scalar/numeric.gen_test.go b/go/arrow/scalar/numeric.gen_test.go index 987cd66c75f3c..807857c1c98c2 100644 --- a/go/arrow/scalar/numeric.gen_test.go +++ b/go/arrow/scalar/numeric.gen_test.go @@ -21,8 +21,8 @@ package scalar_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/scalar/numeric.gen_test.go.tmpl b/go/arrow/scalar/numeric.gen_test.go.tmpl index c975cc9db6e1c..7f45898a20d29 100644 --- a/go/arrow/scalar/numeric.gen_test.go.tmpl +++ b/go/arrow/scalar/numeric.gen_test.go.tmpl @@ -19,8 +19,8 @@ package scalar_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/scalar/parse.go b/go/arrow/scalar/parse.go index fcffe1ea6adf8..5002f98a65c42 100644 --- a/go/arrow/scalar/parse.go +++ b/go/arrow/scalar/parse.go @@ -25,12 +25,12 @@ import ( "strings" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/memory" ) type TypeToScalar interface { diff --git a/go/arrow/scalar/scalar.go b/go/arrow/scalar/scalar.go index 395771fa89c94..8b0d3ace2ad78 100644 --- a/go/arrow/scalar/scalar.go +++ b/go/arrow/scalar/scalar.go @@ -26,16 +26,16 @@ import ( "strconv" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/encoded" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/float16" - "github.com/apache/arrow/go/v14/arrow/internal/debug" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/encoded" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/scalar/scalar_test.go b/go/arrow/scalar/scalar_test.go index 405571a14880d..ce8170301b0a2 100644 --- a/go/arrow/scalar/scalar_test.go +++ b/go/arrow/scalar/scalar_test.go @@ -25,12 +25,12 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/scalar" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/scalar/temporal.go b/go/arrow/scalar/temporal.go index 880416f7dd5a3..ee13c84429e8e 100644 --- a/go/arrow/scalar/temporal.go +++ b/go/arrow/scalar/temporal.go @@ -22,7 +22,7 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) func temporalToString(s TemporalScalar) string { diff --git a/go/arrow/schema.go b/go/arrow/schema.go index a7fa434131925..e84f350a53637 100644 --- a/go/arrow/schema.go +++ b/go/arrow/schema.go @@ -21,7 +21,7 @@ import ( "sort" "strings" - "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/endian" ) type Metadata struct { diff --git a/go/arrow/schema_test.go b/go/arrow/schema_test.go index 5e79723f90352..fddf1d7f131ec 100644 --- a/go/arrow/schema_test.go +++ b/go/arrow/schema_test.go @@ -21,7 +21,7 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/endian" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/table.go b/go/arrow/table.go index 5a68085f8df93..82dc283706b65 100644 --- a/go/arrow/table.go +++ b/go/arrow/table.go @@ -20,7 +20,7 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) // Table represents a logical sequence of chunked arrays of equal length. It is diff --git a/go/arrow/tensor/numeric.gen.go b/go/arrow/tensor/numeric.gen.go index 5b7bbde150fe2..d207f0bfa2c9c 100644 --- a/go/arrow/tensor/numeric.gen.go +++ b/go/arrow/tensor/numeric.gen.go @@ -19,7 +19,7 @@ package tensor import ( - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) // Int8 is an n-dim array of int8s. diff --git a/go/arrow/tensor/numeric.gen.go.tmpl b/go/arrow/tensor/numeric.gen.go.tmpl index c693082192247..e03f986da3f8f 100644 --- a/go/arrow/tensor/numeric.gen.go.tmpl +++ b/go/arrow/tensor/numeric.gen.go.tmpl @@ -17,8 +17,8 @@ package tensor import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" ) {{range .In}} diff --git a/go/arrow/tensor/numeric.gen_test.go b/go/arrow/tensor/numeric.gen_test.go index 00922a84b6b03..3a7c3570c216b 100644 --- a/go/arrow/tensor/numeric.gen_test.go +++ b/go/arrow/tensor/numeric.gen_test.go @@ -23,10 +23,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/tensor" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/tensor" ) func TestTensorInt8(t *testing.T) { diff --git a/go/arrow/tensor/numeric.gen_test.go.tmpl b/go/arrow/tensor/numeric.gen_test.go.tmpl index 665bb8dc68da9..34f01b40d8084 100644 --- a/go/arrow/tensor/numeric.gen_test.go.tmpl +++ b/go/arrow/tensor/numeric.gen_test.go.tmpl @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/tensor" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/tensor" ) {{range .In}} diff --git a/go/arrow/tensor/tensor.go b/go/arrow/tensor/tensor.go index 27c380345a498..1f2ed7e82141b 100644 --- a/go/arrow/tensor/tensor.go +++ b/go/arrow/tensor/tensor.go @@ -21,8 +21,8 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) // Interface represents an n-dimensional array of numerical data. diff --git a/go/arrow/tensor/tensor_test.go b/go/arrow/tensor/tensor_test.go index 4cfe69f780421..552c4c1c223e0 100644 --- a/go/arrow/tensor/tensor_test.go +++ b/go/arrow/tensor/tensor_test.go @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/tensor" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/tensor" ) func TestTensor(t *testing.T) { diff --git a/go/arrow/type_traits_boolean.go b/go/arrow/type_traits_boolean.go index 6a46bdec702ff..c164d45954fc0 100644 --- a/go/arrow/type_traits_boolean.go +++ b/go/arrow/type_traits_boolean.go @@ -17,7 +17,7 @@ package arrow import ( - "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/bitutil" ) type booleanTraits struct{} diff --git a/go/arrow/type_traits_decimal128.go b/go/arrow/type_traits_decimal128.go index d2d3aae371663..f573ad3c65a4c 100644 --- a/go/arrow/type_traits_decimal128.go +++ b/go/arrow/type_traits_decimal128.go @@ -20,8 +20,8 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/endian" ) // Decimal128 traits diff --git a/go/arrow/type_traits_decimal256.go b/go/arrow/type_traits_decimal256.go index 256ed68fffc8d..adf3cc3e0bc31 100644 --- a/go/arrow/type_traits_decimal256.go +++ b/go/arrow/type_traits_decimal256.go @@ -20,8 +20,8 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/endian" ) // Decimal256 traits diff --git a/go/arrow/type_traits_float16.go b/go/arrow/type_traits_float16.go index c40363d3785da..e59efd4c248d8 100644 --- a/go/arrow/type_traits_float16.go +++ b/go/arrow/type_traits_float16.go @@ -20,8 +20,8 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/float16" ) // Float16 traits diff --git a/go/arrow/type_traits_interval.go b/go/arrow/type_traits_interval.go index 35e6057090f10..5fbd7a5248918 100644 --- a/go/arrow/type_traits_interval.go +++ b/go/arrow/type_traits_interval.go @@ -20,8 +20,8 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/internal/debug" ) var ( diff --git a/go/arrow/type_traits_numeric.gen.go b/go/arrow/type_traits_numeric.gen.go index 6edd75291155c..57606c0fce6df 100644 --- a/go/arrow/type_traits_numeric.gen.go +++ b/go/arrow/type_traits_numeric.gen.go @@ -23,7 +23,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/endian" ) var ( diff --git a/go/arrow/type_traits_numeric.gen.go.tmpl b/go/arrow/type_traits_numeric.gen.go.tmpl index ffae975c1aa15..c491047b51429 100644 --- a/go/arrow/type_traits_numeric.gen.go.tmpl +++ b/go/arrow/type_traits_numeric.gen.go.tmpl @@ -21,7 +21,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/endian" ) var ( diff --git a/go/arrow/type_traits_numeric.gen_test.go b/go/arrow/type_traits_numeric.gen_test.go index 74395485736ff..ac2d0726ed6fa 100644 --- a/go/arrow/type_traits_numeric.gen_test.go +++ b/go/arrow/type_traits_numeric.gen_test.go @@ -22,7 +22,7 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) func TestInt64Traits(t *testing.T) { diff --git a/go/arrow/type_traits_numeric.gen_test.go.tmpl b/go/arrow/type_traits_numeric.gen_test.go.tmpl index 96685f31327bd..4948f42cf542e 100644 --- a/go/arrow/type_traits_numeric.gen_test.go.tmpl +++ b/go/arrow/type_traits_numeric.gen_test.go.tmpl @@ -20,7 +20,7 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) {{- range .In}} diff --git a/go/arrow/type_traits_test.go b/go/arrow/type_traits_test.go index 85ad4da9d041d..0ae88b4bd82b6 100644 --- a/go/arrow/type_traits_test.go +++ b/go/arrow/type_traits_test.go @@ -22,10 +22,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/float16" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/float16" ) func TestBooleanTraits(t *testing.T) { diff --git a/go/arrow/type_traits_timestamp.go b/go/arrow/type_traits_timestamp.go index 7c393b3559f82..c1a9aba3db386 100644 --- a/go/arrow/type_traits_timestamp.go +++ b/go/arrow/type_traits_timestamp.go @@ -20,7 +20,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/endian" ) var TimestampTraits timestampTraits diff --git a/go/arrow/util/byte_size.go b/go/arrow/util/byte_size.go index 141b78c459119..6d6fc021f8005 100644 --- a/go/arrow/util/byte_size.go +++ b/go/arrow/util/byte_size.go @@ -17,9 +17,9 @@ package util import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" ) func isArrayDataNil(arrayData arrow.ArrayData) bool { diff --git a/go/arrow/util/byte_size_test.go b/go/arrow/util/byte_size_test.go index 052bf3adee158..a218c69558fe9 100644 --- a/go/arrow/util/byte_size_test.go +++ b/go/arrow/util/byte_size_test.go @@ -20,10 +20,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/util" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/util" "github.com/stretchr/testify/assert" ) diff --git a/go/go.mod b/go/go.mod index de73c210d159d..a6c2af7025d32 100644 --- a/go/go.mod +++ b/go/go.mod @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -module github.com/apache/arrow/go/v14 +module github.com/apache/arrow/go/v15 go 1.20 diff --git a/go/internal/bitutils/bit_block_counter.go b/go/internal/bitutils/bit_block_counter.go index 86818bfd45ab7..99eece34cd068 100644 --- a/go/internal/bitutils/bit_block_counter.go +++ b/go/internal/bitutils/bit_block_counter.go @@ -21,8 +21,8 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/internal/utils" ) func loadWord(byt []byte) uint64 { diff --git a/go/internal/bitutils/bit_block_counter_test.go b/go/internal/bitutils/bit_block_counter_test.go index ad5e989d1fa65..790105c290182 100644 --- a/go/internal/bitutils/bit_block_counter_test.go +++ b/go/internal/bitutils/bit_block_counter_test.go @@ -19,9 +19,9 @@ package bitutils_test import ( "testing" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" "github.com/stretchr/testify/assert" "golang.org/x/exp/rand" ) diff --git a/go/internal/bitutils/bit_run_reader.go b/go/internal/bitutils/bit_run_reader.go index a1686a4909b8b..f09149d7ec5df 100644 --- a/go/internal/bitutils/bit_run_reader.go +++ b/go/internal/bitutils/bit_run_reader.go @@ -22,9 +22,9 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/internal/utils" ) // BitRun represents a run of bits with the same value of length Len diff --git a/go/internal/bitutils/bit_run_reader_test.go b/go/internal/bitutils/bit_run_reader_test.go index 0e1500bff811a..7db76768a9476 100644 --- a/go/internal/bitutils/bit_run_reader_test.go +++ b/go/internal/bitutils/bit_run_reader_test.go @@ -21,9 +21,9 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/internal/bitutils" "github.com/stretchr/testify/assert" ) diff --git a/go/internal/bitutils/bit_set_run_reader.go b/go/internal/bitutils/bit_set_run_reader.go index a2269ffecb870..6764ca7912679 100644 --- a/go/internal/bitutils/bit_set_run_reader.go +++ b/go/internal/bitutils/bit_set_run_reader.go @@ -20,8 +20,8 @@ import ( "encoding/binary" "math/bits" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/internal/utils" ) // IsMultipleOf64 returns whether v is a multiple of 64. diff --git a/go/internal/bitutils/bit_set_run_reader_test.go b/go/internal/bitutils/bit_set_run_reader_test.go index 205954b3800a8..832993671ef6d 100644 --- a/go/internal/bitutils/bit_set_run_reader_test.go +++ b/go/internal/bitutils/bit_set_run_reader_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/utils" "github.com/stretchr/testify/suite" ) diff --git a/go/internal/bitutils/bitmap_generate.go b/go/internal/bitutils/bitmap_generate.go index 78219d8120bcc..08b5fceab57d3 100644 --- a/go/internal/bitutils/bitmap_generate.go +++ b/go/internal/bitutils/bitmap_generate.go @@ -16,7 +16,7 @@ package bitutils -import "github.com/apache/arrow/go/v14/arrow/bitutil" +import "github.com/apache/arrow/go/v15/arrow/bitutil" // GenerateBits writes sequential bits to a bitmap. Bits preceding the // initial start offset are preserved, bits following the bitmap may diff --git a/go/internal/bitutils/bitmap_generate_test.go b/go/internal/bitutils/bitmap_generate_test.go index 788b2198c3a78..c9a6203864a20 100644 --- a/go/internal/bitutils/bitmap_generate_test.go +++ b/go/internal/bitutils/bitmap_generate_test.go @@ -19,7 +19,7 @@ package bitutils_test import ( "testing" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/bitutils" "golang.org/x/exp/rand" ) diff --git a/go/internal/hashing/xxh3_memo_table.gen.go b/go/internal/hashing/xxh3_memo_table.gen.go index cc996552b49e5..39b82cdeff9a2 100644 --- a/go/internal/hashing/xxh3_memo_table.gen.go +++ b/go/internal/hashing/xxh3_memo_table.gen.go @@ -21,9 +21,9 @@ package hashing import ( "math" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/internal/utils" ) type payloadInt8 struct { diff --git a/go/internal/hashing/xxh3_memo_table.gen.go.tmpl b/go/internal/hashing/xxh3_memo_table.gen.go.tmpl index 25164341d227e..527008ad63c3c 100644 --- a/go/internal/hashing/xxh3_memo_table.gen.go.tmpl +++ b/go/internal/hashing/xxh3_memo_table.gen.go.tmpl @@ -17,8 +17,8 @@ package hashing import ( - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/internal/utils" ) {{range .In}} diff --git a/go/internal/types/extension_types.go b/go/internal/types/extension_types.go index 1a31014874bbe..e24c89efc7b8b 100644 --- a/go/internal/types/extension_types.go +++ b/go/internal/types/extension_types.go @@ -24,9 +24,9 @@ import ( "reflect" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/internal/json" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/internal/json" "github.com/google/uuid" "golang.org/x/xerrors" ) diff --git a/go/internal/types/extension_types_test.go b/go/internal/types/extension_types_test.go index 16273890b4c9c..f93f1000c9e4a 100644 --- a/go/internal/types/extension_types_test.go +++ b/go/internal/types/extension_types_test.go @@ -20,11 +20,11 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/json" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v15/internal/types" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/internal/utils/transpose_ints_def.go b/go/internal/utils/transpose_ints_def.go index cc3b0abb59363..c52598d7148ea 100644 --- a/go/internal/utils/transpose_ints_def.go +++ b/go/internal/utils/transpose_ints_def.go @@ -19,7 +19,7 @@ package utils import ( "errors" - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) //go:generate go run ../../arrow/_tools/tmpl -i -data=transpose_ints.tmpldata -d arch=avx2 transpose_ints_simd.go.tmpl=transpose_ints_avx2_amd64.go diff --git a/go/internal/utils/transpose_ints_test.go b/go/internal/utils/transpose_ints_test.go index e41b61156abdc..73b2bbce3fc14 100644 --- a/go/internal/utils/transpose_ints_test.go +++ b/go/internal/utils/transpose_ints_test.go @@ -22,7 +22,7 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/internal/utils" ) var ( diff --git a/go/parquet/cmd/parquet_reader/dumper.go b/go/parquet/cmd/parquet_reader/dumper.go index a04dcf23b2d8d..4cb2ea4a96fee 100644 --- a/go/parquet/cmd/parquet_reader/dumper.go +++ b/go/parquet/cmd/parquet_reader/dumper.go @@ -22,9 +22,9 @@ import ( "reflect" "time" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/schema" ) const defaultBatchSize = 128 diff --git a/go/parquet/cmd/parquet_reader/main.go b/go/parquet/cmd/parquet_reader/main.go index e7d1cd7fc5a7d..0d651d8c294c5 100644 --- a/go/parquet/cmd/parquet_reader/main.go +++ b/go/parquet/cmd/parquet_reader/main.go @@ -25,11 +25,11 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/internal/json" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/docopt/docopt-go" ) diff --git a/go/parquet/cmd/parquet_schema/main.go b/go/parquet/cmd/parquet_schema/main.go index 28d1288b54aeb..01d541e8ac925 100644 --- a/go/parquet/cmd/parquet_schema/main.go +++ b/go/parquet/cmd/parquet_schema/main.go @@ -20,8 +20,8 @@ import ( "fmt" "os" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/docopt/docopt-go" ) diff --git a/go/parquet/compress/brotli.go b/go/parquet/compress/brotli.go index 1c1ae1dffaca5..a1199d9711435 100644 --- a/go/parquet/compress/brotli.go +++ b/go/parquet/compress/brotli.go @@ -21,7 +21,7 @@ import ( "io" "github.com/andybalholm/brotli" - "github.com/apache/arrow/go/v14/parquet/internal/debug" + "github.com/apache/arrow/go/v15/parquet/internal/debug" ) type brotliCodec struct{} diff --git a/go/parquet/compress/compress.go b/go/parquet/compress/compress.go index 4c4c792c2f614..f61147eb1ea10 100644 --- a/go/parquet/compress/compress.go +++ b/go/parquet/compress/compress.go @@ -23,7 +23,7 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" ) // Compression is an alias to the thrift compression codec enum type for easy use diff --git a/go/parquet/compress/compress_test.go b/go/parquet/compress/compress_test.go index e8041a64c0996..d1c55b15bc3cc 100644 --- a/go/parquet/compress/compress_test.go +++ b/go/parquet/compress/compress_test.go @@ -22,7 +22,7 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/compress" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/compress/zstd.go b/go/parquet/compress/zstd.go index 129af62807175..fd8ec81953359 100644 --- a/go/parquet/compress/zstd.go +++ b/go/parquet/compress/zstd.go @@ -20,7 +20,7 @@ import ( "io" "sync" - "github.com/apache/arrow/go/v14/parquet/internal/debug" + "github.com/apache/arrow/go/v15/parquet/internal/debug" "github.com/klauspost/compress/zstd" ) diff --git a/go/parquet/doc.go b/go/parquet/doc.go index 99a605a284fae..afeee00587ef5 100644 --- a/go/parquet/doc.go +++ b/go/parquet/doc.go @@ -29,9 +29,9 @@ // Install // // You can download the library and cli utilities via: -// go get -u github.com/apache/arrow/go/v14/parquet -// go install github.com/apache/arrow/go/v14/parquet/cmd/parquet_reader@latest -// go install github.com/apache/arrow/go/v14/parquet/cmd/parquet_schema@latest +// go get -u github.com/apache/arrow/go/v15/parquet +// go install github.com/apache/arrow/go/v15/parquet/cmd/parquet_reader@latest +// go install github.com/apache/arrow/go/v15/parquet/cmd/parquet_schema@latest // // Modules // diff --git a/go/parquet/encryption_properties.go b/go/parquet/encryption_properties.go index dd85a856820fe..6eb6cf1fe5680 100644 --- a/go/parquet/encryption_properties.go +++ b/go/parquet/encryption_properties.go @@ -20,7 +20,7 @@ import ( "crypto/rand" "unicode/utf8" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" ) // Constants that will be used as the default values with encryption/decryption diff --git a/go/parquet/encryption_properties_test.go b/go/parquet/encryption_properties_test.go index 0b1118a5cfa66..ab028927c5ecb 100644 --- a/go/parquet/encryption_properties_test.go +++ b/go/parquet/encryption_properties_test.go @@ -19,8 +19,8 @@ package parquet_test import ( "testing" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/encryption_read_config_test.go b/go/parquet/encryption_read_config_test.go index 8a84c19900dd0..53b7ba3c621c0 100644 --- a/go/parquet/encryption_read_config_test.go +++ b/go/parquet/encryption_read_config_test.go @@ -23,10 +23,10 @@ import ( "path" "testing" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/encryption_write_config_test.go b/go/parquet/encryption_write_config_test.go index 6b2d049e1390c..558b89208c706 100644 --- a/go/parquet/encryption_write_config_test.go +++ b/go/parquet/encryption_write_config_test.go @@ -23,10 +23,10 @@ import ( "path/filepath" "testing" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/file/column_reader.go b/go/parquet/file/column_reader.go index b623bd5074d02..766638d88f26c 100644 --- a/go/parquet/file/column_reader.go +++ b/go/parquet/file/column_reader.go @@ -21,13 +21,13 @@ import ( "fmt" "sync" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/column_reader_test.go b/go/parquet/file/column_reader_test.go index 3ce05c6cfe47e..21ea52e2b7bbc 100755 --- a/go/parquet/file/column_reader_test.go +++ b/go/parquet/file/column_reader_test.go @@ -24,12 +24,12 @@ import ( "sync" "testing" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/testutils" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/testutils" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/file/column_reader_types.gen.go b/go/parquet/file/column_reader_types.gen.go index 31dccc157642f..3fb113780f811 100644 --- a/go/parquet/file/column_reader_types.gen.go +++ b/go/parquet/file/column_reader_types.gen.go @@ -21,9 +21,9 @@ package file import ( "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" ) // Int32ColumnChunkReader is the Typed Column chunk reader instance for reading diff --git a/go/parquet/file/column_reader_types.gen.go.tmpl b/go/parquet/file/column_reader_types.gen.go.tmpl index 6557963e69f50..261b5f0bfacab 100644 --- a/go/parquet/file/column_reader_types.gen.go.tmpl +++ b/go/parquet/file/column_reader_types.gen.go.tmpl @@ -17,8 +17,8 @@ package file import ( - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" ) {{range .In}} diff --git a/go/parquet/file/column_writer.go b/go/parquet/file/column_writer.go index 9d9111042df28..0b0d1145d604e 100755 --- a/go/parquet/file/column_writer.go +++ b/go/parquet/file/column_writer.go @@ -21,14 +21,14 @@ import ( "encoding/binary" "io" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" ) //go:generate go run ../../arrow/_tools/tmpl/main.go -i -data=../internal/encoding/physical_types.tmpldata column_writer_types.gen.go.tmpl diff --git a/go/parquet/file/column_writer_test.go b/go/parquet/file/column_writer_test.go index e2ae6b0f4a71e..134c290fa6894 100755 --- a/go/parquet/file/column_writer_test.go +++ b/go/parquet/file/column_writer_test.go @@ -24,19 +24,19 @@ import ( "sync" "testing" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - arrutils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/testutils" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + arrutils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/testutils" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" diff --git a/go/parquet/file/column_writer_types.gen.go b/go/parquet/file/column_writer_types.gen.go index 5594f63249fb8..8b3be25f3ea29 100644 --- a/go/parquet/file/column_writer_types.gen.go +++ b/go/parquet/file/column_writer_types.gen.go @@ -22,12 +22,12 @@ import ( "errors" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/column_writer_types.gen.go.tmpl b/go/parquet/file/column_writer_types.gen.go.tmpl index c00e1dabb5fe6..7df69b4a219c6 100644 --- a/go/parquet/file/column_writer_types.gen.go.tmpl +++ b/go/parquet/file/column_writer_types.gen.go.tmpl @@ -19,10 +19,10 @@ package file import ( "fmt" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" ) {{range .In}} diff --git a/go/parquet/file/file_reader.go b/go/parquet/file/file_reader.go index 5481d5c7738b0..afff579ded5b7 100644 --- a/go/parquet/file/file_reader.go +++ b/go/parquet/file/file_reader.go @@ -25,10 +25,10 @@ import ( "runtime" "sync" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v15/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/file_reader_mmap.go b/go/parquet/file/file_reader_mmap.go index 89c5a84cee2e1..03e12adf08c8c 100644 --- a/go/parquet/file/file_reader_mmap.go +++ b/go/parquet/file/file_reader_mmap.go @@ -22,7 +22,7 @@ package file import ( "io" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/parquet" "golang.org/x/exp/mmap" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/file_reader_mmap_windows.go b/go/parquet/file/file_reader_mmap_windows.go index 6b08422163fd1..06a9e97160fe0 100644 --- a/go/parquet/file/file_reader_mmap_windows.go +++ b/go/parquet/file/file_reader_mmap_windows.go @@ -22,7 +22,7 @@ package file import ( "errors" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/parquet" ) func mmapOpen(filename string) (parquet.ReaderAtSeeker, error) { diff --git a/go/parquet/file/file_reader_test.go b/go/parquet/file/file_reader_test.go index 2a9b097139062..f3248925cf5b9 100644 --- a/go/parquet/file/file_reader_test.go +++ b/go/parquet/file/file_reader_test.go @@ -25,16 +25,16 @@ import ( "path" "testing" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/thrift" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/thrift" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" libthrift "github.com/apache/thrift/lib/go/thrift" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/parquet/file/file_writer.go b/go/parquet/file/file_writer.go index c6289434bbe6e..48eb8f44d167b 100644 --- a/go/parquet/file/file_writer.go +++ b/go/parquet/file/file_writer.go @@ -21,11 +21,11 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" ) // Writer is the primary interface for writing a parquet file diff --git a/go/parquet/file/file_writer_test.go b/go/parquet/file/file_writer_test.go index af083ebe60e4f..beceff9732584 100644 --- a/go/parquet/file/file_writer_test.go +++ b/go/parquet/file/file_writer_test.go @@ -22,13 +22,13 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/internal/testutils" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/internal/testutils" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/parquet/file/level_conversion.go b/go/parquet/file/level_conversion.go index ac53dcd18abae..c23bdda445963 100755 --- a/go/parquet/file/level_conversion.go +++ b/go/parquet/file/level_conversion.go @@ -22,11 +22,11 @@ import ( "math/bits" "unsafe" - shared_utils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/bmi" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - "github.com/apache/arrow/go/v14/parquet/schema" + shared_utils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/bmi" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/level_conversion_test.go b/go/parquet/file/level_conversion_test.go index 343edb687980d..5d5bdde90dc7e 100644 --- a/go/parquet/file/level_conversion_test.go +++ b/go/parquet/file/level_conversion_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/parquet/internal/bmi" - "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/parquet/internal/bmi" + "github.com/apache/arrow/go/v15/parquet/internal/utils" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/file/page_reader.go b/go/parquet/file/page_reader.go index ab8679b127f1b..01f253aff8864 100644 --- a/go/parquet/file/page_reader.go +++ b/go/parquet/file/page_reader.go @@ -23,13 +23,13 @@ import ( "sync" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/thrift" - "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/thrift" + "github.com/apache/arrow/go/v15/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/page_writer.go b/go/parquet/file/page_writer.go index b2b6d5061b0fc..c16476fbb232c 100644 --- a/go/parquet/file/page_writer.go +++ b/go/parquet/file/page_writer.go @@ -20,15 +20,15 @@ import ( "bytes" "sync" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/thrift" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/thrift" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v15/parquet/metadata" libthrift "github.com/apache/thrift/lib/go/thrift" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/record_reader.go b/go/parquet/file/record_reader.go index 3311a6d77da39..ad836d29ef483 100755 --- a/go/parquet/file/record_reader.go +++ b/go/parquet/file/record_reader.go @@ -23,14 +23,14 @@ import ( "unsafe" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/row_group_reader.go b/go/parquet/file/row_group_reader.go index f1aebf1085a23..3c1c1edb0b484 100644 --- a/go/parquet/file/row_group_reader.go +++ b/go/parquet/file/row_group_reader.go @@ -20,10 +20,10 @@ import ( "fmt" "sync" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v15/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/row_group_writer.go b/go/parquet/file/row_group_writer.go index 75f0dede5246f..935c13d13a67f 100644 --- a/go/parquet/file/row_group_writer.go +++ b/go/parquet/file/row_group_writer.go @@ -17,10 +17,10 @@ package file import ( - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v15/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/row_group_writer_test.go b/go/parquet/file/row_group_writer_test.go index bc95e99d91582..2ea317cb91001 100644 --- a/go/parquet/file/row_group_writer_test.go +++ b/go/parquet/file/row_group_writer_test.go @@ -20,10 +20,10 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/apache/thrift/lib/go/thrift" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/internal/bmi/bmi_test.go b/go/parquet/internal/bmi/bmi_test.go index 460c6ec4e24c6..a5278dfef2211 100644 --- a/go/parquet/internal/bmi/bmi_test.go +++ b/go/parquet/internal/bmi/bmi_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v14/parquet/internal/bmi" + "github.com/apache/arrow/go/v15/parquet/internal/bmi" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/internal/encoding/boolean_decoder.go b/go/parquet/internal/encoding/boolean_decoder.go index 337a6db967a2e..3782dc85ea814 100644 --- a/go/parquet/internal/encoding/boolean_decoder.go +++ b/go/parquet/internal/encoding/boolean_decoder.go @@ -23,10 +23,10 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/arrow/bitutil" - shared_utils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + shared_utils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/utils" ) // PlainBooleanDecoder is for the Plain Encoding type, there is no diff --git a/go/parquet/internal/encoding/boolean_encoder.go b/go/parquet/internal/encoding/boolean_encoder.go index 3970e05fca289..3e01bde369d8b 100644 --- a/go/parquet/internal/encoding/boolean_encoder.go +++ b/go/parquet/internal/encoding/boolean_encoder.go @@ -19,10 +19,10 @@ package encoding import ( "encoding/binary" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/debug" - "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/debug" + "github.com/apache/arrow/go/v15/parquet/internal/utils" ) const ( diff --git a/go/parquet/internal/encoding/byte_array_decoder.go b/go/parquet/internal/encoding/byte_array_decoder.go index c3ca1de5888f7..82ce9f84265c5 100644 --- a/go/parquet/internal/encoding/byte_array_decoder.go +++ b/go/parquet/internal/encoding/byte_array_decoder.go @@ -19,12 +19,12 @@ package encoding import ( "encoding/binary" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - pqutils "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + pqutils "github.com/apache/arrow/go/v15/parquet/internal/utils" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/byte_array_encoder.go b/go/parquet/internal/encoding/byte_array_encoder.go index cb49178435a6f..9270b5531768c 100644 --- a/go/parquet/internal/encoding/byte_array_encoder.go +++ b/go/parquet/internal/encoding/byte_array_encoder.go @@ -21,11 +21,11 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" ) // PlainByteArrayEncoder encodes byte arrays according to the spec for Plain encoding diff --git a/go/parquet/internal/encoding/decoder.go b/go/parquet/internal/encoding/decoder.go index b441868e1df85..cee624730e993 100644 --- a/go/parquet/internal/encoding/decoder.go +++ b/go/parquet/internal/encoding/decoder.go @@ -20,16 +20,16 @@ import ( "bytes" "reflect" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" - shared_utils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/debug" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" + shared_utils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/debug" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/delta_bit_packing.go b/go/parquet/internal/encoding/delta_bit_packing.go index 1fb91634e977b..a00f3457cac7a 100644 --- a/go/parquet/internal/encoding/delta_bit_packing.go +++ b/go/parquet/internal/encoding/delta_bit_packing.go @@ -23,11 +23,11 @@ import ( "math/bits" "reflect" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" - shared_utils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" + shared_utils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/utils" ) // see the deltaBitPack encoder for a description of the encoding format that is diff --git a/go/parquet/internal/encoding/delta_byte_array.go b/go/parquet/internal/encoding/delta_byte_array.go index 60a20a3ff0d72..b35d022fd5deb 100644 --- a/go/parquet/internal/encoding/delta_byte_array.go +++ b/go/parquet/internal/encoding/delta_byte_array.go @@ -17,9 +17,9 @@ package encoding import ( - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/delta_byte_array_test.go b/go/parquet/internal/encoding/delta_byte_array_test.go index 69c632bf69971..0a206796f742a 100644 --- a/go/parquet/internal/encoding/delta_byte_array_test.go +++ b/go/parquet/internal/encoding/delta_byte_array_test.go @@ -18,8 +18,8 @@ package encoding import ( "fmt" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" "github.com/stretchr/testify/assert" "testing" ) diff --git a/go/parquet/internal/encoding/delta_length_byte_array.go b/go/parquet/internal/encoding/delta_length_byte_array.go index 981a067b07e08..d5a99c187d11e 100644 --- a/go/parquet/internal/encoding/delta_length_byte_array.go +++ b/go/parquet/internal/encoding/delta_length_byte_array.go @@ -17,9 +17,9 @@ package encoding import ( - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/encoder.go b/go/parquet/internal/encoding/encoder.go index f6b57fe63c433..7023309397a3b 100644 --- a/go/parquet/internal/encoding/encoder.go +++ b/go/parquet/internal/encoding/encoder.go @@ -21,14 +21,14 @@ import ( "math/bits" "reflect" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v15/parquet/schema" ) //go:generate go run ../../../arrow/_tools/tmpl/main.go -i -data=physical_types.tmpldata plain_encoder_types.gen.go.tmpl typed_encoder.gen.go.tmpl diff --git a/go/parquet/internal/encoding/encoding_benchmarks_test.go b/go/parquet/internal/encoding/encoding_benchmarks_test.go index 0bb4e895f316d..e0645e9de54e4 100644 --- a/go/parquet/internal/encoding/encoding_benchmarks_test.go +++ b/go/parquet/internal/encoding/encoding_benchmarks_test.go @@ -21,14 +21,14 @@ import ( "math" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/hashing" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/internal/testutils" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/hashing" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/internal/testutils" + "github.com/apache/arrow/go/v15/parquet/schema" ) const ( diff --git a/go/parquet/internal/encoding/encoding_test.go b/go/parquet/internal/encoding/encoding_test.go index b0d86321e09f1..48e2316b0f897 100644 --- a/go/parquet/internal/encoding/encoding_test.go +++ b/go/parquet/internal/encoding/encoding_test.go @@ -26,13 +26,13 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/internal/testutils" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/internal/testutils" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go b/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go index e4516bf4a1ac1..1e589fc2e7be1 100644 --- a/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go +++ b/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go @@ -19,8 +19,8 @@ package encoding import ( "math" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go b/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go index b5aee337d5bf0..39202c8e25d9f 100644 --- a/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go +++ b/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go @@ -19,9 +19,9 @@ package encoding import ( "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/parquet" ) // PlainFixedLenByteArrayEncoder writes the raw bytes of the byte array diff --git a/go/parquet/internal/encoding/levels.go b/go/parquet/internal/encoding/levels.go index e04ec19d5473d..caf832059334b 100644 --- a/go/parquet/internal/encoding/levels.go +++ b/go/parquet/internal/encoding/levels.go @@ -23,11 +23,11 @@ import ( "math/bits" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - shared_utils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + shared_utils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/utils" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/levels_test.go b/go/parquet/internal/encoding/levels_test.go index ae3a3a8c497fc..304ce32b3106d 100644 --- a/go/parquet/internal/encoding/levels_test.go +++ b/go/parquet/internal/encoding/levels_test.go @@ -21,11 +21,11 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/internal/encoding/memo_table.go b/go/parquet/internal/encoding/memo_table.go index 170eab47d6160..810e8633b886e 100644 --- a/go/parquet/internal/encoding/memo_table.go +++ b/go/parquet/internal/encoding/memo_table.go @@ -20,11 +20,11 @@ import ( "math" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/hashing" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/hashing" + "github.com/apache/arrow/go/v15/parquet" ) //go:generate go run ../../../arrow/_tools/tmpl/main.go -i -data=physical_types.tmpldata memo_table_types.gen.go.tmpl diff --git a/go/parquet/internal/encoding/memo_table_test.go b/go/parquet/internal/encoding/memo_table_test.go index 1eb61cb2559a3..1b9337010f855 100644 --- a/go/parquet/internal/encoding/memo_table_test.go +++ b/go/parquet/internal/encoding/memo_table_test.go @@ -20,11 +20,11 @@ import ( "math" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/hashing" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/hashing" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/internal/encoding/memo_table_types.gen.go b/go/parquet/internal/encoding/memo_table_types.gen.go index c9af6178bbb98..4da2721437814 100644 --- a/go/parquet/internal/encoding/memo_table_types.gen.go +++ b/go/parquet/internal/encoding/memo_table_types.gen.go @@ -19,8 +19,8 @@ package encoding import ( - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" ) // standard map based implementation of memo tables which can be more efficient diff --git a/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl b/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl index 7e88238d67b71..75335f25ff1f7 100644 --- a/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl +++ b/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl @@ -17,7 +17,7 @@ package encoding import ( - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/parquet" ) // standard map based implementation of memo tables which can be more efficient diff --git a/go/parquet/internal/encoding/plain_encoder_types.gen.go b/go/parquet/internal/encoding/plain_encoder_types.gen.go index 23db13b7a4121..09403d74cb06f 100644 --- a/go/parquet/internal/encoding/plain_encoder_types.gen.go +++ b/go/parquet/internal/encoding/plain_encoder_types.gen.go @@ -24,11 +24,11 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl b/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl index e45419050302d..2838c63a41857 100644 --- a/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl +++ b/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl @@ -20,10 +20,10 @@ import ( "encoding/binary" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/internal/bitutils" ) var ( diff --git a/go/parquet/internal/encoding/typed_encoder.gen.go b/go/parquet/internal/encoding/typed_encoder.gen.go index 411e87c17e7d5..25fa309e0a38f 100644 --- a/go/parquet/internal/encoding/typed_encoder.gen.go +++ b/go/parquet/internal/encoding/typed_encoder.gen.go @@ -22,15 +22,15 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" - shared_utils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" + shared_utils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl b/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl index 69415ccca4a26..66c2649d599c6 100644 --- a/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl +++ b/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl @@ -17,13 +17,13 @@ package encoding import ( - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - shared_utils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + shared_utils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/internal/bitutils" ) // fully typed encoder interfaces to enable writing against encoder/decoders diff --git a/go/parquet/internal/encoding/types.go b/go/parquet/internal/encoding/types.go index 2d13a9c8f877a..4ab3ab1a1c954 100644 --- a/go/parquet/internal/encoding/types.go +++ b/go/parquet/internal/encoding/types.go @@ -20,11 +20,11 @@ import ( "io" "sync" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encryption/aes.go b/go/parquet/internal/encryption/aes.go index cc0cea0112fee..b6e9130ef81cd 100644 --- a/go/parquet/internal/encryption/aes.go +++ b/go/parquet/internal/encryption/aes.go @@ -29,7 +29,7 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/parquet" ) // important constants for handling the aes encryption diff --git a/go/parquet/internal/encryption/decryptor.go b/go/parquet/internal/encryption/decryptor.go index 62a4f2ee9e56e..658559e6082c4 100644 --- a/go/parquet/internal/encryption/decryptor.go +++ b/go/parquet/internal/encryption/decryptor.go @@ -19,8 +19,8 @@ package encryption import ( "io" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" ) // FileDecryptor is an interface used by the filereader for decrypting an diff --git a/go/parquet/internal/encryption/encryptor.go b/go/parquet/internal/encryption/encryptor.go index 2ef09e558a23f..bdbae4740a44f 100644 --- a/go/parquet/internal/encryption/encryptor.go +++ b/go/parquet/internal/encryption/encryptor.go @@ -19,8 +19,8 @@ package encryption import ( "io" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" ) // FileEncryptor is the interface for constructing encryptors for the different diff --git a/go/parquet/internal/testutils/pagebuilder.go b/go/parquet/internal/testutils/pagebuilder.go index 9c1fd72a1bdf0..48ac331640087 100644 --- a/go/parquet/internal/testutils/pagebuilder.go +++ b/go/parquet/internal/testutils/pagebuilder.go @@ -22,13 +22,13 @@ import ( "io" "reflect" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/mock" ) diff --git a/go/parquet/internal/testutils/primitive_typed.go b/go/parquet/internal/testutils/primitive_typed.go index 2bd0a29dd2970..daab0d427ba10 100644 --- a/go/parquet/internal/testutils/primitive_typed.go +++ b/go/parquet/internal/testutils/primitive_typed.go @@ -20,11 +20,11 @@ import ( "fmt" "reflect" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" ) type PrimitiveTypedTest struct { diff --git a/go/parquet/internal/testutils/random.go b/go/parquet/internal/testutils/random.go index d9a06da43ba4e..bb101ebf9a92d 100644 --- a/go/parquet/internal/testutils/random.go +++ b/go/parquet/internal/testutils/random.go @@ -24,13 +24,13 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/endian" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/pqarrow" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/pqarrow" "golang.org/x/exp/rand" "gonum.org/v1/gonum/stat/distuv" diff --git a/go/parquet/internal/testutils/random_arrow.go b/go/parquet/internal/testutils/random_arrow.go index 360b8e7476430..d886db0360b84 100644 --- a/go/parquet/internal/testutils/random_arrow.go +++ b/go/parquet/internal/testutils/random_arrow.go @@ -17,9 +17,9 @@ package testutils import ( - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/exp/rand" ) diff --git a/go/parquet/internal/testutils/utils.go b/go/parquet/internal/testutils/utils.go index 1122f4286f386..3da76c17ddc32 100644 --- a/go/parquet/internal/testutils/utils.go +++ b/go/parquet/internal/testutils/utils.go @@ -19,7 +19,7 @@ package testutils import ( "reflect" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/parquet" ) var typeToParquetTypeMap = map[reflect.Type]parquet.Type{ diff --git a/go/parquet/internal/thrift/helpers.go b/go/parquet/internal/thrift/helpers.go index 357d9020989d3..3835830ac6c2d 100644 --- a/go/parquet/internal/thrift/helpers.go +++ b/go/parquet/internal/thrift/helpers.go @@ -23,7 +23,7 @@ import ( "context" "io" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" "github.com/apache/thrift/lib/go/thrift" ) diff --git a/go/parquet/internal/utils/bit_benchmark_test.go b/go/parquet/internal/utils/bit_benchmark_test.go index 2518e48a4baa8..14353380a5694 100644 --- a/go/parquet/internal/utils/bit_benchmark_test.go +++ b/go/parquet/internal/utils/bit_benchmark_test.go @@ -20,9 +20,9 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/parquet/internal/testutils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/parquet/internal/testutils" ) type linearBitRunReader struct { diff --git a/go/parquet/internal/utils/bit_packing_arm64.go b/go/parquet/internal/utils/bit_packing_arm64.go index 92fe0f97b77c3..e9fb2ef1d810b 100644 --- a/go/parquet/internal/utils/bit_packing_arm64.go +++ b/go/parquet/internal/utils/bit_packing_arm64.go @@ -23,7 +23,7 @@ import ( "github.com/klauspost/cpuid/v2" // import for side effect of initializing feature flags // based on ARM_ENABLE_EXT env var - _ "github.com/apache/arrow/go/v14/parquet/internal/bmi" + _ "github.com/apache/arrow/go/v15/parquet/internal/bmi" ) func init() { diff --git a/go/parquet/internal/utils/bit_reader.go b/go/parquet/internal/utils/bit_reader.go index cf61395c0b16c..bf9741c79878b 100644 --- a/go/parquet/internal/utils/bit_reader.go +++ b/go/parquet/internal/utils/bit_reader.go @@ -24,10 +24,10 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" ) // masks for grabbing the trailing bits based on the number of trailing bits desired diff --git a/go/parquet/internal/utils/bit_reader_test.go b/go/parquet/internal/utils/bit_reader_test.go index c285f5165cc0a..5ce1b799b463b 100644 --- a/go/parquet/internal/utils/bit_reader_test.go +++ b/go/parquet/internal/utils/bit_reader_test.go @@ -25,11 +25,11 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet/internal/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "golang.org/x/exp/rand" diff --git a/go/parquet/internal/utils/bit_writer.go b/go/parquet/internal/utils/bit_writer.go index 99a1db065590d..6cb255f5b0473 100644 --- a/go/parquet/internal/utils/bit_writer.go +++ b/go/parquet/internal/utils/bit_writer.go @@ -21,7 +21,7 @@ import ( "io" "log" - "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/bitutil" ) // WriterAtBuffer is a convenience struct for providing a WriteAt function diff --git a/go/parquet/internal/utils/bitmap_writer.go b/go/parquet/internal/utils/bitmap_writer.go index 447222139b773..3ef99291e3748 100644 --- a/go/parquet/internal/utils/bitmap_writer.go +++ b/go/parquet/internal/utils/bitmap_writer.go @@ -20,7 +20,7 @@ import ( "encoding/binary" "math/bits" - "github.com/apache/arrow/go/v14/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/bitutil" ) // BitmapWriter is an interface for bitmap writers so that we can use multiple diff --git a/go/parquet/internal/utils/bitmap_writer_test.go b/go/parquet/internal/utils/bitmap_writer_test.go index 9f38f1ef291ea..3dddc7567903e 100644 --- a/go/parquet/internal/utils/bitmap_writer_test.go +++ b/go/parquet/internal/utils/bitmap_writer_test.go @@ -22,8 +22,8 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/parquet/internal/utils" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/parquet/internal/utils" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/internal/utils/rle.go b/go/parquet/internal/utils/rle.go index fef322c6fd560..21ac2ce6b5db5 100644 --- a/go/parquet/internal/utils/rle.go +++ b/go/parquet/internal/utils/rle.go @@ -24,10 +24,10 @@ import ( "encoding/binary" "math" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/utils/typed_rle_dict.gen.go b/go/parquet/internal/utils/typed_rle_dict.gen.go index dff22711b120f..886d24564db4b 100644 --- a/go/parquet/internal/utils/typed_rle_dict.gen.go +++ b/go/parquet/internal/utils/typed_rle_dict.gen.go @@ -19,9 +19,9 @@ package utils import ( - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl b/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl index 090275745ff59..abcb419055a92 100644 --- a/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl +++ b/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl @@ -17,9 +17,9 @@ package utils import ( - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/internal/utils" ) {{range .In}} diff --git a/go/parquet/metadata/app_version.go b/go/parquet/metadata/app_version.go index 85ace57365bd3..9966827026106 100644 --- a/go/parquet/metadata/app_version.go +++ b/go/parquet/metadata/app_version.go @@ -21,8 +21,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" ) var ( diff --git a/go/parquet/metadata/column_chunk.go b/go/parquet/metadata/column_chunk.go index e6ea6ff1ea183..729f741e1b4f9 100644 --- a/go/parquet/metadata/column_chunk.go +++ b/go/parquet/metadata/column_chunk.go @@ -22,13 +22,13 @@ import ( "io" "reflect" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/thrift" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/thrift" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/metadata/file.go b/go/parquet/metadata/file.go index dddd95c5df670..3335140c2e1c8 100644 --- a/go/parquet/metadata/file.go +++ b/go/parquet/metadata/file.go @@ -24,12 +24,12 @@ import ( "reflect" "unicode/utf8" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/thrift" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/thrift" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/metadata/metadata_test.go b/go/parquet/metadata/metadata_test.go index b685dd2223274..8caa319f83e63 100644 --- a/go/parquet/metadata/metadata_test.go +++ b/go/parquet/metadata/metadata_test.go @@ -21,9 +21,9 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/metadata/row_group.go b/go/parquet/metadata/row_group.go index 16af67241e959..2923720371abe 100644 --- a/go/parquet/metadata/row_group.go +++ b/go/parquet/metadata/row_group.go @@ -20,10 +20,10 @@ import ( "fmt" "reflect" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encryption" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encryption" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" ) // RowGroupMetaData is a proxy around the thrift RowGroup meta data object diff --git a/go/parquet/metadata/stat_compare_test.go b/go/parquet/metadata/stat_compare_test.go index d02123cbef039..041696d84d17a 100644 --- a/go/parquet/metadata/stat_compare_test.go +++ b/go/parquet/metadata/stat_compare_test.go @@ -20,8 +20,8 @@ import ( "encoding/binary" "testing" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/metadata/statistics.go b/go/parquet/metadata/statistics.go index 245b06a7348c4..606e76ffd0454 100644 --- a/go/parquet/metadata/statistics.go +++ b/go/parquet/metadata/statistics.go @@ -22,14 +22,14 @@ import ( "math" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/debug" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/debug" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" ) //go:generate go run ../../arrow/_tools/tmpl/main.go -i -data=../internal/encoding/physical_types.tmpldata statistics_types.gen.go.tmpl diff --git a/go/parquet/metadata/statistics_test.go b/go/parquet/metadata/statistics_test.go index 35d8b7821c51a..47798d3c4dd3c 100644 --- a/go/parquet/metadata/statistics_test.go +++ b/go/parquet/metadata/statistics_test.go @@ -21,11 +21,11 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/metadata/statistics_types.gen.go b/go/parquet/metadata/statistics_types.gen.go index e6aa7f1801a0f..e8fb9877c8444 100644 --- a/go/parquet/metadata/statistics_types.gen.go +++ b/go/parquet/metadata/statistics_types.gen.go @@ -22,14 +22,14 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" - shared_utils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" + shared_utils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/metadata/statistics_types.gen.go.tmpl b/go/parquet/metadata/statistics_types.gen.go.tmpl index 35470f06046bf..4b6253a8574ea 100644 --- a/go/parquet/metadata/statistics_types.gen.go.tmpl +++ b/go/parquet/metadata/statistics_types.gen.go.tmpl @@ -19,13 +19,13 @@ package metadata import ( "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" - "github.com/apache/arrow/go/v14/parquet/internal/utils" - shared_utils "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/internal/bitutils" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v15/parquet/internal/utils" + shared_utils "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/internal/bitutils" ) {{range .In}} diff --git a/go/parquet/pqarrow/column_readers.go b/go/parquet/pqarrow/column_readers.go index 759a3d8675927..02f94c941c3d9 100644 --- a/go/parquet/pqarrow/column_readers.go +++ b/go/parquet/pqarrow/column_readers.go @@ -26,16 +26,16 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/sync/errgroup" ) diff --git a/go/parquet/pqarrow/encode_arrow.go b/go/parquet/pqarrow/encode_arrow.go index c3a0a50c43f45..81b4527b105cb 100644 --- a/go/parquet/pqarrow/encode_arrow.go +++ b/go/parquet/pqarrow/encode_arrow.go @@ -25,16 +25,16 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/debug" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/debug" ) // get the count of the number of leaf arrays for the type diff --git a/go/parquet/pqarrow/encode_arrow_test.go b/go/parquet/pqarrow/encode_arrow_test.go index 3c20cf2d4757b..281ca0d526476 100644 --- a/go/parquet/pqarrow/encode_arrow_test.go +++ b/go/parquet/pqarrow/encode_arrow_test.go @@ -25,22 +25,22 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/bitutil" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" - "github.com/apache/arrow/go/v14/parquet/internal/testutils" - "github.com/apache/arrow/go/v14/parquet/pqarrow" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/parquet/internal/testutils" + "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/parquet/pqarrow/encode_dict_compute.go b/go/parquet/pqarrow/encode_dict_compute.go index 59e25cec19132..b43b4002ed0af 100644 --- a/go/parquet/pqarrow/encode_dict_compute.go +++ b/go/parquet/pqarrow/encode_dict_compute.go @@ -21,14 +21,14 @@ package pqarrow import ( "context" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/debug" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/debug" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" ) func isDictEncoding(enc parquet.Encoding) bool { diff --git a/go/parquet/pqarrow/encode_dict_nocompute.go b/go/parquet/pqarrow/encode_dict_nocompute.go index 31d0f1bc1ce27..73ec5cfc46682 100644 --- a/go/parquet/pqarrow/encode_dict_nocompute.go +++ b/go/parquet/pqarrow/encode_dict_nocompute.go @@ -21,8 +21,8 @@ package pqarrow import ( "errors" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/parquet/file" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/parquet/file" ) func writeDictionaryArrow(*arrowWriteContext, file.ColumnChunkWriter, arrow.Array, []int16, []int16, bool) (err error) { diff --git a/go/parquet/pqarrow/encode_dictionary_test.go b/go/parquet/pqarrow/encode_dictionary_test.go index 995f3fb329534..28ebee53e1b83 100644 --- a/go/parquet/pqarrow/encode_dictionary_test.go +++ b/go/parquet/pqarrow/encode_dictionary_test.go @@ -26,14 +26,14 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/internal/testutils" - "github.com/apache/arrow/go/v14/parquet/pqarrow" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/internal/testutils" + "github.com/apache/arrow/go/v15/parquet/pqarrow" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/parquet/pqarrow/file_reader.go b/go/parquet/pqarrow/file_reader.go index d91010c62c19d..3534cc87b78b5 100755 --- a/go/parquet/pqarrow/file_reader.go +++ b/go/parquet/pqarrow/file_reader.go @@ -23,13 +23,13 @@ import ( "sync" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/arrio" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/arrio" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/file_reader_test.go b/go/parquet/pqarrow/file_reader_test.go index d1f3ae1c984a2..9c1b4252f5fc6 100644 --- a/go/parquet/pqarrow/file_reader_test.go +++ b/go/parquet/pqarrow/file_reader_test.go @@ -26,13 +26,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/pqarrow" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/pqarrow" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/pqarrow/file_writer.go b/go/parquet/pqarrow/file_writer.go index aa0bae7b1fdfb..21f16c0b67938 100644 --- a/go/parquet/pqarrow/file_writer.go +++ b/go/parquet/pqarrow/file_writer.go @@ -22,12 +22,12 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/internal/utils" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/metadata" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/helpers.go b/go/parquet/pqarrow/helpers.go index e8b740d12ba62..39d17cf80ae68 100644 --- a/go/parquet/pqarrow/helpers.go +++ b/go/parquet/pqarrow/helpers.go @@ -17,7 +17,7 @@ package pqarrow import ( - "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v15/arrow" ) func releaseArrays(arrays []arrow.Array) { diff --git a/go/parquet/pqarrow/path_builder.go b/go/parquet/pqarrow/path_builder.go index c4915d58dd59f..0f1158bd1e9f0 100644 --- a/go/parquet/pqarrow/path_builder.go +++ b/go/parquet/pqarrow/path_builder.go @@ -21,11 +21,11 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/bitutils" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/path_builder_test.go b/go/parquet/pqarrow/path_builder_test.go index 16b994582df02..e0a60262d3f4a 100644 --- a/go/parquet/pqarrow/path_builder_test.go +++ b/go/parquet/pqarrow/path_builder_test.go @@ -20,10 +20,10 @@ import ( "context" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/parquet/pqarrow/properties.go b/go/parquet/pqarrow/properties.go index 98427bbf9398c..cc100fa80d87b 100755 --- a/go/parquet/pqarrow/properties.go +++ b/go/parquet/pqarrow/properties.go @@ -19,9 +19,9 @@ package pqarrow import ( "context" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet/internal/encoding" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet/internal/encoding" ) // ArrowWriterProperties are used to determine how to manipulate the arrow data diff --git a/go/parquet/pqarrow/reader_writer_test.go b/go/parquet/pqarrow/reader_writer_test.go index e3f66834c2cee..9d09bcec15da6 100644 --- a/go/parquet/pqarrow/reader_writer_test.go +++ b/go/parquet/pqarrow/reader_writer_test.go @@ -22,12 +22,12 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/pqarrow" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/pqarrow" "golang.org/x/exp/rand" "gonum.org/v1/gonum/stat/distuv" ) diff --git a/go/parquet/pqarrow/schema.go b/go/parquet/pqarrow/schema.go index 9ba8554898986..b23c37ea39102 100644 --- a/go/parquet/pqarrow/schema.go +++ b/go/parquet/pqarrow/schema.go @@ -22,15 +22,15 @@ import ( "math" "strconv" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/file" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/schema_test.go b/go/parquet/pqarrow/schema_test.go index e195358af6c98..b5e7dc8fad34a 100644 --- a/go/parquet/pqarrow/schema_test.go +++ b/go/parquet/pqarrow/schema_test.go @@ -20,15 +20,15 @@ import ( "encoding/base64" "testing" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/types" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/apache/arrow/go/v14/parquet/pqarrow" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/reader_properties.go b/go/parquet/reader_properties.go index dd2032958a51c..be2377527d782 100644 --- a/go/parquet/reader_properties.go +++ b/go/parquet/reader_properties.go @@ -21,8 +21,8 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/internal/utils" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/internal/utils" ) // ReaderProperties are used to define how the file reader will handle buffering and allocating buffers diff --git a/go/parquet/reader_writer_properties_test.go b/go/parquet/reader_writer_properties_test.go index 6bd93a02550e4..698129471adda 100644 --- a/go/parquet/reader_writer_properties_test.go +++ b/go/parquet/reader_writer_properties_test.go @@ -20,9 +20,9 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/compress" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/column.go b/go/parquet/schema/column.go index 1575037d59d23..02d0d83e8465e 100644 --- a/go/parquet/schema/column.go +++ b/go/parquet/schema/column.go @@ -20,8 +20,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" ) // Column encapsulates the information necessary to interpret primitive diff --git a/go/parquet/schema/converted_types.go b/go/parquet/schema/converted_types.go index 6a4702b819259..58051dba7f19b 100644 --- a/go/parquet/schema/converted_types.go +++ b/go/parquet/schema/converted_types.go @@ -17,7 +17,7 @@ package schema import ( - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" ) // ConvertedType corresponds to the ConvertedType in the parquet.Thrift, diff --git a/go/parquet/schema/converted_types_test.go b/go/parquet/schema/converted_types_test.go index a4f9ea0018196..8b8e061466474 100644 --- a/go/parquet/schema/converted_types_test.go +++ b/go/parquet/schema/converted_types_test.go @@ -19,7 +19,7 @@ package schema_test import ( "testing" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/helpers.go b/go/parquet/schema/helpers.go index 23a5a0b667f26..13075a065f073 100644 --- a/go/parquet/schema/helpers.go +++ b/go/parquet/schema/helpers.go @@ -17,7 +17,7 @@ package schema import ( - "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v15/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/helpers_test.go b/go/parquet/schema/helpers_test.go index 6d136e4ecd155..98f3cab36d433 100644 --- a/go/parquet/schema/helpers_test.go +++ b/go/parquet/schema/helpers_test.go @@ -21,8 +21,8 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/logical_types.go b/go/parquet/schema/logical_types.go index a0607a1a2f62d..1ea44fc56c615 100644 --- a/go/parquet/schema/logical_types.go +++ b/go/parquet/schema/logical_types.go @@ -20,10 +20,10 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v14/internal/json" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/internal/debug" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/internal/debug" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" ) // DecimalMetadata is a struct for managing scale and precision information between diff --git a/go/parquet/schema/logical_types_test.go b/go/parquet/schema/logical_types_test.go index 8531a061abb77..49edf1748c52c 100644 --- a/go/parquet/schema/logical_types_test.go +++ b/go/parquet/schema/logical_types_test.go @@ -19,9 +19,9 @@ package schema_test import ( "testing" - "github.com/apache/arrow/go/v14/internal/json" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/node.go b/go/parquet/schema/node.go index 28259c8d61f58..ff23624afa35d 100644 --- a/go/parquet/schema/node.go +++ b/go/parquet/schema/node.go @@ -19,8 +19,8 @@ package schema import ( "fmt" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" "github.com/apache/thrift/lib/go/thrift" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/reflection.go b/go/parquet/schema/reflection.go index bc863f1390030..d79edb92408f8 100644 --- a/go/parquet/schema/reflection.go +++ b/go/parquet/schema/reflection.go @@ -22,8 +22,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/reflection_test.go b/go/parquet/schema/reflection_test.go index e18feada7a796..06ad7191a5749 100644 --- a/go/parquet/schema/reflection_test.go +++ b/go/parquet/schema/reflection_test.go @@ -22,8 +22,8 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/schema.go b/go/parquet/schema/schema.go index 1b3363f58d11f..ace2775763a06 100644 --- a/go/parquet/schema/schema.go +++ b/go/parquet/schema/schema.go @@ -35,8 +35,8 @@ import ( "io" "strings" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/schema_element_test.go b/go/parquet/schema/schema_element_test.go index e9a2ef5eb31f6..dd1d293e5cfd7 100644 --- a/go/parquet/schema/schema_element_test.go +++ b/go/parquet/schema/schema_element_test.go @@ -19,8 +19,8 @@ package schema import ( "testing" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/schema/schema_flatten_test.go b/go/parquet/schema/schema_flatten_test.go index 90741ae9b43bb..34f4ac8d3c450 100644 --- a/go/parquet/schema/schema_flatten_test.go +++ b/go/parquet/schema/schema_flatten_test.go @@ -19,8 +19,8 @@ package schema import ( "testing" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/schema/schema_test.go b/go/parquet/schema/schema_test.go index e69a13e07986e..b60c7dfaafcd1 100644 --- a/go/parquet/schema/schema_test.go +++ b/go/parquet/schema/schema_test.go @@ -20,9 +20,9 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v14/parquet" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v14/parquet/schema" + "github.com/apache/arrow/go/v15/parquet" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/parquet/schema" "github.com/apache/thrift/lib/go/thrift" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" diff --git a/go/parquet/types.go b/go/parquet/types.go index 7b6df2ea9d16c..5b6f8fb0251d9 100644 --- a/go/parquet/types.go +++ b/go/parquet/types.go @@ -24,8 +24,8 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v14/arrow" - format "github.com/apache/arrow/go/v14/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v15/arrow" + format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" ) const ( diff --git a/go/parquet/writer_properties.go b/go/parquet/writer_properties.go index acd43194c1be7..9e33bddf7faa7 100644 --- a/go/parquet/writer_properties.go +++ b/go/parquet/writer_properties.go @@ -17,8 +17,8 @@ package parquet import ( - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/parquet/compress" ) // Constants for default property values used for the default reader, writer and column props. @@ -46,7 +46,7 @@ const ( DefaultStatsEnabled = true // If the stats are larger than 4K the writer will skip writing them out anyways. DefaultMaxStatsSize int64 = 4096 - DefaultCreatedBy = "parquet-go version 14.0.1-SNAPSHOT" + DefaultCreatedBy = "parquet-go version 15.0.0-SNAPSHOT" DefaultRootName = "schema" ) diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml index 20602f3b80771..c0410ea4c2314 100644 --- a/java/adapter/avro/pom.xml +++ b/java/adapter/avro/pom.xml @@ -16,7 +16,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT ../../pom.xml diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index f97bcaf74a27c..2490f708e6f24 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -16,7 +16,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT ../../pom.xml diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index 2213a2a7d0783..72ba13ea81738 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -111,7 +111,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT ../../pom.xml diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index a248389aeb406..8c6a9fb0151ef 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT arrow-algorithm Arrow Algorithms diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 01f5e93dc0438..1f6f854f60013 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-bom - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT pom Arrow Bill of Materials Arrow Bill of Materials diff --git a/java/c/pom.xml b/java/c/pom.xml index 7a4631704fa80..d66379d356624 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -13,7 +13,7 @@ arrow-java-root org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT 4.0.0 diff --git a/java/compression/pom.xml b/java/compression/pom.xml index 283c2af2d7d76..129268b9e6a07 100644 --- a/java/compression/pom.xml +++ b/java/compression/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT arrow-compression Arrow Compression diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index cdc1a411798c3..b533a1733521b 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT 4.0.0 diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 172a8583d8c88..7b69179053d7f 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -14,7 +14,7 @@ arrow-flight org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT ../pom.xml diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml index f113ba50759a9..e7bb9508d2403 100644 --- a/java/flight/flight-grpc/pom.xml +++ b/java/flight/flight-grpc/pom.xml @@ -13,7 +13,7 @@ arrow-flight org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/java/flight/flight-integration-tests/pom.xml b/java/flight/flight-integration-tests/pom.xml index 6b7796641b607..778cb7df11f61 100644 --- a/java/flight/flight-integration-tests/pom.xml +++ b/java/flight/flight-integration-tests/pom.xml @@ -15,7 +15,7 @@ arrow-flight org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT ../pom.xml diff --git a/java/flight/flight-sql-jdbc-core/pom.xml b/java/flight/flight-sql-jdbc-core/pom.xml index 34efb04fa8702..cbeaa88f1e2f7 100644 --- a/java/flight/flight-sql-jdbc-core/pom.xml +++ b/java/flight/flight-sql-jdbc-core/pom.xml @@ -16,7 +16,7 @@ arrow-flight org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml b/java/flight/flight-sql-jdbc-driver/pom.xml index 69e3bfad8d883..263538ba48b37 100644 --- a/java/flight/flight-sql-jdbc-driver/pom.xml +++ b/java/flight/flight-sql-jdbc-driver/pom.xml @@ -16,7 +16,7 @@ arrow-flight org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index e49cb8fa4dd5c..09100d9731ad5 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -14,7 +14,7 @@ arrow-flight org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT ../pom.xml diff --git a/java/flight/pom.xml b/java/flight/pom.xml index fd65c054d971e..7ddda94f77b49 100644 --- a/java/flight/pom.xml +++ b/java/flight/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT 4.0.0 diff --git a/java/format/pom.xml b/java/format/pom.xml index 58dd21e3288f2..3f581311e20ea 100644 --- a/java/format/pom.xml +++ b/java/format/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT arrow-format diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index e87ba2bb4eb88..cfda0b0c527a9 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT org.apache.arrow.gandiva diff --git a/java/memory/memory-core/pom.xml b/java/memory/memory-core/pom.xml index 5d1886032b3e8..8f28699045383 100644 --- a/java/memory/memory-core/pom.xml +++ b/java/memory/memory-core/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT 4.0.0 diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml index 56db6924bf746..307f6ad81a0f1 100644 --- a/java/memory/memory-netty/pom.xml +++ b/java/memory/memory-netty/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT 4.0.0 diff --git a/java/memory/memory-unsafe/pom.xml b/java/memory/memory-unsafe/pom.xml index 68bd61e742741..9f8137308195d 100644 --- a/java/memory/memory-unsafe/pom.xml +++ b/java/memory/memory-unsafe/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT 4.0.0 diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 6fc9d11730eaf..c10263b97f1bf 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT arrow-memory Arrow Memory diff --git a/java/performance/pom.xml b/java/performance/pom.xml index 6aa1599425ad8..102832491ec05 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -14,7 +14,7 @@ arrow-java-root org.apache.arrow - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT arrow-performance jar diff --git a/java/pom.xml b/java/pom.xml index caf470ee2b2b2..2a9997b7012b7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT pom Apache Arrow Java Root POM diff --git a/java/tools/pom.xml b/java/tools/pom.xml index ce6243e98dacf..128825c224369 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT arrow-tools Arrow Tools diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 0cfa421fbf7d4..4c8bf1e594aa4 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 14.0.1-SNAPSHOT + 15.0.0-SNAPSHOT arrow-vector Arrow Vectors diff --git a/js/package.json b/js/package.json index 3bad73b6fdf76..33bc4849903ef 100644 --- a/js/package.json +++ b/js/package.json @@ -124,5 +124,5 @@ "engines": { "node": ">=12.0" }, - "version": "14.0.1-SNAPSHOT" + "version": "15.0.0-SNAPSHOT" } diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index df20ad9680707..47d2acd613f8b 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -94,7 +94,7 @@ endfunction() set(CMAKE_CXX_STANDARD 17) -set(MLARROW_VERSION "14.0.1-SNAPSHOT") +set(MLARROW_VERSION "15.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MLARROW_BASE_VERSION "${MLARROW_VERSION}") project(mlarrow VERSION "${MLARROW_BASE_VERSION}") diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 95d23933a88b7..86285c3dc381b 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -21,7 +21,7 @@ cmake_minimum_required(VERSION 3.16) project(pyarrow) -set(PYARROW_VERSION "14.0.1-SNAPSHOT") +set(PYARROW_VERSION "15.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" PYARROW_BASE_VERSION "${PYARROW_VERSION}") # Running from a Python sdist tarball diff --git a/python/setup.py b/python/setup.py index 4e45b8c68fe16..b1c825d84d5a9 100755 --- a/python/setup.py +++ b/python/setup.py @@ -407,7 +407,7 @@ def get_outputs(self): # If the event of not running from a git clone (e.g. from a git archive # or a Python sdist), see if we can set the version number ourselves -default_version = '14.0.1-SNAPSHOT' +default_version = '15.0.0-SNAPSHOT' if (not os.path.exists('../.git') and not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')): os.environ['SETUPTOOLS_SCM_PRETEND_VERSION'] = \ diff --git a/r/NEWS.md b/r/NEWS.md index d2d5f08288cb8..e00c6b51b597d 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -21,8 +21,6 @@ # arrow 14.0.0 -# arrow 14.0.0 - ## New features * When reading partitioned CSV datasets and supplying a schema to diff --git a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb index 86d8ffba80c90..8551b647cb86f 100644 --- a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb +++ b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowCUDA - VERSION = "14.0.1-SNAPSHOT" + VERSION = "15.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb index b0a29b7f7b365..acfdd675687be 100644 --- a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb +++ b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowDataset - VERSION = "14.0.1-SNAPSHOT" + VERSION = "15.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb index 6734ab6ae2536..3354678e30032 100644 --- a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb +++ b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlightSQL - VERSION = "14.0.1-SNAPSHOT" + VERSION = "15.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight/lib/arrow-flight/version.rb b/ruby/red-arrow-flight/lib/arrow-flight/version.rb index b3b778108199e..f2141a68432e5 100644 --- a/ruby/red-arrow-flight/lib/arrow-flight/version.rb +++ b/ruby/red-arrow-flight/lib/arrow-flight/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlight - VERSION = "14.0.1-SNAPSHOT" + VERSION = "15.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow/lib/arrow/version.rb b/ruby/red-arrow/lib/arrow/version.rb index 222b72d10bb34..235a7df75d672 100644 --- a/ruby/red-arrow/lib/arrow/version.rb +++ b/ruby/red-arrow/lib/arrow/version.rb @@ -16,7 +16,7 @@ # under the License. module Arrow - VERSION = "14.0.1-SNAPSHOT" + VERSION = "15.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-gandiva/lib/gandiva/version.rb b/ruby/red-gandiva/lib/gandiva/version.rb index 576ee097743db..6a1835f0e50e8 100644 --- a/ruby/red-gandiva/lib/gandiva/version.rb +++ b/ruby/red-gandiva/lib/gandiva/version.rb @@ -16,7 +16,7 @@ # under the License. module Gandiva - VERSION = "14.0.1-SNAPSHOT" + VERSION = "15.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-parquet/lib/parquet/version.rb b/ruby/red-parquet/lib/parquet/version.rb index 17a9157a36047..c5a945c4e4297 100644 --- a/ruby/red-parquet/lib/parquet/version.rb +++ b/ruby/red-parquet/lib/parquet/version.rb @@ -16,7 +16,7 @@ # under the License. module Parquet - VERSION = "14.0.1-SNAPSHOT" + VERSION = "15.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") From aa9f8ecdaf3cb8eb2fb478abff32fb96eb350305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sun, 12 Nov 2023 13:56:02 -0600 Subject: [PATCH 36/57] MINOR: [Dev] Add Bryce Mecum to list of collaborators (#38678) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Disclaimer: is a colleague of mine. Opening PR as documented on: https://arrow.apache.org/docs/developers/reviewing.html#collaborators Bryce Mecum (@ amoeba) has been helping with the project for some time and usually helps reviewing breaking changes and critical fixes in order to properly label the issues for the release. It would be helpful if he can add the corresponding labels and assign issues. ### What changes are included in this PR? Add to list of collaborators. ### Are these changes tested? No ### Are there any user-facing changes? No Authored-by: Raúl Cumplido Signed-off-by: Sutou Kouhei --- .asf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.asf.yaml b/.asf.yaml index 4693631be38b3..d2522ecae0b43 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -25,6 +25,7 @@ github: - davisusanibar - felipecrv - js8544 + - amoeba notifications: commits: commits@arrow.apache.org From 79c72a67f90719fb1af315a9b2b8caa25d667775 Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Mon, 13 Nov 2023 14:45:25 +0100 Subject: [PATCH 37/57] GH-38528: [Python][Compute] Describe strptime format semantics (#38665) ### Rationale for this change To prevent possible confusion with the compute strptime function, we now explicitly mention that the C/C++ semantics are followed. ### What changes are included in this PR? The documentation of the `format` parameter of the `strptime` function is expanded. ### Are these changes tested? N/A documentation change only. ### Are there any user-facing changes? Just the documentation. * Closes: #38528 Lead-authored-by: Dan Homola Co-authored-by: Dan Homola Co-authored-by: Rok Mihevc Signed-off-by: Rok Mihevc --- python/pyarrow/_compute.pyx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/pyarrow/_compute.pyx b/python/pyarrow/_compute.pyx index 25f77d8160ea8..51dfdbf8ebbbe 100644 --- a/python/pyarrow/_compute.pyx +++ b/python/pyarrow/_compute.pyx @@ -1682,6 +1682,9 @@ class StrptimeOptions(_StrptimeOptions): ---------- format : str Pattern for parsing input strings as timestamps, such as "%Y/%m/%d". + Note that the semantics of the format follow the C/C++ strptime, not the Python one. + There are differences in behavior, for example how the "%y" placeholder + handles years with less than four digits. unit : str Timestamp unit of the output. Accepted values are "s", "ms", "us", "ns". From e18ace315226525cf624b64f6c1244aa06d67e75 Mon Sep 17 00:00:00 2001 From: sgilmore10 <74676073+sgilmore10@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:05:44 -0500 Subject: [PATCH 38/57] GH-38415: [MATLAB] Add indexing "slice" method to C++ Array Proxy class (#38674) ### Rationale for this change In order to support MATLAB indexing semantics on the `arrow.array.Array` classes, we first need to add a method for "slicing" an Arrow Array to the C++ Array Proxy classes. ### What changes are included in this PR? 1. Added a `slice` method to the C++ Arrow Array Proxy class. 2. Added a `slice` method to `arrow.array.Array`. This method invokes the `slice` method on the proxy class. **Example** ```matlab >> a = arrow.array([1 2 NaN 4 5 NaN 7]) a = Float64Array with 7 elements and 2 null values: 1 | 2 | null | 4 | 5 | null | 7 % Note: MATLAB uses 1-based indexing. >> slice = a.slice(int64(2), int64(4)) slice = Float64Array with 4 elements and 1 null value: 2 | null | 4 | 5 ``` ### Are these changes tested? Yes, added a new test file called `tSlice.m`. ### Are there any user-facing changes? The `slice` method is public, but [hidden](https://www.mathworks.com/help/matlab/matlab_oop/method-attributes.html). * Closes: #38415 Authored-by: Sarah Gilmore Signed-off-by: Kevin Gurney --- .../src/cpp/arrow/matlab/array/proxy/array.cc | 36 ++++- .../src/cpp/arrow/matlab/array/proxy/array.h | 2 + matlab/src/cpp/arrow/matlab/error/error.h | 4 + matlab/src/cpp/arrow/matlab/index/validate.cc | 16 ++ matlab/src/cpp/arrow/matlab/index/validate.h | 3 + matlab/src/matlab/+arrow/+array/Array.m | 10 ++ matlab/test/arrow/array/tSlice.m | 138 ++++++++++++++++++ 7 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 matlab/test/arrow/array/tSlice.m diff --git a/matlab/src/cpp/arrow/matlab/array/proxy/array.cc b/matlab/src/cpp/arrow/matlab/array/proxy/array.cc index 09d9473df4795..bc5ab093b4534 100644 --- a/matlab/src/cpp/arrow/matlab/array/proxy/array.cc +++ b/matlab/src/cpp/arrow/matlab/array/proxy/array.cc @@ -18,8 +18,10 @@ #include "arrow/util/utf8.h" #include "arrow/matlab/array/proxy/array.h" +#include "arrow/matlab/array/proxy/wrap.h" #include "arrow/matlab/bit/unpack.h" #include "arrow/matlab/error/error.h" +#include "arrow/matlab/index/validate.h" #include "arrow/matlab/type/proxy/wrap.h" #include "arrow/pretty_print.h" #include "arrow/type_traits.h" @@ -38,7 +40,7 @@ namespace arrow::matlab::array::proxy { REGISTER_METHOD(Array, getValid); REGISTER_METHOD(Array, getType); REGISTER_METHOD(Array, isEqual); - + REGISTER_METHOD(Array, slice); } std::shared_ptr Array::unwrap() { @@ -144,4 +146,36 @@ namespace arrow::matlab::array::proxy { mda::ArrayFactory factory; context.outputs[0] = factory.createScalar(is_equal); } + + void Array::slice(libmexclass::proxy::method::Context& context) { + namespace mda = ::matlab::data; + + mda::StructArray opts = context.inputs[0]; + const mda::TypedArray offset_mda = opts[0]["Offset"]; + const mda::TypedArray length_mda = opts[0]["Length"]; + + const auto matlab_offset = int64_t(offset_mda[0]); + MATLAB_ERROR_IF_NOT_OK_WITH_CONTEXT(arrow::matlab::index::validateSliceOffset(matlab_offset), + context, error::ARRAY_SLICE_NON_POSITIVE_OFFSET); + + // Note: MATLAB uses 1-based indexing, so subtract 1. + const int64_t offset = matlab_offset - 1; + const int64_t length = int64_t(length_mda[0]); + MATLAB_ERROR_IF_NOT_OK_WITH_CONTEXT(arrow::matlab::index::validateSliceLength(length), + context, error::ARRAY_SLICE_NEGATIVE_LENGTH); + + auto sliced_array = array->Slice(offset, length); + const auto type_id = static_cast(sliced_array->type_id()); + MATLAB_ASSIGN_OR_ERROR_WITH_CONTEXT(auto sliced_array_proxy, + array::proxy::wrap(sliced_array), + context, error::ARRAY_SLICE_FAILED_TO_CREATE_ARRAY_PROXY); + + const auto proxy_id = libmexclass::proxy::ProxyManager::manageProxy(sliced_array_proxy); + + mda::ArrayFactory factory; + mda::StructArray output = factory.createStructArray({1, 1}, {"ProxyID", "TypeID"}); + output[0]["ProxyID"] = factory.createScalar(proxy_id); + output[0]["TypeID"] = factory.createScalar(type_id); + context.outputs[0] = output; + } } diff --git a/matlab/src/cpp/arrow/matlab/array/proxy/array.h b/matlab/src/cpp/arrow/matlab/array/proxy/array.h index 0ab7b279bc92e..1e3164ed01a72 100644 --- a/matlab/src/cpp/arrow/matlab/array/proxy/array.h +++ b/matlab/src/cpp/arrow/matlab/array/proxy/array.h @@ -44,6 +44,8 @@ class Array : public libmexclass::proxy::Proxy { void isEqual(libmexclass::proxy::method::Context& context); + void slice(libmexclass::proxy::method::Context& context); + std::shared_ptr array; }; diff --git a/matlab/src/cpp/arrow/matlab/error/error.h b/matlab/src/cpp/arrow/matlab/error/error.h index 5aa8f05c8c315..e6be411b62a05 100644 --- a/matlab/src/cpp/arrow/matlab/error/error.h +++ b/matlab/src/cpp/arrow/matlab/error/error.h @@ -206,4 +206,8 @@ namespace arrow::matlab::error { static const char* ARRAY_VALIDATE_MINIMAL_FAILED = "arrow:array:ValidateMinimalFailed"; static const char* ARRAY_VALIDATE_FULL_FAILED = "arrow:array:ValidateFullFailed"; static const char* ARRAY_VALIDATE_UNSUPPORTED_ENUM = "arrow:array:ValidateUnsupportedEnum"; + static const char* ARRAY_SLICE_NON_POSITIVE_OFFSET = "arrow:array:slice:NonPositiveOffset"; + static const char* ARRAY_SLICE_NEGATIVE_LENGTH = "arrow:array:slice:NegativeLength"; + static const char* ARRAY_SLICE_FAILED_TO_CREATE_ARRAY_PROXY = "arrow:array:slice:FailedToCreateArrayProxy"; + } diff --git a/matlab/src/cpp/arrow/matlab/index/validate.cc b/matlab/src/cpp/arrow/matlab/index/validate.cc index b24653f1b814c..84e8e424e171f 100644 --- a/matlab/src/cpp/arrow/matlab/index/validate.cc +++ b/matlab/src/cpp/arrow/matlab/index/validate.cc @@ -53,4 +53,20 @@ namespace arrow::matlab::index { } return arrow::Status::OK(); } + + arrow::Status validateSliceOffset(const int64_t matlab_offset) { + if (matlab_offset < 1) { + const std::string msg = "Slice offset must be positive"; + return arrow::Status::Invalid(std::move(msg)); + } + return arrow::Status::OK(); + } + + arrow::Status validateSliceLength(const int64_t length) { + if (length < 0) { + const std::string msg = "Slice length must be nonnegative"; + return arrow::Status::Invalid(std::move(msg)); + } + return arrow::Status::OK(); + } } \ No newline at end of file diff --git a/matlab/src/cpp/arrow/matlab/index/validate.h b/matlab/src/cpp/arrow/matlab/index/validate.h index 40e109c19e9ef..2fa88ef8f1b5a 100644 --- a/matlab/src/cpp/arrow/matlab/index/validate.h +++ b/matlab/src/cpp/arrow/matlab/index/validate.h @@ -23,4 +23,7 @@ namespace arrow::matlab::index { arrow::Status validateNonEmptyContainer(const int32_t num_fields); arrow::Status validateInRange(const int32_t matlab_index, const int32_t num_fields); + arrow::Status validateSliceOffset(const int64_t matlab_offset); + arrow::Status validateSliceLength(const int64_t length); + } \ No newline at end of file diff --git a/matlab/src/matlab/+arrow/+array/Array.m b/matlab/src/matlab/+arrow/+array/Array.m index 293ad87ad4316..4402055932b60 100644 --- a/matlab/src/matlab/+arrow/+array/Array.m +++ b/matlab/src/matlab/+arrow/+array/Array.m @@ -98,4 +98,14 @@ function displayScalarObject(obj) tf = obj.Proxy.isEqual(proxyIDs); end end + + methods (Hidden) + function array = slice(obj, offset, length) + sliceStruct = struct(Offset=offset, Length=length); + arrayStruct = obj.Proxy.slice(sliceStruct); + traits = arrow.type.traits.traits(arrow.type.ID(arrayStruct.TypeID)); + proxy = libmexclass.proxy.Proxy(Name=traits.ArrayProxyClassName, ID=arrayStruct.ProxyID); + array = traits.ArrayConstructor(proxy); + end + end end diff --git a/matlab/test/arrow/array/tSlice.m b/matlab/test/arrow/array/tSlice.m new file mode 100644 index 0000000000000..c99503371a41c --- /dev/null +++ b/matlab/test/arrow/array/tSlice.m @@ -0,0 +1,138 @@ +%TSLICE Unit tests verifying the behavior of arrow.array.Array's slice +%method. + +% Licensed to the Apache Software Foundation (ASF) under one or more +% contributor license agreements. See the NOTICE file distributed with +% this work for additional information regarding copyright ownership. +% The ASF licenses this file to you under the Apache License, Version +% 2.0 (the "License"); you may not use this file except in compliance +% with the License. You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +% implied. See the License for the specific language governing +% permissions and limitations under the License. + +classdef tSlice < matlab.unittest.TestCase + + methods(Test) + function BooleanArray(testCase) + % Verify the slice method returns the expected array when + % called on a Boolean Array. + boolArray = arrow.array([true true false true true false], Valid=[1 2 3 6]); + slice = boolArray.slice(int64(2), int64(4)); + testCase.verifyEqual(slice.NumElements, int64(4)); + testCase.verifyEqual(slice.Valid, [true; true; false; false]); + testCase.verifyEqual(toMATLAB(slice), [true; false; false; false]); + end + + function NumericArray(testCase) + % Verify the slice method returns the expected array when + % called on a Numeric Array. + float64Array = arrow.array(1:10, Valid=[2 3 4 5 8 10]); + slice = float64Array.slice(int64(4), int64(5)); + testCase.verifyEqual(slice.NumElements, int64(5)); + testCase.verifyEqual(slice.Valid, [true; true; false; false; true]); + testCase.verifyEqual(toMATLAB(slice), [4; 5; NaN; NaN; 8]); + end + + function DateArray(testCase) + % Verify the slice method returns the expected array when + % called on a Date Array. + import arrow.array.Date32Array + dates = datetime(2023, 11, 8:16); + date32Array = Date32Array.fromMATLAB(dates, Valid=[4 5 6 9]); + slice = date32Array.slice(int64(3), int64(4)); + testCase.verifyEqual(slice.NumElements, int64(4)); + testCase.verifyEqual(slice.Valid, [false; true; true; true]); + expected = [NaT; dates(4:6)']; + testCase.verifyEqual(toMATLAB(slice), expected); + end + + function TimeArray(testCase) + % Verify the slice method returns the expected array when + % called on a Time Array. + times = seconds(10:20); + time64Array = arrow.array(times, Valid=[2 4 6 7 8 10]); + slice = time64Array.slice(int64(5), int64(6)); + testCase.verifyEqual(slice.NumElements, int64(6)); + testCase.verifyEqual(slice.Valid, [false; true; true; true; false; true]); + expected = [NaN; times(6:8)'; NaN; times(10)]; + testCase.verifyEqual(toMATLAB(slice), expected); + end + + function TimestampArray(testCase) + % Verify the slice method returns the expected array when + % called on a TimestampArray. + dates = datetime(2023, 11, 8:16); + date32Array = arrow.array(dates, Valid=[1 2 4 5 6 8]); + slice = date32Array.slice(int64(5), int64(3)); + testCase.verifyEqual(slice.NumElements, int64(3)); + testCase.verifyEqual(slice.Valid, [true; true; false]); + expected = [dates(5:6)'; NaT]; + testCase.verifyEqual(toMATLAB(slice), expected); + end + + function StringArray(testCase) + % Verify the slice method returns the expected array when + % called on a StringArray. + stringArray = arrow.array(["a" "b" "c" "d" "e" "f" "g"], Valid=[1 3 4 5 6]); + slice = stringArray.slice(int64(2), int64(3)); + testCase.verifyEqual(slice.NumElements, int64(3)); + testCase.verifyEqual(slice.Valid, [false; true; true]); + expected = [missing; "c"; "d"]; + testCase.verifyEqual(toMATLAB(slice), expected); + end + + function ListArray(testCase) + % Verify the slice method returns the expected array when + % called on a ListArray. + cellArray = {missing, [1, 2, 3], missing, [4, NaN], [6, 7, 8], missing}; + listArray = arrow.array(cellArray); + slice = listArray.slice(int64(2), int64(4)); + testCase.verifyEqual(slice.NumElements, int64(4)); + testCase.verifyEqual(slice.Valid, [true; false; true; true]); + expected = {[1; 2; 3]; missing; [4; NaN]; [6; 7; 8]}; + testCase.verifyEqual(toMATLAB(slice), expected); + end + + function StructArray(testCase) + % Verify the slice method returns the expected array when + % called on a StructArray. + numbers = [NaN; 2; 3; 4; 5; 6; 7; NaN; 9; 10]; + text = ["a"; missing; "c"; "d"; "e"; missing; "g"; "h"; "i"; "j"]; + t = table(numbers, text); + structArray = arrow.array(t, Valid=[1 2 3 6 7 8 10]); + slice = structArray.slice(int64(5), int64(4)); + testCase.verifyEqual(slice.NumElements, int64(4)); + testCase.verifyEqual(slice.Valid, [false; true; true; true]); + expected = t(5:8, :); + expected.numbers(1) = NaN; + expected.text(1) = missing; + testCase.verifyEqual(toMATLAB(slice), expected); + end + + function NonPositiveOffsetError(testCase) + % Verify the slice method throws an error whose identifier is + % "arrow:array:slice:NonPositiveOffset" if given a non-positive + % value as the offset. + array = arrow.array(1:10); + fcn = @() array.slice(int64(0), int64(2)); + testCase.verifyError(fcn, "arrow:array:slice:NonPositiveOffset"); + fcn = @() array.slice(int64(-1), int64(2)); + testCase.verifyError(fcn, "arrow:array:slice:NonPositiveOffset"); + end + + function NegativeLengthError(testCase) + % Verify the slice method throws an error whose identifier is + % "arrow:array:slice:NegativeLength" if given a negative value + % as the length. + array = arrow.array(1:10); + fcn = @() array.slice(int64(1), int64(-1)); + testCase.verifyError(fcn, "arrow:array:slice:NegativeLength"); + end + end +end \ No newline at end of file From 1ff43ab5ee13de5c3130acf10c7aa8eb9680baab Mon Sep 17 00:00:00 2001 From: Tim Schaub Date: Mon, 13 Nov 2023 10:01:01 -0700 Subject: [PATCH 39/57] GH-38516: [Go][Parquet] Increment the number of rows written when appending a new row group (#38517) ### Rationale for this change This makes it so the `NumRows` method on the `file.Writer` reports the total number of rows written across multiple row groups. ### Are these changes tested? A regression test is added that asserts that the total number of rows written matches expectations. * Closes: #38516 Authored-by: Tim Schaub Signed-off-by: Matt Topol --- go/parquet/file/file_writer.go | 1 + go/parquet/file/file_writer_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go/parquet/file/file_writer.go b/go/parquet/file/file_writer.go index 48eb8f44d167b..1d7f7840dac50 100644 --- a/go/parquet/file/file_writer.go +++ b/go/parquet/file/file_writer.go @@ -121,6 +121,7 @@ func (fw *Writer) AppendRowGroup() SerialRowGroupWriter { func (fw *Writer) appendRowGroup(buffered bool) *rowGroupWriter { if fw.rowGroupWriter != nil { + fw.nrows += fw.rowGroupWriter.nrows fw.rowGroupWriter.Close() } fw.rowGroups++ diff --git a/go/parquet/file/file_writer_test.go b/go/parquet/file/file_writer_test.go index beceff9732584..f32e403a8d534 100644 --- a/go/parquet/file/file_writer_test.go +++ b/go/parquet/file/file_writer_test.go @@ -97,6 +97,8 @@ func (t *SerializeTestSuite) fileSerializeTest(codec compress.Compression, expec writer.Close() nrows := t.numRowGroups * t.rowsPerRG + t.EqualValues(nrows, writer.NumRows()) + reader, err := file.NewParquetReader(bytes.NewReader(sink.Bytes())) t.NoError(err) t.Equal(t.numCols, reader.MetaData().Schema.NumColumns()) From bff5fb95ac987354ecd7d69466e11b9f33a67d08 Mon Sep 17 00:00:00 2001 From: Ben Harkins <60872452+benibus@users.noreply.github.com> Date: Mon, 13 Nov 2023 16:02:49 -0500 Subject: [PATCH 40/57] GH-37582: [Go][Parquet] Implement Float16 logical type (#37599) ### Rationale for this change There is an active proposal for a Float16 logical type in Parquet (https://github.com/apache/parquet-format/pull/184) with C++/Python implementations in progress (https://github.com/apache/arrow/pull/36073), so we should add one for Go as well. ### What changes are included in this PR? - [x] Adds `LogicalType` definitions and methods for `Float16` - [x] Adds support for `Float16` column statistics and comparators - [x] Adds support for interchange between Parquet and Arrow's half-precision float ### Are these changes tested? Yes ### Are there any user-facing changes? Yes * Closes: #37582 Authored-by: benibus Signed-off-by: Matt Topol --- go/arrow/float16/float16.go | 46 ++- go/arrow/float16/float16_test.go | 43 ++ go/parquet/file/column_writer_types.gen.go | 13 +- .../file/column_writer_types.gen.go.tmpl | 28 +- .../gen-go/parquet/GoUnusedProtection__.go | 2 +- .../internal/gen-go/parquet/parquet-consts.go | 9 +- go/parquet/internal/gen-go/parquet/parquet.go | 382 +++++++++++++++--- go/parquet/internal/testutils/random.go | 23 ++ go/parquet/internal/testutils/random_arrow.go | 23 ++ go/parquet/metadata/statistics.go | 45 ++- go/parquet/metadata/statistics_test.go | 62 ++- go/parquet/metadata/statistics_types.gen.go | 315 +++++++++++++++ .../metadata/statistics_types.gen.go.tmpl | 47 ++- go/parquet/metadata/statistics_types.tmpldata | 60 +++ go/parquet/pqarrow/column_readers.go | 16 + go/parquet/pqarrow/encode_arrow.go | 25 ++ go/parquet/pqarrow/encode_arrow_test.go | 9 + go/parquet/pqarrow/schema.go | 6 + go/parquet/pqarrow/schema_test.go | 20 +- go/parquet/schema/logical_types.go | 46 +++ go/parquet/schema/logical_types_test.go | 12 + go/parquet/schema/reflection.go | 6 + go/parquet/schema/reflection_test.go | 7 + go/parquet/schema/schema_element_test.go | 9 +- go/parquet/schema/schema_test.go | 4 + 25 files changed, 1183 insertions(+), 75 deletions(-) create mode 100644 go/parquet/metadata/statistics_types.tmpldata diff --git a/go/arrow/float16/float16.go b/go/arrow/float16/float16.go index 4e03d13df0cae..e0192495eb971 100644 --- a/go/arrow/float16/float16.go +++ b/go/arrow/float16/float16.go @@ -17,6 +17,7 @@ package float16 import ( + "encoding/binary" "math" "strconv" ) @@ -29,6 +30,11 @@ type Num struct { bits uint16 } +var ( + MaxNum = Num{bits: 0b0111101111111111} + MinNum = MaxNum.Negate() +) + // New creates a new half-precision floating point value from the provided // float32 value. func New(f float32) Num { @@ -86,6 +92,11 @@ func (n Num) Div(rhs Num) Num { return New(n.Float32() / rhs.Float32()) } +// Equal returns true if the value represented by n is == other +func (n Num) Equal(other Num) bool { + return n.Float32() == other.Float32() +} + // Greater returns true if the value represented by n is > other func (n Num) Greater(other Num) bool { return n.Float32() > other.Float32() @@ -152,14 +163,39 @@ func (n Num) Abs() Num { } func (n Num) Sign() int { - f := n.Float32() - if f > 0 { - return 1 - } else if f == 0 { + if n.IsZero() { return 0 + } else if n.Signbit() { + return -1 } - return -1 + return 1 } +func (n Num) Signbit() bool { return (n.bits & 0x8000) != 0 } + +func (n Num) IsNaN() bool { return (n.bits & 0x7fff) > 0x7c00 } + +func (n Num) IsZero() bool { return (n.bits & 0x7fff) == 0 } + func (f Num) Uint16() uint16 { return f.bits } func (f Num) String() string { return strconv.FormatFloat(float64(f.Float32()), 'g', -1, 32) } + +func Inf() Num { return Num{bits: 0x7c00} } + +func NaN() Num { return Num{bits: 0x7fff} } + +func FromBits(src uint16) Num { return Num{bits: src} } + +func FromLEBytes(src []byte) Num { + return Num{bits: binary.LittleEndian.Uint16(src)} +} + +func (f Num) PutLEBytes(dst []byte) { + binary.LittleEndian.PutUint16(dst, f.bits) +} + +func (f Num) ToLEBytes() []byte { + dst := make([]byte, 2) + f.PutLEBytes(dst) + return dst +} diff --git a/go/arrow/float16/float16_test.go b/go/arrow/float16/float16_test.go index 55c3ea8b30404..cfde440c5f9e4 100644 --- a/go/arrow/float16/float16_test.go +++ b/go/arrow/float16/float16_test.go @@ -238,6 +238,7 @@ func TestSign(t *testing.T) { }{ {Num{bits: 0x4580}, 1}, // 5.5 {Num{bits: 0x0000}, 0}, // 0 + {Num{bits: 0x8000}, 0}, // -0 {Num{bits: 0xC580}, -1}, // -5.5 } { t.Run("sign", func(t *testing.T) { @@ -248,3 +249,45 @@ func TestSign(t *testing.T) { }) } } + +func TestSignbit(t *testing.T) { + for _, tc := range []struct { + n Num + want bool + }{ + {Num{bits: 0x4580}, false}, // 5.5 + {Num{bits: 0x0000}, false}, // 0 + {Num{bits: 0x8000}, true}, // -0 + {Num{bits: 0xC580}, true}, // -5.5 + } { + t.Run("signbit", func(t *testing.T) { + n := tc.n.Signbit() + if got, want := n, tc.want; got != want { + t.Fatalf("invalid value. got=%v, want=%v", got, want) + } + }) + } +} + +func TestIsNaN(t *testing.T) { + for _, tc := range []struct { + n Num + want bool + }{ + {NaN(), true}, + {NaN().Negate(), true}, + {Inf(), false}, + {Inf().Negate(), false}, + {Num{bits: 0x7c01}, true}, // nan + {Num{bits: 0xfc01}, true}, // -nan + {Num{bits: 0x7e00}, true}, // nan + {Num{bits: 0xfe00}, true}, // -nan + } { + t.Run("isnan", func(t *testing.T) { + n := tc.n.IsNaN() + if got, want := n, tc.want; got != want { + t.Fatalf("invalid value. got=%v, want=%v", got, want) + } + }) + } +} diff --git a/go/parquet/file/column_writer_types.gen.go b/go/parquet/file/column_writer_types.gen.go index 8b3be25f3ea29..b4d7954639319 100644 --- a/go/parquet/file/column_writer_types.gen.go +++ b/go/parquet/file/column_writer_types.gen.go @@ -28,6 +28,7 @@ import ( "github.com/apache/arrow/go/v15/parquet/internal/encoding" format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v15/parquet/schema" "golang.org/x/xerrors" ) @@ -1629,7 +1630,11 @@ func (w *FixedLenByteArrayColumnChunkWriter) WriteDictIndices(indices arrow.Arra func (w *FixedLenByteArrayColumnChunkWriter) writeValues(values []parquet.FixedLenByteArray, numNulls int64) { w.currentEncoder.(encoding.FixedLenByteArrayEncoder).Put(values) if w.pageStatistics != nil { - w.pageStatistics.(*metadata.FixedLenByteArrayStatistics).Update(values, numNulls) + if w.Descr().LogicalType().Equals(schema.Float16LogicalType{}) { + w.pageStatistics.(*metadata.Float16Statistics).Update(values, numNulls) + } else { + w.pageStatistics.(*metadata.FixedLenByteArrayStatistics).Update(values, numNulls) + } } } @@ -1641,7 +1646,11 @@ func (w *FixedLenByteArrayColumnChunkWriter) writeValuesSpaced(spacedValues []pa } if w.pageStatistics != nil { nulls := numValues - numRead - w.pageStatistics.(*metadata.FixedLenByteArrayStatistics).UpdateSpaced(spacedValues, validBits, validBitsOffset, nulls) + if w.Descr().LogicalType().Equals(schema.Float16LogicalType{}) { + w.pageStatistics.(*metadata.Float16Statistics).UpdateSpaced(spacedValues, validBits, validBitsOffset, nulls) + } else { + w.pageStatistics.(*metadata.FixedLenByteArrayStatistics).UpdateSpaced(spacedValues, validBits, validBitsOffset, nulls) + } } } diff --git a/go/parquet/file/column_writer_types.gen.go.tmpl b/go/parquet/file/column_writer_types.gen.go.tmpl index 7df69b4a219c6..70bcfe679eb92 100644 --- a/go/parquet/file/column_writer_types.gen.go.tmpl +++ b/go/parquet/file/column_writer_types.gen.go.tmpl @@ -18,7 +18,7 @@ package file import ( "fmt" - + "github.com/apache/arrow/go/v15/parquet" "github.com/apache/arrow/go/v15/parquet/metadata" "github.com/apache/arrow/go/v15/parquet/internal/encoding" @@ -83,7 +83,7 @@ func (w *{{.Name}}ColumnChunkWriter) WriteBatch(values []{{.name}}, defLevels, r // writes a large number of values, the DataPage size can be much above the limit. // The purpose of this chunking is to bound this. Even if a user writes large number // of values, the chunking will ensure the AddDataPage() is called at a reasonable - // pagesize limit + // pagesize limit var n int64 switch { case defLevels != nil: @@ -107,7 +107,7 @@ func (w *{{.Name}}ColumnChunkWriter) WriteBatch(values []{{.name}}, defLevels, r valueOffset += toWrite w.checkDictionarySizeLimit() }) - return + return } // WriteBatchSpaced writes a batch of repetition levels, definition levels, and values to the @@ -132,7 +132,7 @@ func (w *{{.Name}}ColumnChunkWriter) WriteBatchSpaced(values []{{.name}}, defLev length = len(values) } doBatches(int64(length), w.props.WriteBatchSize(), func(offset, batch int64) { - var vals []{{.name}} + var vals []{{.name}} info := w.maybeCalculateValidityBits(levelSliceOrNil(defLevels, offset, batch), batch) w.writeLevelsSpaced(batch, levelSliceOrNil(defLevels, offset, batch), levelSliceOrNil(repLevels, offset, batch)) @@ -165,7 +165,7 @@ func (w *{{.Name}}ColumnChunkWriter) WriteDictIndices(indices arrow.Array, defLe } } }() - + valueOffset := int64(0) length := len(defLevels) if defLevels == nil { @@ -193,14 +193,22 @@ func (w *{{.Name}}ColumnChunkWriter) WriteDictIndices(indices arrow.Array, defLe valueOffset += info.numSpaced() }) - + return } func (w *{{.Name}}ColumnChunkWriter) writeValues(values []{{.name}}, numNulls int64) { w.currentEncoder.(encoding.{{.Name}}Encoder).Put(values) if w.pageStatistics != nil { +{{- if ne .Name "FixedLenByteArray"}} w.pageStatistics.(*metadata.{{.Name}}Statistics).Update(values, numNulls) +{{- else}} + if w.Descr().LogicalType().Equals(schema.Float16LogicalType{}) { + w.pageStatistics.(*metadata.Float16Statistics).Update(values, numNulls) + } else { + w.pageStatistics.(*metadata.{{.Name}}Statistics).Update(values, numNulls) + } +{{- end}} } } @@ -212,7 +220,15 @@ func (w *{{.Name}}ColumnChunkWriter) writeValuesSpaced(spacedValues []{{.name}}, } if w.pageStatistics != nil { nulls := numValues - numRead +{{- if ne .Name "FixedLenByteArray"}} w.pageStatistics.(*metadata.{{.Name}}Statistics).UpdateSpaced(spacedValues, validBits, validBitsOffset, nulls) +{{- else}} + if w.Descr().LogicalType().Equals(schema.Float16LogicalType{}) { + w.pageStatistics.(*metadata.Float16Statistics).UpdateSpaced(spacedValues, validBits, validBitsOffset, nulls) + } else { + w.pageStatistics.(*metadata.{{.Name}}Statistics).UpdateSpaced(spacedValues, validBits, validBitsOffset, nulls) + } +{{- end}} } } diff --git a/go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go b/go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go index 1de0c8dee4cbd..01f1eb5aa99e3 100644 --- a/go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go +++ b/go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go @@ -1,4 +1,4 @@ -// Code generated by Thrift Compiler (0.16.0). DO NOT EDIT. +// Code generated by Thrift Compiler (0.18.1). DO NOT EDIT. package parquet diff --git a/go/parquet/internal/gen-go/parquet/parquet-consts.go b/go/parquet/internal/gen-go/parquet/parquet-consts.go index d4a63b22b890a..ab0a73c596e7d 100644 --- a/go/parquet/internal/gen-go/parquet/parquet-consts.go +++ b/go/parquet/internal/gen-go/parquet/parquet-consts.go @@ -1,21 +1,28 @@ -// Code generated by Thrift Compiler (0.16.0). DO NOT EDIT. +// Code generated by Thrift Compiler (0.18.1). DO NOT EDIT. package parquet import ( "bytes" "context" + "errors" "fmt" "time" thrift "github.com/apache/thrift/lib/go/thrift" + "strings" + "regexp" ) // (needed to ensure safety because of naive import list construction.) var _ = thrift.ZERO var _ = fmt.Printf +var _ = errors.New var _ = context.Background var _ = time.Now var _ = bytes.Equal +// (needed by validator.) +var _ = strings.Contains +var _ = regexp.MatchString func init() { diff --git a/go/parquet/internal/gen-go/parquet/parquet.go b/go/parquet/internal/gen-go/parquet/parquet.go index d4508f8e4529f..9dcedae8888d3 100644 --- a/go/parquet/internal/gen-go/parquet/parquet.go +++ b/go/parquet/internal/gen-go/parquet/parquet.go @@ -1,4 +1,4 @@ -// Code generated by Thrift Compiler (0.16.0). DO NOT EDIT. +// Code generated by Thrift Compiler (0.18.1). DO NOT EDIT. package parquet @@ -10,14 +10,20 @@ import ( "fmt" "time" thrift "github.com/apache/thrift/lib/go/thrift" + "strings" + "regexp" ) // (needed to ensure safety because of naive import list construction.) var _ = thrift.ZERO var _ = fmt.Printf +var _ = errors.New var _ = context.Background var _ = time.Now var _ = bytes.Equal +// (needed by validator.) +var _ = strings.Contains +var _ = regexp.MatchString //Types supported by Parquet. These types are intended to be used in combination //with the encodings to control the on disk storage format. @@ -94,9 +100,10 @@ func (p * Type) Value() (driver.Value, error) { } return int64(*p), nil } -//Common types used by frameworks(e.g. hive, pig) using parquet. This helps map -//between types in those frameworks to the base types in parquet. This is only -//metadata and not needed to read or write the data. +//DEPRECATED: Common types used by frameworks(e.g. hive, pig) using parquet. +//ConvertedType is superseded by LogicalType. This enum should not be extended. +// +//See LogicalTypes.md for conversion between ConvertedType and LogicalType. type ConvertedType int64 const ( ConvertedType_UTF8 ConvertedType = 0 @@ -897,6 +904,9 @@ func (p *Statistics) String() string { return fmt.Sprintf("Statistics(%+v)", *p) } +func (p *Statistics) Validate() error { + return nil +} // Empty structs to use as logical type annotations type StringType struct { } @@ -958,6 +968,9 @@ func (p *StringType) String() string { return fmt.Sprintf("StringType(%+v)", *p) } +func (p *StringType) Validate() error { + return nil +} type UUIDType struct { } @@ -1018,6 +1031,9 @@ func (p *UUIDType) String() string { return fmt.Sprintf("UUIDType(%+v)", *p) } +func (p *UUIDType) Validate() error { + return nil +} type MapType struct { } @@ -1078,6 +1094,9 @@ func (p *MapType) String() string { return fmt.Sprintf("MapType(%+v)", *p) } +func (p *MapType) Validate() error { + return nil +} type ListType struct { } @@ -1138,6 +1157,9 @@ func (p *ListType) String() string { return fmt.Sprintf("ListType(%+v)", *p) } +func (p *ListType) Validate() error { + return nil +} type EnumType struct { } @@ -1198,6 +1220,9 @@ func (p *EnumType) String() string { return fmt.Sprintf("EnumType(%+v)", *p) } +func (p *EnumType) Validate() error { + return nil +} type DateType struct { } @@ -1258,6 +1283,72 @@ func (p *DateType) String() string { return fmt.Sprintf("DateType(%+v)", *p) } +func (p *DateType) Validate() error { + return nil +} +type Float16Type struct { +} + +func NewFloat16Type() *Float16Type { + return &Float16Type{} +} + +func (p *Float16Type) Read(ctx context.Context, iprot thrift.TProtocol) error { + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { break; } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *Float16Type) Write(ctx context.Context, oprot thrift.TProtocol) error { + if err := oprot.WriteStructBegin(ctx, "Float16Type"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) } + return nil +} + +func (p *Float16Type) Equals(other *Float16Type) bool { + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true +} + +func (p *Float16Type) String() string { + if p == nil { + return "" + } + return fmt.Sprintf("Float16Type(%+v)", *p) +} + +func (p *Float16Type) Validate() error { + return nil +} // Logical type to annotate a column that is always null. // // Sometimes when discovering the schema of existing data, values are always @@ -1323,6 +1414,9 @@ func (p *NullType) String() string { return fmt.Sprintf("NullType(%+v)", *p) } +func (p *NullType) Validate() error { + return nil +} // Decimal logical type annotation // // To maintain forward-compatibility in v1, implementations using this logical @@ -1478,6 +1572,9 @@ func (p *DecimalType) String() string { return fmt.Sprintf("DecimalType(%+v)", *p) } +func (p *DecimalType) Validate() error { + return nil +} // Time units for logical types type MilliSeconds struct { } @@ -1539,6 +1636,9 @@ func (p *MilliSeconds) String() string { return fmt.Sprintf("MilliSeconds(%+v)", *p) } +func (p *MilliSeconds) Validate() error { + return nil +} type MicroSeconds struct { } @@ -1599,6 +1699,9 @@ func (p *MicroSeconds) String() string { return fmt.Sprintf("MicroSeconds(%+v)", *p) } +func (p *MicroSeconds) Validate() error { + return nil +} type NanoSeconds struct { } @@ -1659,6 +1762,9 @@ func (p *NanoSeconds) String() string { return fmt.Sprintf("NanoSeconds(%+v)", *p) } +func (p *NanoSeconds) Validate() error { + return nil +} // Attributes: // - MILLIS // - MICROS @@ -1879,6 +1985,9 @@ func (p *TimeUnit) String() string { return fmt.Sprintf("TimeUnit(%+v)", *p) } +func (p *TimeUnit) Validate() error { + return nil +} // Timestamp logical type annotation // // Allowed for physical types: INT64 @@ -2038,6 +2147,9 @@ func (p *TimestampType) String() string { return fmt.Sprintf("TimestampType(%+v)", *p) } +func (p *TimestampType) Validate() error { + return nil +} // Time logical type annotation // // Allowed for physical types: INT32 (millis), INT64 (micros, nanos) @@ -2197,6 +2309,9 @@ func (p *TimeType) String() string { return fmt.Sprintf("TimeType(%+v)", *p) } +func (p *TimeType) Validate() error { + return nil +} // Integer logical type annotation // // bitWidth must be 8, 16, 32, or 64. @@ -2352,6 +2467,9 @@ func (p *IntType) String() string { return fmt.Sprintf("IntType(%+v)", *p) } +func (p *IntType) Validate() error { + return nil +} // Embedded JSON logical type annotation // // Allowed for physical types: BINARY @@ -2415,6 +2533,9 @@ func (p *JsonType) String() string { return fmt.Sprintf("JsonType(%+v)", *p) } +func (p *JsonType) Validate() error { + return nil +} // Embedded BSON logical type annotation // // Allowed for physical types: BINARY @@ -2478,11 +2599,14 @@ func (p *BsonType) String() string { return fmt.Sprintf("BsonType(%+v)", *p) } +func (p *BsonType) Validate() error { + return nil +} // LogicalType annotations to replace ConvertedType. // // To maintain compatibility, implementations using LogicalType for a -// SchemaElement must also set the corresponding ConvertedType from the -// following table. +// SchemaElement must also set the corresponding ConvertedType (if any) +// from the following table. // // Attributes: // - STRING @@ -2498,6 +2622,7 @@ func (p *BsonType) String() string { // - JSON // - BSON // - UUID +// - FLOAT16 type LogicalType struct { STRING *StringType `thrift:"STRING,1" db:"STRING" json:"STRING,omitempty"` MAP *MapType `thrift:"MAP,2" db:"MAP" json:"MAP,omitempty"` @@ -2513,6 +2638,7 @@ type LogicalType struct { JSON *JsonType `thrift:"JSON,12" db:"JSON" json:"JSON,omitempty"` BSON *BsonType `thrift:"BSON,13" db:"BSON" json:"BSON,omitempty"` UUID *UUIDType `thrift:"UUID,14" db:"UUID" json:"UUID,omitempty"` + FLOAT16 *Float16Type `thrift:"FLOAT16,15" db:"FLOAT16" json:"FLOAT16,omitempty"` } func NewLogicalType() *LogicalType { @@ -2610,6 +2736,13 @@ func (p *LogicalType) GetUUID() *UUIDType { } return p.UUID } +var LogicalType_FLOAT16_DEFAULT *Float16Type +func (p *LogicalType) GetFLOAT16() *Float16Type { + if !p.IsSetFLOAT16() { + return LogicalType_FLOAT16_DEFAULT + } +return p.FLOAT16 +} func (p *LogicalType) CountSetFieldsLogicalType() int { count := 0 if (p.IsSetSTRING()) { @@ -2651,6 +2784,9 @@ func (p *LogicalType) CountSetFieldsLogicalType() int { if (p.IsSetUUID()) { count++ } + if (p.IsSetFLOAT16()) { + count++ + } return count } @@ -2707,6 +2843,10 @@ func (p *LogicalType) IsSetUUID() bool { return p.UUID != nil } +func (p *LogicalType) IsSetFLOAT16() bool { + return p.FLOAT16 != nil +} + func (p *LogicalType) Read(ctx context.Context, iprot thrift.TProtocol) error { if _, err := iprot.ReadStructBegin(ctx); err != nil { return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) @@ -2850,6 +2990,16 @@ func (p *LogicalType) Read(ctx context.Context, iprot thrift.TProtocol) error { return err } } + case 15: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField15(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } default: if err := iprot.Skip(ctx, fieldTypeId); err != nil { return err @@ -2969,6 +3119,14 @@ func (p *LogicalType) ReadField14(ctx context.Context, iprot thrift.TProtocol) return nil } +func (p *LogicalType) ReadField15(ctx context.Context, iprot thrift.TProtocol) error { + p.FLOAT16 = &Float16Type{} + if err := p.FLOAT16.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.FLOAT16), err) + } + return nil +} + func (p *LogicalType) Write(ctx context.Context, oprot thrift.TProtocol) error { if c := p.CountSetFieldsLogicalType(); c != 1 { return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) @@ -2989,6 +3147,7 @@ func (p *LogicalType) Write(ctx context.Context, oprot thrift.TProtocol) error { if err := p.writeField12(ctx, oprot); err != nil { return err } if err := p.writeField13(ctx, oprot); err != nil { return err } if err := p.writeField14(ctx, oprot); err != nil { return err } + if err := p.writeField15(ctx, oprot); err != nil { return err } } if err := oprot.WriteFieldStop(ctx); err != nil { return thrift.PrependError("write field stop error: ", err) } @@ -3166,6 +3325,19 @@ func (p *LogicalType) writeField14(ctx context.Context, oprot thrift.TProtocol) return err } +func (p *LogicalType) writeField15(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetFLOAT16() { + if err := oprot.WriteFieldBegin(ctx, "FLOAT16", thrift.STRUCT, 15); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 15:FLOAT16: ", p), err) } + if err := p.FLOAT16.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.FLOAT16), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 15:FLOAT16: ", p), err) } + } + return err +} + func (p *LogicalType) Equals(other *LogicalType) bool { if p == other { return true @@ -3185,6 +3357,7 @@ func (p *LogicalType) Equals(other *LogicalType) bool { if !p.JSON.Equals(other.JSON) { return false } if !p.BSON.Equals(other.BSON) { return false } if !p.UUID.Equals(other.UUID) { return false } + if !p.FLOAT16.Equals(other.FLOAT16) { return false } return true } @@ -3195,6 +3368,9 @@ func (p *LogicalType) String() string { return fmt.Sprintf("LogicalType(%+v)", *p) } +func (p *LogicalType) Validate() error { + return nil +} // Represents a element inside a schema definition. // - if it is a group (inner node) then type is undefined and num_children is defined // - if it is a primitive type (leaf) then type is defined and num_children is undefined @@ -3202,7 +3378,7 @@ func (p *LogicalType) String() string { // // Attributes: // - Type: Data type for this field. Not set if the current element is a non-leaf node -// - TypeLength: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales. +// - TypeLength: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the values. // Otherwise, if specified, this is the maximum bit length to store any of the values. // (e.g. a low cardinality INT col could have this set to 3). Note that this is // in the schema, and therefore fixed for the entire file. @@ -3213,10 +3389,14 @@ func (p *LogicalType) String() string { // the nesting is flattened to a single list by a depth-first traversal. // The children count is used to construct the nested relationship. // This field is not set when the element is a primitive type -// - ConvertedType: When the schema is the result of a conversion from another model +// - ConvertedType: DEPRECATED: When the schema is the result of a conversion from another model. // Used to record the original type to help with cross conversion. -// - Scale: Used when this column contains decimal data. +// +// This is superseded by logicalType. +// - Scale: DEPRECATED: Used when this column contains decimal data. // See the DECIMAL converted type for more details. +// +// This is superseded by using the DecimalType annotation in logicalType. // - Precision // - FieldID: When the original schema supports field ids, this will save the // original field id in the parquet schema @@ -3776,6 +3956,9 @@ func (p *SchemaElement) String() string { return fmt.Sprintf("SchemaElement(%+v)", *p) } +func (p *SchemaElement) Validate() error { + return nil +} // Data page header // // Attributes: @@ -4059,6 +4242,9 @@ func (p *DataPageHeader) String() string { return fmt.Sprintf("DataPageHeader(%+v)", *p) } +func (p *DataPageHeader) Validate() error { + return nil +} type IndexPageHeader struct { } @@ -4119,6 +4305,14 @@ func (p *IndexPageHeader) String() string { return fmt.Sprintf("IndexPageHeader(%+v)", *p) } +func (p *IndexPageHeader) Validate() error { + return nil +} +// The dictionary page must be placed at the first position of the column chunk +// if it is partly or completely dictionary encoded. At most one dictionary page +// can be placed in a column chunk. +// +// // Attributes: // - NumValues: Number of values in the dictionary * // - Encoding: Encoding using this dictionary page * @@ -4319,6 +4513,9 @@ func (p *DictionaryPageHeader) String() string { return fmt.Sprintf("DictionaryPageHeader(%+v)", *p) } +func (p *DictionaryPageHeader) Validate() error { + return nil +} // New page format allowing reading levels without decompressing the data // Repetition and definition levels are uncompressed // The remaining section containing the data is compressed if is_compressed is true @@ -4738,6 +4935,9 @@ func (p *DataPageHeaderV2) String() string { return fmt.Sprintf("DataPageHeaderV2(%+v)", *p) } +func (p *DataPageHeaderV2) Validate() error { + return nil +} // Block-based algorithm type annotation. * type SplitBlockAlgorithm struct { } @@ -4799,6 +4999,9 @@ func (p *SplitBlockAlgorithm) String() string { return fmt.Sprintf("SplitBlockAlgorithm(%+v)", *p) } +func (p *SplitBlockAlgorithm) Validate() error { + return nil +} // The algorithm used in Bloom filter. * // // Attributes: @@ -4923,6 +5126,9 @@ func (p *BloomFilterAlgorithm) String() string { return fmt.Sprintf("BloomFilterAlgorithm(%+v)", *p) } +func (p *BloomFilterAlgorithm) Validate() error { + return nil +} // Hash strategy type annotation. xxHash is an extremely fast non-cryptographic hash // algorithm. It uses 64 bits version of xxHash. // @@ -4986,6 +5192,9 @@ func (p *XxHash) String() string { return fmt.Sprintf("XxHash(%+v)", *p) } +func (p *XxHash) Validate() error { + return nil +} // The hash function used in Bloom filter. This function takes the hash of a column value // using plain encoding. // @@ -5112,6 +5321,9 @@ func (p *BloomFilterHash) String() string { return fmt.Sprintf("BloomFilterHash(%+v)", *p) } +func (p *BloomFilterHash) Validate() error { + return nil +} // The compression used in the Bloom filter. // type Uncompressed struct { @@ -5174,6 +5386,9 @@ func (p *Uncompressed) String() string { return fmt.Sprintf("Uncompressed(%+v)", *p) } +func (p *Uncompressed) Validate() error { + return nil +} // Attributes: // - UNCOMPRESSED type BloomFilterCompression struct { @@ -5296,6 +5511,9 @@ func (p *BloomFilterCompression) String() string { return fmt.Sprintf("BloomFilterCompression(%+v)", *p) } +func (p *BloomFilterCompression) Validate() error { + return nil +} // Bloom filter header is stored at beginning of Bloom filter data of each column // and followed by its bitset. // @@ -5553,36 +5771,29 @@ func (p *BloomFilterHeader) String() string { return fmt.Sprintf("BloomFilterHeader(%+v)", *p) } +func (p *BloomFilterHeader) Validate() error { + return nil +} // Attributes: // - Type: the type of the page: indicates which of the *_header fields is set * // - UncompressedPageSize: Uncompressed page size in bytes (not including this header) * // - CompressedPageSize: Compressed (and potentially encrypted) page size in bytes, not including this header * -// - Crc: The 32bit CRC for the page, to be be calculated as follows: -// - Using the standard CRC32 algorithm -// - On the data only, i.e. this header should not be included. 'Data' -// hereby refers to the concatenation of the repetition levels, the -// definition levels and the column value, in this exact order. -// - On the encoded versions of the repetition levels, definition levels and -// column values -// - On the compressed versions of the repetition levels, definition levels -// and column values where possible; -// - For v1 data pages, the repetition levels, definition levels and column -// values are always compressed together. If a compression scheme is -// specified, the CRC shall be calculated on the compressed version of -// this concatenation. If no compression scheme is specified, the CRC -// shall be calculated on the uncompressed version of this concatenation. -// - For v2 data pages, the repetition levels and definition levels are -// handled separately from the data and are never compressed (only -// encoded). If a compression scheme is specified, the CRC shall be -// calculated on the concatenation of the uncompressed repetition levels, -// uncompressed definition levels and the compressed column values. -// If no compression scheme is specified, the CRC shall be calculated on -// the uncompressed concatenation. -// - In encrypted columns, CRC is calculated after page encryption; the -// encryption itself is performed after page compression (if compressed) +// - Crc: The 32-bit CRC checksum for the page, to be be calculated as follows: +// +// - The standard CRC32 algorithm is used (with polynomial 0x04C11DB7, +// the same as in e.g. GZip). +// - All page types can have a CRC (v1 and v2 data pages, dictionary pages, +// etc.). +// - The CRC is computed on the serialization binary representation of the page +// (as written to disk), excluding the page header. For example, for v1 +// data pages, the CRC is computed on the concatenation of repetition levels, +// definition levels and column values (optionally compressed, optionally +// encrypted). +// - The CRC computation therefore takes place after any compression +// and encryption steps, if any. +// // If enabled, this allows for disabling checksumming in HDFS if only a few // pages need to be read. -// // - DataPageHeader // - IndexPageHeader // - DictionaryPageHeader @@ -6006,6 +6217,9 @@ func (p *PageHeader) String() string { return fmt.Sprintf("PageHeader(%+v)", *p) } +func (p *PageHeader) Validate() error { + return nil +} // Wrapper struct to store key values // // Attributes: @@ -6165,6 +6379,9 @@ func (p *KeyValue) String() string { return fmt.Sprintf("KeyValue(%+v)", *p) } +func (p *KeyValue) Validate() error { + return nil +} // Wrapper struct to specify sort order // // Attributes: @@ -6358,6 +6575,9 @@ func (p *SortingColumn) String() string { return fmt.Sprintf("SortingColumn(%+v)", *p) } +func (p *SortingColumn) Validate() error { + return nil +} // statistics of a given page type and encoding // // Attributes: @@ -6552,6 +6772,9 @@ func (p *PageEncodingStats) String() string { return fmt.Sprintf("PageEncodingStats(%+v)", *p) } +func (p *PageEncodingStats) Validate() error { + return nil +} // Description for column metadata // // Attributes: @@ -7346,6 +7569,9 @@ func (p *ColumnMetaData) String() string { return fmt.Sprintf("ColumnMetaData(%+v)", *p) } +func (p *ColumnMetaData) Validate() error { + return nil +} type EncryptionWithFooterKey struct { } @@ -7406,6 +7632,9 @@ func (p *EncryptionWithFooterKey) String() string { return fmt.Sprintf("EncryptionWithFooterKey(%+v)", *p) } +func (p *EncryptionWithFooterKey) Validate() error { + return nil +} // Attributes: // - PathInSchema: Column path in schema * // - KeyMetadata: Retrieval metadata of column encryption key * @@ -7581,6 +7810,9 @@ func (p *EncryptionWithColumnKey) String() string { return fmt.Sprintf("EncryptionWithColumnKey(%+v)", *p) } +func (p *EncryptionWithColumnKey) Validate() error { + return nil +} // Attributes: // - ENCRYPTION_WITH_FOOTER_KEY // - ENCRYPTION_WITH_COLUMN_KEY @@ -7752,6 +7984,9 @@ func (p *ColumnCryptoMetaData) String() string { return fmt.Sprintf("ColumnCryptoMetaData(%+v)", *p) } +func (p *ColumnCryptoMetaData) Validate() error { + return nil +} // Attributes: // - FilePath: File where column data is stored. If not set, assumed to be same file as // metadata. This path is relative to the current file. @@ -8254,6 +8489,9 @@ func (p *ColumnChunk) String() string { return fmt.Sprintf("ColumnChunk(%+v)", *p) } +func (p *ColumnChunk) Validate() error { + return nil +} // Attributes: // - Columns: Metadata for each column chunk in this row group. // This list must have the same order as the SchemaElement list in FileMetaData. @@ -8694,6 +8932,9 @@ func (p *RowGroup) String() string { return fmt.Sprintf("RowGroup(%+v)", *p) } +func (p *RowGroup) Validate() error { + return nil +} // Empty struct to signal the order defined by the physical or logical type type TypeDefinedOrder struct { } @@ -8755,6 +8996,9 @@ func (p *TypeDefinedOrder) String() string { return fmt.Sprintf("TypeDefinedOrder(%+v)", *p) } +func (p *TypeDefinedOrder) Validate() error { + return nil +} // Union to specify the order used for the min_value and max_value fields for a // column. This union takes the role of an enhanced enum that allows rich // elements (which will be needed for a collation-based ordering in the future). @@ -8808,6 +9052,13 @@ func (p *TypeDefinedOrder) String() string { // - If the min is +0, the row group may contain -0 values as well. // - If the max is -0, the row group may contain +0 values as well. // - When looking for NaN values, min and max should be ignored. +// +// When writing statistics the following rules should be followed: +// - NaNs should not be written to min or max statistics fields. +// - If the computed max value is zero (whether negative or positive), +// `+0.0` should be written into the max statistics field. +// - If the computed min value is zero (whether negative or positive), +// `-0.0` should be written into the min statistics field. type ColumnOrder struct { TYPE_ORDER *TypeDefinedOrder `thrift:"TYPE_ORDER,1" db:"TYPE_ORDER" json:"TYPE_ORDER,omitempty"` } @@ -8928,6 +9179,9 @@ func (p *ColumnOrder) String() string { return fmt.Sprintf("ColumnOrder(%+v)", *p) } +func (p *ColumnOrder) Validate() error { + return nil +} // Attributes: // - Offset: Offset of the page in the file * // - CompressedPageSize: Size of the page, including header. Sum of compressed_page_size and header @@ -9120,6 +9374,9 @@ func (p *PageLocation) String() string { return fmt.Sprintf("PageLocation(%+v)", *p) } +func (p *PageLocation) Validate() error { + return nil +} // Attributes: // - PageLocations: PageLocations, ordered by increasing PageLocation.offset. It is required // that page_locations[i].first_row_index < page_locations[i+1].first_row_index. @@ -9251,6 +9508,9 @@ func (p *OffsetIndex) String() string { return fmt.Sprintf("OffsetIndex(%+v)", *p) } +func (p *OffsetIndex) Validate() error { + return nil +} // Description for ColumnIndex. // Each [i] refers to the page at OffsetIndex.page_locations[i] // @@ -9260,15 +9520,16 @@ func (p *OffsetIndex) String() string { // have to set the corresponding entries in min_values and max_values to // byte[0], so that all lists have the same length. If false, the // corresponding entries in min_values and max_values must be valid. -// - MinValues: Two lists containing lower and upper bounds for the values of each page. -// These may be the actual minimum and maximum values found on a page, but -// can also be (more compact) values that do not exist on a page. For -// example, instead of storing ""Blart Versenwald III", a writer may set -// min_values[i]="B", max_values[i]="C". Such more compact values must still -// be valid values within the column's logical type. Readers must make sure -// that list entries are populated before using them by inspecting null_pages. +// - MinValues: Two lists containing lower and upper bounds for the values of each page +// determined by the ColumnOrder of the column. These may be the actual +// minimum and maximum values found on a page, but can also be (more compact) +// values that do not exist on a page. For example, instead of storing ""Blart +// Versenwald III", a writer may set min_values[i]="B", max_values[i]="C". +// Such more compact values must still be valid values within the column's +// logical type. Readers must make sure that list entries are populated before +// using them by inspecting null_pages. // - MaxValues -// - BoundaryOrder: Stores whether both min_values and max_values are orderd and if so, in +// - BoundaryOrder: Stores whether both min_values and max_values are ordered and if so, in // which direction. This allows readers to perform binary searches in both // lists. Readers cannot assume that max_values[i] <= min_values[i+1], even // if the lists are ordered. @@ -9644,6 +9905,9 @@ func (p *ColumnIndex) String() string { return fmt.Sprintf("ColumnIndex(%+v)", *p) } +func (p *ColumnIndex) Validate() error { + return nil +} // Attributes: // - AadPrefix: AAD prefix * // - AadFileUnique: Unique file identifier part of AAD suffix * @@ -9848,6 +10112,9 @@ func (p *AesGcmV1) String() string { return fmt.Sprintf("AesGcmV1(%+v)", *p) } +func (p *AesGcmV1) Validate() error { + return nil +} // Attributes: // - AadPrefix: AAD prefix * // - AadFileUnique: Unique file identifier part of AAD suffix * @@ -10052,6 +10319,9 @@ func (p *AesGcmCtrV1) String() string { return fmt.Sprintf("AesGcmCtrV1(%+v)", *p) } +func (p *AesGcmCtrV1) Validate() error { + return nil +} // Attributes: // - AES_GCM_V1 // - AES_GCM_CTR_V1 @@ -10223,6 +10493,9 @@ func (p *EncryptionAlgorithm) String() string { return fmt.Sprintf("EncryptionAlgorithm(%+v)", *p) } +func (p *EncryptionAlgorithm) Validate() error { + return nil +} // Description for file metadata // // Attributes: @@ -10240,17 +10513,20 @@ func (p *EncryptionAlgorithm) String() string { // version (build ). // e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55) // -// - ColumnOrders: Sort order used for the min_value and max_value fields of each column in -// this file. Sort orders are listed in the order matching the columns in the -// schema. The indexes are not necessary the same though, because only leaf -// nodes of the schema are represented in the list of sort orders. +// - ColumnOrders: Sort order used for the min_value and max_value fields in the Statistics +// objects and the min_values and max_values fields in the ColumnIndex +// objects of each column in this file. Sort orders are listed in the order +// matching the columns in the schema. The indexes are not necessary the same +// though, because only leaf nodes of the schema are represented in the list +// of sort orders. // -// Without column_orders, the meaning of the min_value and max_value fields is -// undefined. To ensure well-defined behaviour, if min_value and max_value are -// written to a Parquet file, column_orders must be written as well. +// Without column_orders, the meaning of the min_value and max_value fields +// in the Statistics object and the ColumnIndex object is undefined. To ensure +// well-defined behaviour, if these fields are written to a Parquet file, +// column_orders must be written as well. // -// The obsolete min and max fields are always sorted by signed comparison -// regardless of column_orders. +// The obsolete min and max fields in the Statistics object are always sorted +// by signed comparison regardless of column_orders. // - EncryptionAlgorithm: Encryption algorithm. This field is set only in encrypted files // with plaintext footer. Files with encrypted footer store algorithm id // in FileCryptoMetaData structure. @@ -10803,6 +11079,9 @@ func (p *FileMetaData) String() string { return fmt.Sprintf("FileMetaData(%+v)", *p) } +func (p *FileMetaData) Validate() error { + return nil +} // Crypto metadata for files with encrypted footer * // // Attributes: @@ -10965,3 +11244,6 @@ func (p *FileCryptoMetaData) String() string { return fmt.Sprintf("FileCryptoMetaData(%+v)", *p) } +func (p *FileCryptoMetaData) Validate() error { + return nil +} diff --git a/go/parquet/internal/testutils/random.go b/go/parquet/internal/testutils/random.go index bb101ebf9a92d..bb9ee0cdf2bba 100644 --- a/go/parquet/internal/testutils/random.go +++ b/go/parquet/internal/testutils/random.go @@ -28,6 +28,7 @@ import ( "github.com/apache/arrow/go/v15/arrow/array" "github.com/apache/arrow/go/v15/arrow/bitutil" "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/float16" "github.com/apache/arrow/go/v15/arrow/memory" "github.com/apache/arrow/go/v15/parquet" "github.com/apache/arrow/go/v15/parquet/pqarrow" @@ -369,6 +370,17 @@ func randFloat64(r *rand.Rand) float64 { } } +// randFloat16 creates a random float value with a normal distribution +// to better spread the values out and ensure we do not return any NaN or Inf values. +func randFloat16(r *rand.Rand) float16.Num { + for { + f := float16.FromBits(uint16(r.Uint64n(math.MaxUint16 + 1))) + if !f.IsNaN() { + return f + } + } +} + // FillRandomFloat32 populates out with random float32 values using seed as the random // seed for the generator to allow consistency for testing. func FillRandomFloat32(seed uint64, out []float32) { @@ -387,6 +399,15 @@ func FillRandomFloat64(seed uint64, out []float64) { } } +// FillRandomFloat16 populates out with random float64 values using seed as the random +// seed for the generator to allow consistency for testing. +func FillRandomFloat16(seed uint64, out []float16.Num) { + r := rand.New(rand.NewSource(seed)) + for idx := range out { + out[idx] = randFloat16(r) + } +} + // FillRandomByteArray populates out with random ByteArray values with lengths between 2 and 12 // using heap as the actual memory storage used for the bytes generated. Each element of // out will be some slice of the bytes in heap, and as such heap must outlive the byte array slices. @@ -456,6 +477,8 @@ func InitValues(values interface{}, heap *memory.Buffer) { FillRandomFloat32(0, arr) case []float64: FillRandomFloat64(0, arr) + case []float16.Num: + FillRandomFloat16(0, arr) case []parquet.Int96: FillRandomInt96(0, arr) case []parquet.ByteArray: diff --git a/go/parquet/internal/testutils/random_arrow.go b/go/parquet/internal/testutils/random_arrow.go index d886db0360b84..7dd2a3e8b77e3 100644 --- a/go/parquet/internal/testutils/random_arrow.go +++ b/go/parquet/internal/testutils/random_arrow.go @@ -19,6 +19,7 @@ package testutils import ( "github.com/apache/arrow/go/v15/arrow" "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/float16" "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/exp/rand" ) @@ -49,6 +50,13 @@ func RandomNonNull(mem memory.Allocator, dt arrow.DataType, size int) arrow.Arra FillRandomFloat64(0, values) bldr.AppendValues(values, nil) return bldr.NewArray() + case arrow.FLOAT16: + bldr := array.NewFloat16Builder(mem) + defer bldr.Release() + values := make([]float16.Num, size) + FillRandomFloat16(0, values) + bldr.AppendValues(values, nil) + return bldr.NewArray() case arrow.INT64: bldr := array.NewInt64Builder(mem) defer bldr.Release() @@ -212,6 +220,21 @@ func RandomNullable(dt arrow.DataType, size int, numNulls int) arrow.Array { values := make([]float64, size) FillRandomFloat64(0, values) + valid := make([]bool, size) + for idx := range valid { + valid[idx] = true + } + for i := 0; i < numNulls; i++ { + valid[i*2] = false + } + bldr.AppendValues(values, valid) + return bldr.NewArray() + case arrow.FLOAT16: + bldr := array.NewFloat16Builder(memory.DefaultAllocator) + defer bldr.Release() + values := make([]float16.Num, size) + FillRandomFloat16(0, values) + valid := make([]bool, size) for idx := range valid { valid[idx] = true diff --git a/go/parquet/metadata/statistics.go b/go/parquet/metadata/statistics.go index 606e76ffd0454..43294272dec35 100644 --- a/go/parquet/metadata/statistics.go +++ b/go/parquet/metadata/statistics.go @@ -23,6 +23,7 @@ import ( "unsafe" "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/float16" "github.com/apache/arrow/go/v15/arrow/memory" "github.com/apache/arrow/go/v15/internal/utils" "github.com/apache/arrow/go/v15/parquet" @@ -32,7 +33,7 @@ import ( "github.com/apache/arrow/go/v15/parquet/schema" ) -//go:generate go run ../../arrow/_tools/tmpl/main.go -i -data=../internal/encoding/physical_types.tmpldata statistics_types.gen.go.tmpl +//go:generate go run ../../arrow/_tools/tmpl/main.go -i -data=statistics_types.tmpldata statistics_types.gen.go.tmpl type StatProvider interface { GetMin() []byte @@ -373,6 +374,9 @@ var ( defaultMinUInt96 parquet.Int96 defaultMaxInt96 parquet.Int96 defaultMaxUInt96 parquet.Int96 + + defaultMinFloat16 parquet.FixedLenByteArray = float16.MaxNum.ToLEBytes() + defaultMaxFloat16 parquet.FixedLenByteArray = float16.MinNum.ToLEBytes() ) func init() { @@ -407,6 +411,14 @@ func (s *Int96Statistics) defaultMax() parquet.Int96 { return defaultMaxInt96 } +func (Float16Statistics) defaultMin() parquet.FixedLenByteArray { + return defaultMinFloat16 +} + +func (Float16Statistics) defaultMax() parquet.FixedLenByteArray { + return defaultMaxFloat16 +} + func (Float32Statistics) defaultMin() float32 { return math.MaxFloat32 } func (Float32Statistics) defaultMax() float32 { return -math.MaxFloat32 } func (Float64Statistics) defaultMin() float64 { return math.MaxFloat64 } @@ -427,6 +439,10 @@ func (FixedLenByteArrayStatistics) equal(a, b parquet.FixedLenByteArray) bool { return bytes.Equal(a, b) } +func (Float16Statistics) equal(a, b parquet.FixedLenByteArray) bool { + return float16.FromLEBytes(a).Equal(float16.FromLEBytes(b)) +} + func (BooleanStatistics) less(a, b bool) bool { return !a && b } @@ -481,6 +497,10 @@ func (s *FixedLenByteArrayStatistics) less(a, b parquet.FixedLenByteArray) bool return signedByteLess([]byte(a), []byte(b)) } +func (Float16Statistics) less(a, b parquet.FixedLenByteArray) bool { + return float16.FromLEBytes(a).Less(float16.FromLEBytes(b)) +} + func (BooleanStatistics) cleanStat(minMax minmaxPairBoolean) *minmaxPairBoolean { return &minMax } func (Int32Statistics) cleanStat(minMax minmaxPairInt32) *minmaxPairInt32 { return &minMax } func (Int64Statistics) cleanStat(minMax minmaxPairInt64) *minmaxPairInt64 { return &minMax } @@ -535,6 +555,29 @@ func (Float64Statistics) cleanStat(minMax minmaxPairFloat64) *minmaxPairFloat64 return &minMax } +func (Float16Statistics) cleanStat(minMax minmaxPairFloat16) *minmaxPairFloat16 { + min := float16.FromLEBytes(minMax[0][:]) + max := float16.FromLEBytes(minMax[1][:]) + + if min.IsNaN() || max.IsNaN() { + return nil + } + + if min.Equal(float16.MaxNum) && max.Equal(float16.MinNum) { + return nil + } + + zero := float16.New(0) + if min.Equal(zero) && !min.Signbit() { + minMax[0] = min.Negate().ToLEBytes() + } + if max.Equal(zero) && max.Signbit() { + minMax[1] = max.Negate().ToLEBytes() + } + + return &minMax +} + func (ByteArrayStatistics) cleanStat(minMax minmaxPairByteArray) *minmaxPairByteArray { if minMax[0] == nil || minMax[1] == nil { return nil diff --git a/go/parquet/metadata/statistics_test.go b/go/parquet/metadata/statistics_test.go index 47798d3c4dd3c..19311dc8955d3 100644 --- a/go/parquet/metadata/statistics_test.go +++ b/go/parquet/metadata/statistics_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v15/arrow/float16" "github.com/apache/arrow/go/v15/arrow/memory" "github.com/apache/arrow/go/v15/parquet" "github.com/apache/arrow/go/v15/parquet/metadata" @@ -32,24 +33,36 @@ import ( // NOTE(zeroshade): tests will be added and updated after merging the "file" package // since the tests that I wrote relied on the file writer/reader for ease of use. +func newFloat16Node(name string, rep parquet.Repetition, fieldID int32) *schema.PrimitiveNode { + return schema.MustPrimitive(schema.NewPrimitiveNodeLogical(name, rep, schema.Float16LogicalType{}, parquet.Types.FixedLenByteArray, 2, fieldID)) +} + func TestCheckNaNs(t *testing.T) { const ( numvals = 8 min = -4.0 max = 3.0 ) - nan := math.NaN() + var ( + nan = math.NaN() + f16Min parquet.FixedLenByteArray = float16.New(float32(min)).ToLEBytes() + f16Max parquet.FixedLenByteArray = float16.New(float32(max)).ToLEBytes() + ) allNans := []float64{nan, nan, nan, nan, nan, nan, nan, nan} allNansf32 := make([]float32, numvals) + allNansf16 := make([]parquet.FixedLenByteArray, numvals) for idx, v := range allNans { allNansf32[idx] = float32(v) + allNansf16[idx] = float16.New(float32(v)).ToLEBytes() } someNans := []float64{nan, max, -3.0, -1.0, nan, 2.0, min, nan} someNansf32 := make([]float32, numvals) + someNansf16 := make([]parquet.FixedLenByteArray, numvals) for idx, v := range someNans { someNansf32[idx] = float32(v) + someNansf16[idx] = float16.New(float32(v)).ToLEBytes() } validBitmap := []byte{0x7F} // 0b01111111 @@ -62,6 +75,8 @@ func TestCheckNaNs(t *testing.T) { s.Update(values.([]float32), 0) case *metadata.Float64Statistics: s.Update(values.([]float64), 0) + case *metadata.Float16Statistics: + s.Update(values.([]parquet.FixedLenByteArray), 0) } assert.False(t, stats.HasMinMax()) } else { @@ -72,6 +87,8 @@ func TestCheckNaNs(t *testing.T) { s.UpdateSpaced(values.([]float32), bitmap, 0, int64(nullCount)) case *metadata.Float64Statistics: s.UpdateSpaced(values.([]float64), bitmap, 0, int64(nullCount)) + case *metadata.Float16Statistics: + s.UpdateSpaced(values.([]parquet.FixedLenByteArray), bitmap, 0, int64(nullCount)) } assert.False(t, stats.HasMinMax()) } @@ -89,6 +106,11 @@ func TestCheckNaNs(t *testing.T) { assert.True(t, stats.HasMinMax()) assert.Equal(t, expectedMin, s.Min()) assert.Equal(t, expectedMax, s.Max()) + case *metadata.Float16Statistics: + s.Update(values.([]parquet.FixedLenByteArray), 0) + assert.True(t, stats.HasMinMax()) + assert.Equal(t, expectedMin, s.Min()) + assert.Equal(t, expectedMax, s.Max()) } } @@ -106,34 +128,48 @@ func TestCheckNaNs(t *testing.T) { assert.True(t, s.HasMinMax()) assert.Equal(t, expectedMin, s.Min()) assert.Equal(t, expectedMax, s.Max()) + case *metadata.Float16Statistics: + s.UpdateSpaced(values.([]parquet.FixedLenByteArray), bitmap, 0, int64(nullCount)) + assert.True(t, s.HasMinMax()) + assert.Equal(t, expectedMin, s.Min()) + assert.Equal(t, expectedMax, s.Max()) } } f32Col := schema.NewColumn(schema.NewFloat32Node("f", parquet.Repetitions.Optional, -1), 1, 1) f64Col := schema.NewColumn(schema.NewFloat64Node("f", parquet.Repetitions.Optional, -1), 1, 1) + f16Col := schema.NewColumn(newFloat16Node("f", parquet.Repetitions.Required, -1), 1, 1) // test values someNanStats := metadata.NewStatistics(f64Col, memory.DefaultAllocator) someNanStatsf32 := metadata.NewStatistics(f32Col, memory.DefaultAllocator) + someNanStatsf16 := metadata.NewStatistics(f16Col, memory.DefaultAllocator) // ingesting only nans should not yield a min or max assertUnsetMinMax(someNanStats, allNans, nil) assertUnsetMinMax(someNanStatsf32, allNansf32, nil) + assertUnsetMinMax(someNanStatsf16, allNansf16, nil) // ingesting a mix should yield a valid min/max assertMinMaxAre(someNanStats, someNans, min, max) assertMinMaxAre(someNanStatsf32, someNansf32, float32(min), float32(max)) + assertMinMaxAre(someNanStatsf16, someNansf16, f16Min, f16Max) // ingesting only nans after a valid min/max should have no effect assertMinMaxAre(someNanStats, allNans, min, max) assertMinMaxAre(someNanStatsf32, allNansf32, float32(min), float32(max)) + assertMinMaxAre(someNanStatsf16, allNansf16, f16Min, f16Max) someNanStats = metadata.NewStatistics(f64Col, memory.DefaultAllocator) someNanStatsf32 = metadata.NewStatistics(f32Col, memory.DefaultAllocator) + someNanStatsf16 = metadata.NewStatistics(f16Col, memory.DefaultAllocator) assertUnsetMinMax(someNanStats, allNans, validBitmap) assertUnsetMinMax(someNanStatsf32, allNansf32, validBitmap) + assertUnsetMinMax(someNanStatsf16, allNansf16, validBitmap) // nans should not pollute min/max when excluded via null bitmap assertMinMaxAreSpaced(someNanStats, someNans, validBitmapNoNaNs, min, max) assertMinMaxAreSpaced(someNanStatsf32, someNansf32, validBitmapNoNaNs, float32(min), float32(max)) + assertMinMaxAreSpaced(someNanStatsf16, someNansf16, validBitmapNoNaNs, f16Min, f16Max) // ingesting nans with a null bitmap should not change the result assertMinMaxAreSpaced(someNanStats, someNans, validBitmap, min, max) assertMinMaxAreSpaced(someNanStatsf32, someNansf32, validBitmap, float32(min), float32(max)) + assertMinMaxAreSpaced(someNanStatsf16, someNansf16, validBitmap, f16Min, f16Max) } func TestCheckNegativeZeroStats(t *testing.T) { @@ -155,37 +191,61 @@ func TestCheckNegativeZeroStats(t *testing.T) { assert.True(t, math.Signbit(s.Min())) assert.Equal(t, zero, s.Max()) assert.False(t, math.Signbit(s.Max())) + case *metadata.Float16Statistics: + s.Update(values.([]parquet.FixedLenByteArray), 0) + assert.True(t, s.HasMinMax()) + var zero float64 + min := float64(float16.FromLEBytes(s.Min()).Float32()) + max := float64(float16.FromLEBytes(s.Max()).Float32()) + assert.Equal(t, zero, min) + assert.True(t, math.Signbit(min)) + assert.Equal(t, zero, max) + assert.False(t, math.Signbit(max)) } } fcol := schema.NewColumn(schema.NewFloat32Node("f", parquet.Repetitions.Optional, -1), 1, 1) dcol := schema.NewColumn(schema.NewFloat64Node("d", parquet.Repetitions.Optional, -1), 1, 1) + hcol := schema.NewColumn(newFloat16Node("h", parquet.Repetitions.Optional, -1), 1, 1) var f32zero float32 var f64zero float64 + var f16PosZero parquet.FixedLenByteArray = float16.New(+f32zero).ToLEBytes() + var f16NegZero parquet.FixedLenByteArray = float16.New(-f32zero).ToLEBytes() + + assert.False(t, float16.FromLEBytes(f16PosZero).Signbit()) + assert.True(t, float16.FromLEBytes(f16NegZero).Signbit()) { fstats := metadata.NewStatistics(fcol, memory.DefaultAllocator) dstats := metadata.NewStatistics(dcol, memory.DefaultAllocator) + hstats := metadata.NewStatistics(hcol, memory.DefaultAllocator) assertMinMaxZeroesSign(fstats, []float32{-f32zero, f32zero}) assertMinMaxZeroesSign(dstats, []float64{-f64zero, f64zero}) + assertMinMaxZeroesSign(hstats, []parquet.FixedLenByteArray{f16NegZero, f16PosZero}) } { fstats := metadata.NewStatistics(fcol, memory.DefaultAllocator) dstats := metadata.NewStatistics(dcol, memory.DefaultAllocator) + hstats := metadata.NewStatistics(hcol, memory.DefaultAllocator) assertMinMaxZeroesSign(fstats, []float32{f32zero, -f32zero}) assertMinMaxZeroesSign(dstats, []float64{f64zero, -f64zero}) + assertMinMaxZeroesSign(hstats, []parquet.FixedLenByteArray{f16PosZero, f16NegZero}) } { fstats := metadata.NewStatistics(fcol, memory.DefaultAllocator) dstats := metadata.NewStatistics(dcol, memory.DefaultAllocator) + hstats := metadata.NewStatistics(hcol, memory.DefaultAllocator) assertMinMaxZeroesSign(fstats, []float32{-f32zero, -f32zero}) assertMinMaxZeroesSign(dstats, []float64{-f64zero, -f64zero}) + assertMinMaxZeroesSign(hstats, []parquet.FixedLenByteArray{f16NegZero, f16NegZero}) } { fstats := metadata.NewStatistics(fcol, memory.DefaultAllocator) dstats := metadata.NewStatistics(dcol, memory.DefaultAllocator) + hstats := metadata.NewStatistics(hcol, memory.DefaultAllocator) assertMinMaxZeroesSign(fstats, []float32{f32zero, f32zero}) assertMinMaxZeroesSign(dstats, []float64{f64zero, f64zero}) + assertMinMaxZeroesSign(hstats, []parquet.FixedLenByteArray{f16PosZero, f16PosZero}) } } diff --git a/go/parquet/metadata/statistics_types.gen.go b/go/parquet/metadata/statistics_types.gen.go index e8fb9877c8444..baecd185d14fc 100644 --- a/go/parquet/metadata/statistics_types.gen.go +++ b/go/parquet/metadata/statistics_types.gen.go @@ -24,6 +24,7 @@ import ( "github.com/apache/arrow/go/v15/arrow" "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/float16" "github.com/apache/arrow/go/v15/arrow/memory" "github.com/apache/arrow/go/v15/internal/bitutils" shared_utils "github.com/apache/arrow/go/v15/internal/utils" @@ -2432,6 +2433,314 @@ func (s *FixedLenByteArrayStatistics) Encode() (enc EncodedStatistics, err error return } +type minmaxPairFloat16 [2]parquet.FixedLenByteArray + +// Float16Statistics is the typed interface for managing stats for a column +// of Float16 type. +type Float16Statistics struct { + statistics + min parquet.FixedLenByteArray + max parquet.FixedLenByteArray + + bitSetReader bitutils.SetBitRunReader +} + +// NewFloat16Statistics constructs an appropriate stat object type using the +// given column descriptor and allocator. +// +// Panics if the physical type of descr is not parquet.Type.FixedLenByteArray +// Panics if the logical type of descr is not schema.Float16LogicalType +func NewFloat16Statistics(descr *schema.Column, mem memory.Allocator) *Float16Statistics { + if descr.PhysicalType() != parquet.Types.FixedLenByteArray { + panic(fmt.Errorf("parquet: invalid type %s for constructing a Float16 stat object", descr.PhysicalType())) + } + if !descr.LogicalType().Equals(schema.Float16LogicalType{}) { + panic(fmt.Errorf("parquet: invalid logical type %s for constructing a Float16 stat object", descr.LogicalType().String())) + } + + return &Float16Statistics{ + statistics: statistics{ + descr: descr, + hasNullCount: true, + hasDistinctCount: true, + order: descr.SortOrder(), + encoder: encoding.NewEncoder(descr.PhysicalType(), parquet.Encodings.Plain, false, descr, mem), + mem: mem, + }, + } +} + +// NewFloat16StatisticsFromEncoded will construct a propertly typed statistics object +// initializing it with the provided information. +func NewFloat16StatisticsFromEncoded(descr *schema.Column, mem memory.Allocator, nvalues int64, encoded StatProvider) *Float16Statistics { + ret := NewFloat16Statistics(descr, mem) + ret.nvalues += nvalues + if encoded.IsSetNullCount() { + ret.IncNulls(encoded.GetNullCount()) + } + if encoded.IsSetDistinctCount() { + ret.IncDistinct(encoded.GetDistinctCount()) + } + + encodedMin := encoded.GetMin() + if encodedMin != nil && len(encodedMin) > 0 { + ret.min = ret.plainDecode(encodedMin) + } + encodedMax := encoded.GetMax() + if encodedMax != nil && len(encodedMax) > 0 { + ret.max = ret.plainDecode(encodedMax) + } + ret.hasMinMax = encoded.IsSetMax() || encoded.IsSetMin() + return ret +} + +func (s *Float16Statistics) plainEncode(src parquet.FixedLenByteArray) []byte { + s.encoder.(encoding.FixedLenByteArrayEncoder).Put([]parquet.FixedLenByteArray{src}) + buf, err := s.encoder.FlushValues() + if err != nil { + panic(err) // recovered by Encode + } + defer buf.Release() + + out := make([]byte, buf.Len()) + copy(out, buf.Bytes()) + return out +} + +func (s *Float16Statistics) plainDecode(src []byte) parquet.FixedLenByteArray { + var buf [1]parquet.FixedLenByteArray + + decoder := encoding.NewDecoder(s.descr.PhysicalType(), parquet.Encodings.Plain, s.descr, s.mem) + decoder.SetData(1, src) + decoder.(encoding.FixedLenByteArrayDecoder).Decode(buf[:]) + return buf[0] +} + +func (s *Float16Statistics) minval(a, b parquet.FixedLenByteArray) parquet.FixedLenByteArray { + switch { + case a == nil: + return b + case b == nil: + return a + case s.less(a, b): + return a + default: + return b + } +} + +func (s *Float16Statistics) maxval(a, b parquet.FixedLenByteArray) parquet.FixedLenByteArray { + switch { + case a == nil: + return b + case b == nil: + return a + case s.less(a, b): + return b + default: + return a + } +} + +// MinMaxEqual returns true if both stat objects have the same Min and Max values +func (s *Float16Statistics) MinMaxEqual(rhs *Float16Statistics) bool { + return s.equal(s.min, rhs.min) && s.equal(s.max, rhs.max) +} + +// Equals returns true only if both objects are the same type, have the same min and +// max values, null count, distinct count and number of values. +func (s *Float16Statistics) Equals(other TypedStatistics) bool { + if s.Type() != other.Type() || !s.descr.LogicalType().Equals(other.Descr().LogicalType()) { + return false + } + rhs, ok := other.(*Float16Statistics) + if !ok { + return false + } + + if s.HasMinMax() != rhs.HasMinMax() { + return false + } + return (s.hasMinMax && s.MinMaxEqual(rhs)) && + s.NullCount() == rhs.NullCount() && + s.DistinctCount() == rhs.DistinctCount() && + s.NumValues() == rhs.NumValues() +} + +func (s *Float16Statistics) coalesce(val, fallback parquet.FixedLenByteArray) parquet.FixedLenByteArray { + if float16.FromLEBytes(val).IsNaN() { + return fallback + } + return val +} + +func (s *Float16Statistics) getMinMax(values []parquet.FixedLenByteArray) (min, max parquet.FixedLenByteArray) { + defMin := s.defaultMin() + defMax := s.defaultMax() + + min = defMin + max = defMax + + for _, v := range values { + min = s.minval(min, s.coalesce(v, defMin)) + max = s.maxval(max, s.coalesce(v, defMax)) + } + return +} + +func (s *Float16Statistics) getMinMaxSpaced(values []parquet.FixedLenByteArray, validBits []byte, validBitsOffset int64) (min, max parquet.FixedLenByteArray) { + min = s.defaultMin() + max = s.defaultMax() + + if s.bitSetReader == nil { + s.bitSetReader = bitutils.NewSetBitRunReader(validBits, validBitsOffset, int64(len(values))) + } else { + s.bitSetReader.Reset(validBits, validBitsOffset, int64(len(values))) + } + + for { + run := s.bitSetReader.NextRun() + if run.Length == 0 { + break + } + for _, v := range values[int(run.Pos):int(run.Pos+run.Length)] { + min = s.minval(min, coalesce(v, s.defaultMin()).(parquet.FixedLenByteArray)) + max = s.maxval(max, coalesce(v, s.defaultMax()).(parquet.FixedLenByteArray)) + } + } + return +} + +func (s *Float16Statistics) Min() parquet.FixedLenByteArray { return s.min } +func (s *Float16Statistics) Max() parquet.FixedLenByteArray { return s.max } + +// Merge merges the stats from other into this stat object, updating +// the null count, distinct count, number of values and the min/max if +// appropriate. +func (s *Float16Statistics) Merge(other TypedStatistics) { + rhs, ok := other.(*Float16Statistics) + if !ok { + panic("incompatible stat type merge") + } + + s.statistics.merge(rhs) + if rhs.HasMinMax() { + s.SetMinMax(rhs.Min(), rhs.Max()) + } +} + +// Update is used to add more values to the current stat object, finding the +// min and max values etc. +func (s *Float16Statistics) Update(values []parquet.FixedLenByteArray, numNull int64) { + s.IncNulls(numNull) + s.nvalues += int64(len(values)) + + if len(values) == 0 { + return + } + + s.SetMinMax(s.getMinMax(values)) +} + +// UpdateSpaced is just like Update, but for spaced values using validBits to determine +// and skip null values. +func (s *Float16Statistics) UpdateSpaced(values []parquet.FixedLenByteArray, validBits []byte, validBitsOffset, numNull int64) { + s.IncNulls(numNull) + notnull := int64(len(values)) - numNull + s.nvalues += notnull + + if notnull == 0 { + return + } + + s.SetMinMax(s.getMinMaxSpaced(values, validBits, validBitsOffset)) +} + +func (s *Float16Statistics) UpdateFromArrow(values arrow.Array, updateCounts bool) error { + if updateCounts { + s.IncNulls(int64(values.NullN())) + s.nvalues += int64(values.Len() - values.NullN()) + } + + if values.NullN() == values.Len() { + return nil + } + + return fmt.Errorf("%w: update float16 stats from Arrow", arrow.ErrNotImplemented) +} + +// SetMinMax updates the min and max values only if they are not currently set +// or if argMin is less than the current min / argMax is greater than the current max +func (s *Float16Statistics) SetMinMax(argMin, argMax parquet.FixedLenByteArray) { + maybeMinMax := s.cleanStat([2]parquet.FixedLenByteArray{argMin, argMax}) + if maybeMinMax == nil { + return + } + + min := (*maybeMinMax)[0] + max := (*maybeMinMax)[1] + + if !s.hasMinMax { + s.hasMinMax = true + s.min = min + s.max = max + } else { + if !s.less(s.min, min) { + s.min = min + } + if s.less(s.max, max) { + s.max = max + } + } +} + +// EncodeMin returns the encoded min value with plain encoding. +// +// ByteArray stats do not include the length in the encoding. +func (s *Float16Statistics) EncodeMin() []byte { + if s.HasMinMax() { + return s.plainEncode(s.min) + } + return nil +} + +// EncodeMax returns the current encoded max value with plain encoding +// +// ByteArray stats do not include the length in the encoding +func (s *Float16Statistics) EncodeMax() []byte { + if s.HasMinMax() { + return s.plainEncode(s.max) + } + return nil +} + +// Encode returns a populated EncodedStatistics object +func (s *Float16Statistics) Encode() (enc EncodedStatistics, err error) { + defer func() { + if r := recover(); r != nil { + switch r := r.(type) { + case error: + err = r + case string: + err = xerrors.New(r) + default: + err = fmt.Errorf("unknown error type thrown from panic: %v", r) + } + } + }() + if s.HasMinMax() { + enc.SetMax(s.EncodeMax()) + enc.SetMin(s.EncodeMin()) + } + if s.HasNullCount() { + enc.SetNullCount(s.NullCount()) + } + if s.HasDistinctCount() { + enc.SetDistinctCount(s.DistinctCount()) + } + return +} + // NewStatistics uses the type in the column descriptor to construct the appropriate // typed stats object. If mem is nil, then memory.DefaultAllocator will be used. func NewStatistics(descr *schema.Column, mem memory.Allocator) TypedStatistics { @@ -2454,6 +2763,9 @@ func NewStatistics(descr *schema.Column, mem memory.Allocator) TypedStatistics { case parquet.Types.ByteArray: return NewByteArrayStatistics(descr, mem) case parquet.Types.FixedLenByteArray: + if descr.LogicalType().Equals(schema.Float16LogicalType{}) { + return NewFloat16Statistics(descr, mem) + } return NewFixedLenByteArrayStatistics(descr, mem) default: panic("not implemented") @@ -2484,6 +2796,9 @@ func NewStatisticsFromEncoded(descr *schema.Column, mem memory.Allocator, nvalue case parquet.Types.ByteArray: return NewByteArrayStatisticsFromEncoded(descr, mem, nvalues, encoded) case parquet.Types.FixedLenByteArray: + if descr.LogicalType().Equals(schema.Float16LogicalType{}) { + return NewFloat16StatisticsFromEncoded(descr, mem, nvalues, encoded) + } return NewFixedLenByteArrayStatisticsFromEncoded(descr, mem, nvalues, encoded) default: panic("not implemented") diff --git a/go/parquet/metadata/statistics_types.gen.go.tmpl b/go/parquet/metadata/statistics_types.gen.go.tmpl index 4b6253a8574ea..93495527c7e54 100644 --- a/go/parquet/metadata/statistics_types.gen.go.tmpl +++ b/go/parquet/metadata/statistics_types.gen.go.tmpl @@ -45,10 +45,18 @@ type {{.Name}}Statistics struct { // given column descriptor and allocator. // // Panics if the physical type of descr is not parquet.Type.{{if .physical}}{{.physical}}{{else}}{{.Name}}{{end}} +{{- if eq .Name "Float16"}} +// Panics if the logical type of descr is not schema.Float16LogicalType +{{- end}} func New{{.Name}}Statistics(descr *schema.Column, mem memory.Allocator) *{{.Name}}Statistics { if descr.PhysicalType() != parquet.Types.{{if .physical}}{{.physical}}{{else}}{{.Name}}{{end}} { panic(fmt.Errorf("parquet: invalid type %s for constructing a {{.Name}} stat object", descr.PhysicalType())) } +{{- if eq .Name "Float16"}} + if !descr.LogicalType().Equals(schema.Float16LogicalType{}) { + panic(fmt.Errorf("parquet: invalid logical type %s for constructing a {{.Name}} stat object", descr.LogicalType().String())) + } +{{- end}} return &{{.Name}}Statistics{ statistics: statistics{ @@ -96,7 +104,7 @@ func (s *{{.Name}}Statistics) plainEncode(src {{.name}}) []byte { copy(out, src) return out {{- else}} - s.encoder.(encoding.{{.Name}}Encoder).Put([]{{.name}}{src}) + s.encoder.(encoding.{{if .logical}}{{.physical}}{{else}}{{.Name}}{{end}}Encoder).Put([]{{.name}}{src}) buf, err := s.encoder.FlushValues() if err != nil { panic(err) // recovered by Encode @@ -117,12 +125,12 @@ func (s *{{.Name}}Statistics) plainDecode(src []byte) {{.name}} { decoder := encoding.NewDecoder(s.descr.PhysicalType(), parquet.Encodings.Plain, s.descr, s.mem) decoder.SetData(1, src) - decoder.(encoding.{{.Name}}Decoder).Decode(buf[:]) + decoder.(encoding.{{if .logical}}{{.physical}}{{else}}{{.Name}}{{end}}Decoder).Decode(buf[:]) return buf[0] {{- end}} } -{{if and (ne .Name "ByteArray") (ne .Name "FixedLenByteArray")}} +{{if and (ne .Name "ByteArray") (ne .Name "FixedLenByteArray") (ne .Name "Float16")}} func (s *{{.Name}}Statistics) minval(a, b {{.name}}) {{.name}} { if s.less(a, b) { return a @@ -172,7 +180,11 @@ func (s *{{.Name}}Statistics) MinMaxEqual(rhs *{{.Name}}Statistics) bool { // Equals returns true only if both objects are the same type, have the same min and // max values, null count, distinct count and number of values. func (s *{{.Name}}Statistics) Equals(other TypedStatistics) bool { +{{- if .logical}} + if s.Type() != other.Type() || !s.descr.LogicalType().Equals(other.Descr().LogicalType()) { +{{- else}} if s.Type() != other.Type() { +{{- end}} return false } rhs, ok := other.(*{{.Name}}Statistics) @@ -194,6 +206,13 @@ func (s *{{.Name}}Statistics) coalesce(val, fallback {{.name}}) {{.name}} { } return val } +{{else if eq .Name "Float16"}} +func (s *{{.Name}}Statistics) coalesce(val, fallback {{.name}}) {{.name}} { + if float16.FromLEBytes(val).IsNaN() { + return fallback + } + return val +} {{end}} func (s *{{.Name}}Statistics) getMinMax(values []{{.name}}) (min, max {{.name}}) { @@ -212,7 +231,7 @@ func (s *{{.Name}}Statistics) getMinMax(values []{{.name}}) (min, max {{.name}}) max = defMax for _, v := range values { -{{- if or (eq .name "float32") (eq .name "float64") }} +{{- if or (eq .name "float32") (eq .name "float64") (eq .Name "Float16") }} min = s.minval(min, s.coalesce(v, defMin)) max = s.maxval(max, s.coalesce(v, defMax)) {{- else}} @@ -261,7 +280,7 @@ func (s *{{.Name}}Statistics) getMinMaxSpaced(values []{{.name}}, validBits []by } {{- else}} for _, v := range values[int(run.Pos):int(run.Pos+run.Length)] { -{{- if or (eq .name "float32") (eq .name "float64") }} +{{- if or (eq .name "float32") (eq .name "float64") (eq .Name "Float16") }} min = s.minval(min, coalesce(v, s.defaultMin()).({{.name}})) max = s.maxval(max, coalesce(v, s.defaultMax()).({{.name}})) {{- else}} @@ -381,7 +400,9 @@ func (s *{{.Name}}Statistics) UpdateFromArrow(values arrow.Array, updateCounts b s.SetMinMax(min, max) return nil {{else if eq .Name "Boolean"}} - return fmt.Errorf("%w: update boolean stats from Arrow", arrow.ErrNotImplemented) + return fmt.Errorf("%w: update boolean stats from Arrow", arrow.ErrNotImplemented) +{{else if eq .Name "Float16"}} + return fmt.Errorf("%w: update float16 stats from Arrow", arrow.ErrNotImplemented) {{else}} if values.DataType().(arrow.FixedWidthDataType).Bytes() != arrow.{{.Name}}SizeBytes { return fmt.Errorf("%w: cannot update {{.name}} stats with %s arrow array", @@ -475,8 +496,15 @@ func NewStatistics(descr *schema.Column, mem memory.Allocator) TypedStatistics { } switch descr.PhysicalType() { {{- range .In}} + {{- if not .logical}} case parquet.Types.{{if .physical}}{{.physical}}{{else}}{{.Name}}{{end}}: + {{- if eq .Name "FixedLenByteArray"}} + if descr.LogicalType().Equals(schema.Float16LogicalType{}) { + return NewFloat16Statistics(descr, mem) + } + {{- end}} return New{{.Name}}Statistics(descr, mem) + {{- end}} {{- end}} default: panic("not implemented") @@ -493,8 +521,15 @@ func NewStatisticsFromEncoded(descr *schema.Column, mem memory.Allocator, nvalue } switch descr.PhysicalType() { {{- range .In}} + {{- if not .logical}} case parquet.Types.{{if .physical}}{{.physical}}{{else}}{{.Name}}{{end}}: + {{- if eq .Name "FixedLenByteArray"}} + if descr.LogicalType().Equals(schema.Float16LogicalType{}) { + return NewFloat16StatisticsFromEncoded(descr, mem, nvalues, encoded) + } + {{- end}} return New{{.Name}}StatisticsFromEncoded(descr, mem, nvalues, encoded) + {{- end}} {{- end}} default: panic("not implemented") diff --git a/go/parquet/metadata/statistics_types.tmpldata b/go/parquet/metadata/statistics_types.tmpldata new file mode 100644 index 0000000000000..400c0a3ca515d --- /dev/null +++ b/go/parquet/metadata/statistics_types.tmpldata @@ -0,0 +1,60 @@ +[ + { + "Name": "Int32", + "name": "int32", + "lower": "int32", + "prefix": "arrow" + }, + { + "Name": "Int64", + "name": "int64", + "lower": "int64", + "prefix": "arrow" + }, + { + "Name": "Int96", + "name": "parquet.Int96", + "lower": "int96", + "prefix": "parquet" + }, + { + "Name": "Float32", + "name": "float32", + "lower": "float32", + "prefix": "arrow", + "physical": "Float" + }, + { + "Name": "Float64", + "name": "float64", + "lower": "float64", + "prefix": "arrow", + "physical": "Double" + }, + { + "Name": "Boolean", + "name": "bool", + "lower": "bool", + "prefix": "arrow" + }, + { + "Name": "ByteArray", + "name": "parquet.ByteArray", + "lower": "byteArray", + "prefix": "parquet" + }, + { + "Name": "FixedLenByteArray", + "name": "parquet.FixedLenByteArray", + "lower": "fixedLenByteArray", + "prefix": "parquet" + }, + { + "Name": "Float16", + "name": "parquet.FixedLenByteArray", + "lower": "float16", + "prefix": "parquet", + "physical": "FixedLenByteArray", + "logical": "Float16LogicalType" + } +] diff --git a/go/parquet/pqarrow/column_readers.go b/go/parquet/pqarrow/column_readers.go index 02f94c941c3d9..49f3fac0a3b7c 100644 --- a/go/parquet/pqarrow/column_readers.go +++ b/go/parquet/pqarrow/column_readers.go @@ -517,6 +517,14 @@ func transferColumnData(rdr file.RecordReader, valueType arrow.DataType, descr * default: return nil, errors.New("time unit not supported") } + case arrow.FLOAT16: + if descr.PhysicalType() != parquet.Types.FixedLenByteArray { + return nil, errors.New("physical type for float16 must be fixed len byte array") + } + if len := arrow.Float16SizeBytes; descr.TypeLength() != len { + return nil, fmt.Errorf("fixed len byte array length for float16 must be %d", len) + } + return transferBinary(rdr, valueType), nil default: return nil, fmt.Errorf("no support for reading columns of type: %s", valueType.Name()) } @@ -563,6 +571,14 @@ func transferBinary(rdr file.RecordReader, dt arrow.DataType) *arrow.Chunked { chunks[idx] = array.MakeFromData(chunk.Data()) chunk.Release() } + case *arrow.Float16Type: + for idx, chunk := range chunks { + data := chunk.Data() + f16_data := array.NewData(dt, data.Len(), data.Buffers(), nil, data.NullN(), data.Offset()) + defer f16_data.Release() + chunks[idx] = array.NewFloat16Data(f16_data) + chunk.Release() + } } return arrow.NewChunked(dt, chunks) } diff --git a/go/parquet/pqarrow/encode_arrow.go b/go/parquet/pqarrow/encode_arrow.go index 81b4527b105cb..1855d3625adb7 100644 --- a/go/parquet/pqarrow/encode_arrow.go +++ b/go/parquet/pqarrow/encode_arrow.go @@ -582,6 +582,31 @@ func writeDenseArrow(ctx *arrowWriteContext, cw file.ColumnChunkWriter, leafArr } wr.WriteBatchSpaced(data, defLevels, repLevels, arr.NullBitmapBytes(), int64(arr.Data().Offset())) } + case *arrow.Float16Type: + typeLen := wr.Descr().TypeLength() + if typeLen != arrow.Float16SizeBytes { + return fmt.Errorf("%w: invalid FixedLenByteArray length to write from float16 column: %d", arrow.ErrInvalid, typeLen) + } + + arr := leafArr.(*array.Float16) + rawValues := arrow.Float16Traits.CastToBytes(arr.Values()) + data := make([]parquet.FixedLenByteArray, arr.Len()) + + if arr.NullN() == 0 { + for idx := range data { + offset := idx * typeLen + data[idx] = rawValues[offset : offset+typeLen] + } + _, err = wr.WriteBatch(data, defLevels, repLevels) + } else { + for idx := range data { + if arr.IsValid(idx) { + offset := idx * typeLen + data[idx] = rawValues[offset : offset+typeLen] + } + } + wr.WriteBatchSpaced(data, defLevels, repLevels, arr.NullBitmapBytes(), int64(arr.Data().Offset())) + } default: return fmt.Errorf("%w: invalid column type to write to FixedLenByteArray: %s", arrow.ErrInvalid, leafArr.DataType().Name()) } diff --git a/go/parquet/pqarrow/encode_arrow_test.go b/go/parquet/pqarrow/encode_arrow_test.go index 281ca0d526476..d588aff701f3d 100644 --- a/go/parquet/pqarrow/encode_arrow_test.go +++ b/go/parquet/pqarrow/encode_arrow_test.go @@ -495,6 +495,8 @@ func getLogicalType(typ arrow.DataType) schema.LogicalType { return schema.DateLogicalType{} case arrow.DATE64: return schema.DateLogicalType{} + case arrow.FLOAT16: + return schema.Float16LogicalType{} case arrow.TIMESTAMP: ts := typ.(*arrow.TimestampType) adjustedUTC := len(ts.TimeZone) == 0 @@ -541,6 +543,8 @@ func getPhysicalType(typ arrow.DataType) parquet.Type { return parquet.Types.Float case arrow.FLOAT64: return parquet.Types.Double + case arrow.FLOAT16: + return parquet.Types.FixedLenByteArray case arrow.BINARY, arrow.LARGE_BINARY, arrow.STRING, arrow.LARGE_STRING: return parquet.Types.ByteArray case arrow.FIXED_SIZE_BINARY, arrow.DECIMAL: @@ -600,6 +604,8 @@ func (ps *ParquetIOTestSuite) makeSimpleSchema(typ arrow.DataType, rep parquet.R byteWidth = int32(typ.ByteWidth) case arrow.DecimalType: byteWidth = pqarrow.DecimalSize(typ.GetPrecision()) + case *arrow.Float16Type: + byteWidth = int32(typ.Bytes()) case *arrow.DictionaryType: valuesType := typ.ValueType switch dt := valuesType.(type) { @@ -607,6 +613,8 @@ func (ps *ParquetIOTestSuite) makeSimpleSchema(typ arrow.DataType, rep parquet.R byteWidth = int32(dt.ByteWidth) case arrow.DecimalType: byteWidth = pqarrow.DecimalSize(dt.GetPrecision()) + case *arrow.Float16Type: + byteWidth = int32(typ.Bytes()) } } @@ -1113,6 +1121,7 @@ var fullTypeList = []arrow.DataType{ arrow.FixedWidthTypes.Date32, arrow.PrimitiveTypes.Float32, arrow.PrimitiveTypes.Float64, + arrow.FixedWidthTypes.Float16, arrow.BinaryTypes.String, arrow.BinaryTypes.Binary, &arrow.FixedSizeBinaryType{ByteWidth: 10}, diff --git a/go/parquet/pqarrow/schema.go b/go/parquet/pqarrow/schema.go index b23c37ea39102..95c477c78b87d 100644 --- a/go/parquet/pqarrow/schema.go +++ b/go/parquet/pqarrow/schema.go @@ -344,6 +344,10 @@ func fieldToNode(name string, field arrow.Field, props *parquet.WriterProperties } else { logicalType = schema.NewTimeLogicalType(true, schema.TimeUnitMicros) } + case arrow.FLOAT16: + typ = parquet.Types.FixedLenByteArray + length = arrow.Float16SizeBytes + logicalType = schema.Float16LogicalType{} case arrow.STRUCT: return structToNode(field.Type.(*arrow.StructType), field.Name, field.Nullable, props, arrprops) case arrow.FIXED_SIZE_LIST, arrow.LIST: @@ -597,6 +601,8 @@ func arrowFromFLBA(logical schema.LogicalType, length int) (arrow.DataType, erro return arrowDecimal(logtype), nil case schema.NoLogicalType, schema.IntervalLogicalType, schema.UUIDLogicalType: return &arrow.FixedSizeBinaryType{ByteWidth: int(length)}, nil + case schema.Float16LogicalType: + return &arrow.Float16Type{}, nil default: return nil, xerrors.New("unhandled logical type " + logical.String() + " for fixed-length byte array") } diff --git a/go/parquet/pqarrow/schema_test.go b/go/parquet/pqarrow/schema_test.go index b5e7dc8fad34a..ee5aad8913470 100644 --- a/go/parquet/pqarrow/schema_test.go +++ b/go/parquet/pqarrow/schema_test.go @@ -280,6 +280,25 @@ func TestConvertArrowDecimals(t *testing.T) { } } +func TestConvertArrowFloat16(t *testing.T) { + parquetFields := make(schema.FieldList, 0) + arrowFields := make([]arrow.Field, 0) + + parquetFields = append(parquetFields, schema.Must(schema.NewPrimitiveNodeLogical("float16", parquet.Repetitions.Required, + schema.Float16LogicalType{}, parquet.Types.FixedLenByteArray, 2, -1))) + arrowFields = append(arrowFields, arrow.Field{Name: "float16", Type: &arrow.Float16Type{}}) + + arrowSchema := arrow.NewSchema(arrowFields, nil) + parquetSchema := schema.NewSchema(schema.MustGroup(schema.NewGroupNode("schema", parquet.Repetitions.Repeated, parquetFields, -1))) + + result, err := pqarrow.ToParquet(arrowSchema, nil, pqarrow.NewArrowWriterProperties(pqarrow.WithDeprecatedInt96Timestamps(true))) + assert.NoError(t, err) + assert.True(t, parquetSchema.Equals(result)) + for i := 0; i < parquetSchema.NumColumns(); i++ { + assert.Truef(t, parquetSchema.Column(i).Equals(result.Column(i)), "Column %d didn't match: %s", i, parquetSchema.Column(i).Name()) + } +} + func TestCoerceTImestampV1(t *testing.T) { parquetFields := make(schema.FieldList, 0) arrowFields := make([]arrow.Field, 0) @@ -418,7 +437,6 @@ func TestUnsupportedTypes(t *testing.T) { typ arrow.DataType }{ // Non-exhaustive list of unsupported types - {typ: &arrow.Float16Type{}}, {typ: &arrow.DurationType{}}, {typ: &arrow.DayTimeIntervalType{}}, {typ: &arrow.MonthIntervalType{}}, diff --git a/go/parquet/schema/logical_types.go b/go/parquet/schema/logical_types.go index 1ea44fc56c615..69e69363887cd 100644 --- a/go/parquet/schema/logical_types.go +++ b/go/parquet/schema/logical_types.go @@ -68,6 +68,8 @@ func getLogicalType(l *format.LogicalType) LogicalType { return BSONLogicalType{} case l.IsSetUUID(): return UUIDLogicalType{} + case l.IsSetFLOAT16(): + return Float16LogicalType{} case l == nil: return NoLogicalType{} default: @@ -1064,6 +1066,50 @@ func (IntervalLogicalType) Equals(rhs LogicalType) bool { return ok } +// Float16LogicalType can only be used with a FixedLength byte array column +// that is exactly 2 bytes long +type Float16LogicalType struct{ baseLogicalType } + +func (Float16LogicalType) SortOrder() SortOrder { + return SortSIGNED +} + +func (Float16LogicalType) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]string{"Type": Float16LogicalType{}.String()}) +} + +func (Float16LogicalType) String() string { + return "Float16" +} + +func (Float16LogicalType) ToConvertedType() (ConvertedType, DecimalMetadata) { + return ConvertedTypes.None, DecimalMetadata{} +} + +func (Float16LogicalType) IsCompatible(c ConvertedType, dec DecimalMetadata) bool { + if dec.IsSet { + return false + } + switch c { + case ConvertedTypes.None, ConvertedTypes.NA: + return true + } + return false +} + +func (Float16LogicalType) IsApplicable(t parquet.Type, tlen int32) bool { + return t == parquet.Types.FixedLenByteArray && tlen == 2 +} + +func (Float16LogicalType) toThrift() *format.LogicalType { + return &format.LogicalType{FLOAT16: format.NewFloat16Type()} +} + +func (Float16LogicalType) Equals(rhs LogicalType) bool { + _, ok := rhs.(Float16LogicalType) + return ok +} + type NullLogicalType struct{ baseLogicalType } func (NullLogicalType) SortOrder() SortOrder { diff --git a/go/parquet/schema/logical_types_test.go b/go/parquet/schema/logical_types_test.go index 49edf1748c52c..c371b47714f41 100644 --- a/go/parquet/schema/logical_types_test.go +++ b/go/parquet/schema/logical_types_test.go @@ -158,6 +158,7 @@ func TestNewTypeIncompatibility(t *testing.T) { expected schema.LogicalType }{ {"uuid", schema.UUIDLogicalType{}, schema.UUIDLogicalType{}}, + {"float16", schema.Float16LogicalType{}, schema.Float16LogicalType{}}, {"null", schema.NullLogicalType{}, schema.NullLogicalType{}}, {"not-utc-time_milli", schema.NewTimeLogicalType(false /* adjutedToUTC */, schema.TimeUnitMillis), &schema.TimeLogicalType{}}, {"not-utc-time-micro", schema.NewTimeLogicalType(false /* adjutedToUTC */, schema.TimeUnitMicros), &schema.TimeLogicalType{}}, @@ -224,6 +225,7 @@ func TestLogicalTypeProperties(t *testing.T) { {"json", schema.JSONLogicalType{}, false, true, true}, {"bson", schema.BSONLogicalType{}, false, true, true}, {"uuid", schema.UUIDLogicalType{}, false, true, true}, + {"float16", schema.Float16LogicalType{}, false, true, true}, {"nological", schema.NoLogicalType{}, false, false, true}, {"unknown", schema.UnknownLogicalType{}, false, false, false}, } @@ -358,6 +360,14 @@ func TestLogicalInapplicableTypes(t *testing.T) { assert.False(t, logical.IsApplicable(tt.typ, tt.len)) }) } + + logical = schema.Float16LogicalType{} + assert.True(t, logical.IsApplicable(parquet.Types.FixedLenByteArray, 2)) + for _, tt := range tests { + t.Run("float16 "+tt.name, func(t *testing.T) { + assert.False(t, logical.IsApplicable(tt.typ, tt.len)) + }) + } } func TestDecimalLogicalTypeApplicability(t *testing.T) { @@ -445,6 +455,7 @@ func TestLogicalTypeRepresentation(t *testing.T) { {"json", schema.JSONLogicalType{}, "JSON", `{"Type": "JSON"}`}, {"bson", schema.BSONLogicalType{}, "BSON", `{"Type": "BSON"}`}, {"uuid", schema.UUIDLogicalType{}, "UUID", `{"Type": "UUID"}`}, + {"float16", schema.Float16LogicalType{}, "Float16", `{"Type": "Float16"}`}, {"none", schema.NoLogicalType{}, "None", `{"Type": "None"}`}, } @@ -490,6 +501,7 @@ func TestLogicalTypeSortOrder(t *testing.T) { {"json", schema.JSONLogicalType{}, schema.SortUNSIGNED}, {"bson", schema.BSONLogicalType{}, schema.SortUNSIGNED}, {"uuid", schema.UUIDLogicalType{}, schema.SortUNSIGNED}, + {"float16", schema.Float16LogicalType{}, schema.SortSIGNED}, {"none", schema.NoLogicalType{}, schema.SortUNKNOWN}, } diff --git a/go/parquet/schema/reflection.go b/go/parquet/schema/reflection.go index d79edb92408f8..c0c8e0533efb0 100644 --- a/go/parquet/schema/reflection.go +++ b/go/parquet/schema/reflection.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" + "github.com/apache/arrow/go/v15/arrow/float16" "github.com/apache/arrow/go/v15/parquet" format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" "golang.org/x/xerrors" @@ -159,6 +160,8 @@ func (t *taggedInfo) UpdateLogicalTypes() { return BSONLogicalType{} case "uuid": return UUIDLogicalType{} + case "float16": + return Float16LogicalType{} default: panic(fmt.Errorf("invalid logical type specified: %s", t)) } @@ -373,6 +376,9 @@ func typeToNode(name string, typ reflect.Type, repType parquet.Repetition, info } return Must(MapOf(name, key, value, repType, fieldID)) case reflect.Struct: + if typ == reflect.TypeOf(float16.Num{}) { + return MustPrimitive(NewPrimitiveNodeLogical(name, repType, Float16LogicalType{}, parquet.Types.FixedLenByteArray, 2, fieldID)) + } // structs are Group nodes fields := make(FieldList, 0) for i := 0; i < typ.NumField(); i++ { diff --git a/go/parquet/schema/reflection_test.go b/go/parquet/schema/reflection_test.go index 06ad7191a5749..e3a880cacc1e8 100644 --- a/go/parquet/schema/reflection_test.go +++ b/go/parquet/schema/reflection_test.go @@ -22,6 +22,7 @@ import ( "reflect" "testing" + "github.com/apache/arrow/go/v15/arrow/float16" "github.com/apache/arrow/go/v15/parquet" "github.com/apache/arrow/go/v15/parquet/schema" "github.com/stretchr/testify/assert" @@ -152,6 +153,9 @@ func ExampleNewSchemaFromStruct_logicaltypes() { JSON string `parquet:"logical=json"` BSON []byte `parquet:"logical=BSON"` UUID [16]byte `parquet:"logical=uuid"` + Float16 [2]byte `parquet:"logical=float16"` + Float16Optional *[2]byte `parquet:"logical=float16"` + Float16Num float16.Num } sc, err := schema.NewSchemaFromStruct(LogicalTypes{}) @@ -180,6 +184,9 @@ func ExampleNewSchemaFromStruct_logicaltypes() { // required byte_array field_id=-1 JSON (JSON); // required byte_array field_id=-1 BSON (BSON); // required fixed_len_byte_array field_id=-1 UUID (UUID); + // required fixed_len_byte_array field_id=-1 Float16 (Float16); + // optional fixed_len_byte_array field_id=-1 Float16Optional (Float16); + // required fixed_len_byte_array field_id=-1 Float16Num (Float16); // } } diff --git a/go/parquet/schema/schema_element_test.go b/go/parquet/schema/schema_element_test.go index dd1d293e5cfd7..d190ffe5a253a 100644 --- a/go/parquet/schema/schema_element_test.go +++ b/go/parquet/schema/schema_element_test.go @@ -159,6 +159,10 @@ func (s *SchemaElementConstructionSuite) TestSimple() { "uuid", UUIDLogicalType{}, parquet.Types.FixedLenByteArray, 16, false, ConvertedTypes.NA, true, func(e *format.SchemaElement) bool { return e.LogicalType.IsSetUUID() }, }, nil}, + {"float16", &schemaElementConstructArgs{ + "float16", Float16LogicalType{}, parquet.Types.FixedLenByteArray, 2, false, ConvertedTypes.NA, true, + func(e *format.SchemaElement) bool { return e.LogicalType.IsSetFLOAT16() }, + }, nil}, {"none", &schemaElementConstructArgs{ "none", NoLogicalType{}, parquet.Types.Int64, -1, false, ConvertedTypes.NA, false, checkNone, @@ -425,7 +429,8 @@ func TestSchemaElementNestedSerialization(t *testing.T) { timestampNode := MustPrimitive(NewPrimitiveNodeLogical("timestamp" /*name */, parquet.Repetitions.Required, NewTimestampLogicalType(false /* adjustedToUTC */, TimeUnitNanos), parquet.Types.Int64, -1 /* type len */, -1 /* fieldID */)) intNode := MustPrimitive(NewPrimitiveNodeLogical("int" /*name */, parquet.Repetitions.Required, NewIntLogicalType(64 /* bitWidth */, false /* signed */), parquet.Types.Int64, -1 /* type len */, -1 /* fieldID */)) decimalNode := MustPrimitive(NewPrimitiveNodeLogical("decimal" /*name */, parquet.Repetitions.Required, NewDecimalLogicalType(16 /* precision */, 6 /* scale */), parquet.Types.Int64, -1 /* type len */, -1 /* fieldID */)) - listNode := MustGroup(NewGroupNodeLogical("list" /*name */, parquet.Repetitions.Repeated, []Node{strNode, dateNode, jsonNode, uuidNode, timestampNode, intNode, decimalNode}, NewListLogicalType(), -1 /* fieldID */)) + float16Node := MustPrimitive(NewPrimitiveNodeLogical("float16" /*name */, parquet.Repetitions.Required, Float16LogicalType{}, parquet.Types.FixedLenByteArray, 2 /* type len */, - /* fieldID */ 1)) + listNode := MustGroup(NewGroupNodeLogical("list" /*name */, parquet.Repetitions.Repeated, []Node{strNode, dateNode, jsonNode, uuidNode, timestampNode, intNode, decimalNode, float16Node}, NewListLogicalType(), -1 /* fieldID */)) listElems := ToThrift(listNode) assert.Equal(t, "list", listElems[0].Name) @@ -440,6 +445,7 @@ func TestSchemaElementNestedSerialization(t *testing.T) { assert.True(t, listElems[5].LogicalType.IsSetTIMESTAMP()) assert.True(t, listElems[6].LogicalType.IsSetINTEGER()) assert.True(t, listElems[7].LogicalType.IsSetDECIMAL()) + assert.True(t, listElems[8].LogicalType.IsSetFLOAT16()) mapNode := MustGroup(NewGroupNodeLogical("map" /* name */, parquet.Repetitions.Required, []Node{}, MapLogicalType{}, -1 /* fieldID */)) mapElems := ToThrift(mapNode) @@ -486,6 +492,7 @@ func TestLogicalTypeSerializationRoundTrip(t *testing.T) { {"json", JSONLogicalType{}, parquet.Types.ByteArray, -1}, {"bson", BSONLogicalType{}, parquet.Types.ByteArray, -1}, {"uuid", UUIDLogicalType{}, parquet.Types.FixedLenByteArray, 16}, + {"float16", Float16LogicalType{}, parquet.Types.FixedLenByteArray, 2}, {"none", NoLogicalType{}, parquet.Types.Boolean, -1}, } diff --git a/go/parquet/schema/schema_test.go b/go/parquet/schema/schema_test.go index b60c7dfaafcd1..cc43c3856d68e 100644 --- a/go/parquet/schema/schema_test.go +++ b/go/parquet/schema/schema_test.go @@ -635,6 +635,10 @@ func TestPanicSchemaNodeCreation(t *testing.T) { schema.MustPrimitive(schema.NewPrimitiveNodeLogical("uuid" /* name */, parquet.Repetitions.Required, schema.UUIDLogicalType{}, parquet.Types.FixedLenByteArray, 64 /* type len */, -1 /* fieldID */)) }, "incompatible primitive length") + assert.Panics(t, func() { + schema.MustPrimitive(schema.NewPrimitiveNodeLogical("float16" /* name */, parquet.Repetitions.Required, schema.Float16LogicalType{}, parquet.Types.FixedLenByteArray, 4 /* type len */, -1 /* fieldID */)) + }, "incompatible primitive length") + assert.Panics(t, func() { schema.MustPrimitive(schema.NewPrimitiveNodeLogical("negative_len" /* name */, parquet.Repetitions.Required, schema.NoLogicalType{}, parquet.Types.FixedLenByteArray, -16 /* type len */, -1 /* fieldID */)) }, "non-positive length for fixed length binary") From fba8389e6e9c41613cf13e9557311b3aec0f349b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:20:33 +0900 Subject: [PATCH 41/57] MINOR: [C#] Bump Grpc.AspNetCore from 2.58.0 to 2.59.0 in /csharp (#38685) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Grpc.AspNetCore](https://github.com/grpc/grpc-dotnet) from 2.58.0 to 2.59.0.
Release notes

Sourced from Grpc.AspNetCore's releases.

Release v2.59.0

What's Changed

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.58.0...v2.59.0

Release v2.59.0-pre1

What's Changed

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.58.0...v2.59.0-pre1

Commits
  • 9124a4c Merge pull request #2316 from apolcyn/bump_r
  • eb9f932 bump to 2.59
  • 20a6e90 Merge pull request #2307 from apolcyn/release_updates
  • 2084009 bump version to 2.59.0-pre1
  • e52049a Merge pull request #2303 from apolcyn/update_59
  • 51ad5ab update grpc tools to 2.59
  • 3c4abba Bump @​babel/traverse from 7.17.3 to 7.23.2 in /examples/Spar/Server/ClientApp...
  • 91be392 Fix HttpContext race by copying values to reader and writer (#2294)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Grpc.AspNetCore&package-manager=nuget&previous-version=2.58.0&new-version=2.59.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- .../Apache.Arrow.Flight.TestWeb.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj b/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj index 81468deea6c09..3aaebb103f9da 100644 --- a/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj +++ b/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj @@ -5,7 +5,7 @@
- + From e7d90b2ea1b56b7b23305efc3653679029ce3e4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:20:57 +0900 Subject: [PATCH 42/57] MINOR: [C#] Bump Grpc.Net.Client from 2.58.0 to 2.59.0 in /csharp (#38686) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Grpc.Net.Client](https://github.com/grpc/grpc-dotnet) from 2.58.0 to 2.59.0.
Release notes

Sourced from Grpc.Net.Client's releases.

Release v2.59.0

What's Changed

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.58.0...v2.59.0

Release v2.59.0-pre1

What's Changed

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.58.0...v2.59.0-pre1

Commits
  • 9124a4c Merge pull request #2316 from apolcyn/bump_r
  • eb9f932 bump to 2.59
  • 20a6e90 Merge pull request #2307 from apolcyn/release_updates
  • 2084009 bump version to 2.59.0-pre1
  • e52049a Merge pull request #2303 from apolcyn/update_59
  • 51ad5ab update grpc tools to 2.59
  • 3c4abba Bump @​babel/traverse from 7.17.3 to 7.23.2 in /examples/Spar/Server/ClientApp...
  • 91be392 Fix HttpContext race by copying values to reader and writer (#2294)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Grpc.Net.Client&package-manager=nuget&previous-version=2.58.0&new-version=2.59.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj index 975bf44db888f..67b37e49c7dc5 100644 --- a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj +++ b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj @@ -6,7 +6,7 @@ - + From 37db2a03fc2d2c54d207fdcf04f1b88c1c1ae11b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:21:38 +0900 Subject: [PATCH 43/57] MINOR: [C#] Bump Microsoft.NET.Test.Sdk from 17.7.2 to 17.8.0 in /csharp (#38687) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Microsoft.NET.Test.Sdk](https://github.com/microsoft/vstest) from 17.7.2 to 17.8.0.
Release notes

Sourced from Microsoft.NET.Test.Sdk's releases.

v17.8.0

What's Changed

Full Changelog: https://github.com/microsoft/vstest/compare/v17.7.2...v17.8.0

v17.8.0-release-23468-02

What's Changed

Full Changelog: https://github.com/microsoft/vstest/compare/v17.8.0-preview-23424-02...v17.8.0-release-23468-02

v17.8.0-preview-23424-02

What's Changed

New Contributors

Full Changelog: https://github.com/microsoft/vstest/compare/v17.8.0-preview-23371-04...v17.8.0-preview-23424-02

v17.8.0-preview-23371-04

Issues Fixed

  • Take System dlls from testhost folder #4610
  • Simplify naming in ExecutionTests
  • Fix XmlException when writing Html log with certain test names #4576

Full Changelog: here

Drops

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Microsoft.NET.Test.Sdk&package-manager=nuget&previous-version=17.7.2&new-version=17.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- .../Apache.Arrow.Compression.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Sql.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj | 2 +- csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj index 835d95fc0c521..475d7ccc3ef28 100644 --- a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj +++ b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj index 10582500d443c..656ee6a2470e4 100644 --- a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj index b371de9b1eb99..53fdd6d62dbcb 100644 --- a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj index 0df097156da34..66becb84c5b66 100644 --- a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj +++ b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj @@ -14,7 +14,7 @@ - + all From 0815a70948b81c1f89252dcdb0c083991fef44a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:21:46 +0900 Subject: [PATCH 44/57] MINOR: [C#] Bump Grpc.AspNetCore.Server from 2.58.0 to 2.59.0 in /csharp (#38688) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Grpc.AspNetCore.Server](https://github.com/grpc/grpc-dotnet) from 2.58.0 to 2.59.0.
Release notes

Sourced from Grpc.AspNetCore.Server's releases.

Release v2.59.0

What's Changed

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.58.0...v2.59.0

Release v2.59.0-pre1

What's Changed

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.58.0...v2.59.0-pre1

Commits
  • 9124a4c Merge pull request #2316 from apolcyn/bump_r
  • eb9f932 bump to 2.59
  • 20a6e90 Merge pull request #2307 from apolcyn/release_updates
  • 2084009 bump version to 2.59.0-pre1
  • e52049a Merge pull request #2303 from apolcyn/update_59
  • 51ad5ab update grpc tools to 2.59
  • 3c4abba Bump @​babel/traverse from 7.17.3 to 7.23.2 in /examples/Spar/Server/ClientApp...
  • 91be392 Fix HttpContext race by copying values to reader and writer (#2294)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Grpc.AspNetCore.Server&package-manager=nuget&previous-version=2.58.0&new-version=2.59.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- .../Apache.Arrow.Flight.AspNetCore.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj b/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj index 47d7e9746dea1..04cf06b1ec440 100644 --- a/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj +++ b/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj @@ -5,7 +5,7 @@ - + From 9e5c5b5274928c92a642657e73e8cf84f17bd7b7 Mon Sep 17 00:00:00 2001 From: Yue Date: Tue, 14 Nov 2023 08:18:10 +0800 Subject: [PATCH 45/57] GH-38653: [Packaging][Java][Python][Ruby] Raise the minimum macOS version to 10.15 catalina to allow using new APIs in C++17 (#38677) ### Rationale for this change For macOS, some new APIs in C++17, e.g. `std::filesystem::path` is only visible when the compiling target is >= macOS 10.15 (Catalina). But currently the minimum macOS target is set to 10.14 (Mojave). * macOS 10.14 Mojave was released on 2018-09-24 [1] * macOS 10.15 Catalina was released on 2019-10-07 [2] * Both macOS 10.14 and 10.15 are end of life [3] There is a [similar issue](https://github.com/ClickHouse/ClickHouse/issues/8541) [4] and [its fix](https://github.com/ClickHouse/ClickHouse/pull/8600) [5] for ClickHouse on year 2020, which raised minimum macOS version from 10.14 to 10.15. ### What changes are included in this PR? Raise the minimum macOS version from 10.14 to 10.15 ### Are these changes tested? This should be tested and verified on CI. ### Are there any user-facing changes? * Users using macOS <= 10.14 may not able to run these CI jobs locally. * The new version of python wheels may not be able to be deployed to environments with macOS <= 10.14 (not completely sure how this affects the deployment) * Closes: #38653 ### References * [1] https://en.wikipedia.org/wiki/MacOS_Mojave * [2] https://en.wikipedia.org/wiki/MacOS_Catalina * [3] https://endoflife.date/macos * [4] https://github.com/ClickHouse/ClickHouse/issues/8541 * [5] https://github.com/ClickHouse/ClickHouse/pull/8600 Authored-by: Yue Ni Signed-off-by: Sutou Kouhei --- ci/scripts/python_wheel_macos_build.sh | 2 +- ci/vcpkg/universal2-osx-static-debug.cmake | 2 +- ci/vcpkg/universal2-osx-static-release.cmake | 2 +- dev/tasks/java-jars/github.yml | 2 +- dev/tasks/tasks.yml | 2 +- python/CMakeLists.txt | 2 +- ruby/red-arrow/ext/arrow/extconf.rb | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index 5a3c6fb6d1f6f..fd845c512dcdb 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -34,7 +34,7 @@ rm -rf ${source_dir}/python/pyarrow/*.so.* echo "=== (${PYTHON_VERSION}) Set SDK, C++ and Wheel flags ===" export _PYTHON_HOST_PLATFORM="macosx-${MACOSX_DEPLOYMENT_TARGET}-${arch}" -export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET:-10.14} +export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET:-10.15} export SDKROOT=${SDKROOT:-$(xcrun --sdk macosx --show-sdk-path)} if [ $arch = "arm64" ]; then diff --git a/ci/vcpkg/universal2-osx-static-debug.cmake b/ci/vcpkg/universal2-osx-static-debug.cmake index 580b4604d522f..8abc1ebf838f1 100644 --- a/ci/vcpkg/universal2-osx-static-debug.cmake +++ b/ci/vcpkg/universal2-osx-static-debug.cmake @@ -21,6 +21,6 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES "x86_64;arm64") -set(VCPKG_OSX_DEPLOYMENT_TARGET "10.14") +set(VCPKG_OSX_DEPLOYMENT_TARGET "10.15") set(VCPKG_BUILD_TYPE debug) diff --git a/ci/vcpkg/universal2-osx-static-release.cmake b/ci/vcpkg/universal2-osx-static-release.cmake index 7247d0af351c5..2eb36c15175b2 100644 --- a/ci/vcpkg/universal2-osx-static-release.cmake +++ b/ci/vcpkg/universal2-osx-static-release.cmake @@ -21,6 +21,6 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES "x86_64;arm64") -set(VCPKG_OSX_DEPLOYMENT_TARGET "10.14") +set(VCPKG_OSX_DEPLOYMENT_TARGET "10.15") set(VCPKG_BUILD_TYPE release) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 7dc53a35ef402..fbce12ee427e1 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -81,7 +81,7 @@ jobs: - { runs_on: ["macos-latest"], arch: "x86_64"} - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], arch: "aarch_64" } env: - MACOSX_DEPLOYMENT_TARGET: "10.13" + MACOSX_DEPLOYMENT_TARGET: "10.15" steps: {{ macros.github_checkout_arrow()|indent }} - name: Set up Python diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index b8d2f6c0a0086..15fac25d26d65 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -487,7 +487,7 @@ tasks: {############################## Wheel OSX ####################################} -{% for macos_version, macos_codename in [("10.14", "mojave")] %} +{% for macos_version, macos_codename in [("10.15", "catalina")] %} {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %} wheel-macos-{{ macos_codename }}-{{ python_tag }}-amd64: diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 86285c3dc381b..529265235c746 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -84,7 +84,7 @@ set(CMAKE_MACOSX_RPATH 1) if(DEFINED ENV{MACOSX_DEPLOYMENT_TARGET}) set(CMAKE_OSX_DEPLOYMENT_TARGET $ENV{MACOSX_DEPLOYMENT_TARGET}) else() - set(CMAKE_OSX_DEPLOYMENT_TARGET 10.14) + set(CMAKE_OSX_DEPLOYMENT_TARGET 10.15) endif() # Generate a Clang compile_commands.json "compilation database" file for use diff --git a/ruby/red-arrow/ext/arrow/extconf.rb b/ruby/red-arrow/ext/arrow/extconf.rb index 8043bb689fc17..7ef3c6c83432a 100644 --- a/ruby/red-arrow/ext/arrow/extconf.rb +++ b/ruby/red-arrow/ext/arrow/extconf.rb @@ -84,7 +84,7 @@ symbols_in_external_bundles.each do |symbol| $DLDFLAGS << " -Wl,-U,#{symbol}" end - mmacosx_version_min = "-mmacosx-version-min=10.14" + mmacosx_version_min = "-mmacosx-version-min=10.15" $CFLAGS << " #{mmacosx_version_min}" $CXXFLAGS << " #{mmacosx_version_min}" end From 160d45c251b5041b9688ff68c3a7c5e091a50989 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Mon, 13 Nov 2023 22:21:12 -0900 Subject: [PATCH 46/57] MINOR: [Docs] Tweak text on docs index page (#38695) ### Rationale for this change These are just some minor style tweaks which made the text and buttons on these cards read more naturally to me. Feel free to ignore or take only some of the changes here. ### Are these changes tested? Yes, I confirmed the docs still build as expected locally. ### Are there any user-facing changes? New language in docs, see above. Authored-by: Bryce Mecum Signed-off-by: AlenkaF --- docs/source/index.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index d01c74f9a482e..8407813bd7abb 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -46,8 +46,8 @@ target environment.** :class-card: contrib-card :shadow: none - Read about the Apache Arrow format - specifications and Protocols. + Read about the Apache Arrow format and its related specifications and + protocols. +++ @@ -57,17 +57,15 @@ target environment.** :color: primary :expand: - To the Specifications + To Specifications .. grid-item-card:: Development :class-card: contrib-card :shadow: none - Find the documentation on the topic of - contributions, reviews, building of the libraries - from source, building of the documentation, - continuous integration, benchmarks and the - release process. + Find documentation on building the libraries from source, building the + documentation, contributing and code reviews, continuous integration, + benchmarking, and the release process. +++ @@ -77,7 +75,7 @@ target environment.** :color: primary :expand: - To the Development + To Development .. _toc.columnar: From a4080209a97a5d66accdeb71c5c1ffa982fed51e Mon Sep 17 00:00:00 2001 From: James Duong Date: Tue, 14 Nov 2023 05:10:36 -0800 Subject: [PATCH 47/57] GH-38662: [Java] Add comparators (#38669) ### Rationale for this change Add missing Default VectorValueComparators for some more types. ### What changes are included in this PR? Add comparators for: - FixedSizeBinaryVector - LargeListVector - FixedSizeListVector - NullVector ### Are these changes tested? Yes, unit tests added. ### Are there any user-facing changes? No * Closes: #38662 Authored-by: James Duong Signed-off-by: David Li --- .../sort/DefaultVectorComparators.java | 140 ++++++++++++++++-- .../sort/TestDefaultVectorComparator.java | 132 +++++++++++++++++ 2 files changed, 259 insertions(+), 13 deletions(-) diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/DefaultVectorComparators.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/DefaultVectorComparators.java index 4f9c8b7d71bab..588876aa99059 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/DefaultVectorComparators.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/DefaultVectorComparators.java @@ -32,11 +32,13 @@ import org.apache.arrow.vector.Decimal256Vector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DurationVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.IntervalDayVector; import org.apache.arrow.vector.IntervalMonthDayNanoVector; +import org.apache.arrow.vector.NullVector; import org.apache.arrow.vector.SmallIntVector; import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeMilliVector; @@ -50,7 +52,9 @@ import org.apache.arrow.vector.UInt8Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VariableWidthVector; -import org.apache.arrow.vector.complex.BaseRepeatedValueVector; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.RepeatedValueVector; +import org.apache.arrow.vector.holders.NullableFixedSizeBinaryHolder; /** * Default comparator implementations for different types of vectors. @@ -111,13 +115,21 @@ public static VectorValueComparator createDefaultComp return (VectorValueComparator) new TimeSecComparator(); } else if (vector instanceof TimeStampVector) { return (VectorValueComparator) new TimeStampComparator(); + } else if (vector instanceof FixedSizeBinaryVector) { + return (VectorValueComparator) new FixedSizeBinaryComparator(); } } else if (vector instanceof VariableWidthVector) { return (VectorValueComparator) new VariableWidthComparator(); - } else if (vector instanceof BaseRepeatedValueVector) { + } else if (vector instanceof RepeatedValueVector) { VectorValueComparator innerComparator = - createDefaultComparator(((BaseRepeatedValueVector) vector).getDataVector()); + createDefaultComparator(((RepeatedValueVector) vector).getDataVector()); return new RepeatedValueComparator(innerComparator); + } else if (vector instanceof FixedSizeListVector) { + VectorValueComparator innerComparator = + createDefaultComparator(((FixedSizeListVector) vector).getDataVector()); + return new FixedSizeListComparator(innerComparator); + } else if (vector instanceof NullVector) { + return (VectorValueComparator) new NullComparator(); } throw new IllegalArgumentException("No default comparator for " + vector.getClass().getCanonicalName()); @@ -674,6 +686,61 @@ public VectorValueComparator createNew() { } } + /** + * Default comparator for {@link org.apache.arrow.vector.FixedSizeBinaryVector}. + * The comparison is in lexicographic order, with null comes first. + */ + public static class FixedSizeBinaryComparator extends VectorValueComparator { + + @Override + public int compare(int index1, int index2) { + NullableFixedSizeBinaryHolder holder1 = new NullableFixedSizeBinaryHolder(); + NullableFixedSizeBinaryHolder holder2 = new NullableFixedSizeBinaryHolder(); + vector1.get(index1, holder1); + vector2.get(index2, holder2); + + return ByteFunctionHelpers.compare( + holder1.buffer, 0, holder1.byteWidth, holder2.buffer, 0, holder2.byteWidth); + } + + @Override + public int compareNotNull(int index1, int index2) { + NullableFixedSizeBinaryHolder holder1 = new NullableFixedSizeBinaryHolder(); + NullableFixedSizeBinaryHolder holder2 = new NullableFixedSizeBinaryHolder(); + vector1.get(index1, holder1); + vector2.get(index2, holder2); + + return ByteFunctionHelpers.compare( + holder1.buffer, 0, holder1.byteWidth, holder2.buffer, 0, holder2.byteWidth); + } + + @Override + public VectorValueComparator createNew() { + return new FixedSizeBinaryComparator(); + } + } + + /** + * Default comparator for {@link org.apache.arrow.vector.NullVector}. + */ + public static class NullComparator extends VectorValueComparator { + @Override + public int compare(int index1, int index2) { + // Values are always equal (and are always null). + return 0; + } + + @Override + public int compareNotNull(int index1, int index2) { + throw new AssertionError("Cannot compare non-null values in a NullVector."); + } + + @Override + public VectorValueComparator createNew() { + return new NullComparator(); + } + } + /** * Default comparator for {@link org.apache.arrow.vector.VariableWidthVector}. * The comparison is in lexicographic order, with null comes first. @@ -705,14 +772,14 @@ public VectorValueComparator createNew() { } /** - * Default comparator for {@link BaseRepeatedValueVector}. + * Default comparator for {@link RepeatedValueVector}. * It works by comparing the underlying vector in a lexicographic order. * @param inner vector type. */ public static class RepeatedValueComparator - extends VectorValueComparator { + extends VectorValueComparator { - private VectorValueComparator innerComparator; + private final VectorValueComparator innerComparator; public RepeatedValueComparator(VectorValueComparator innerComparator) { this.innerComparator = innerComparator; @@ -720,16 +787,16 @@ public RepeatedValueComparator(VectorValueComparator innerComparator) { @Override public int compareNotNull(int index1, int index2) { - int startIdx1 = vector1.getOffsetBuffer().getInt(index1 * OFFSET_WIDTH); - int startIdx2 = vector2.getOffsetBuffer().getInt(index2 * OFFSET_WIDTH); + int startIdx1 = vector1.getOffsetBuffer().getInt((long) index1 * OFFSET_WIDTH); + int startIdx2 = vector2.getOffsetBuffer().getInt((long) index2 * OFFSET_WIDTH); - int endIdx1 = vector1.getOffsetBuffer().getInt((index1 + 1) * OFFSET_WIDTH); - int endIdx2 = vector2.getOffsetBuffer().getInt((index2 + 1) * OFFSET_WIDTH); + int endIdx1 = vector1.getOffsetBuffer().getInt((long) (index1 + 1) * OFFSET_WIDTH); + int endIdx2 = vector2.getOffsetBuffer().getInt((long) (index2 + 1) * OFFSET_WIDTH); int length1 = endIdx1 - startIdx1; int length2 = endIdx2 - startIdx2; - int length = length1 < length2 ? length1 : length2; + int length = Math.min(length1, length2); for (int i = 0; i < length; i++) { int result = innerComparator.compare(startIdx1 + i, startIdx2 + i); @@ -741,13 +808,60 @@ public int compareNotNull(int index1, int index2) { } @Override - public VectorValueComparator createNew() { + public VectorValueComparator createNew() { VectorValueComparator newInnerComparator = innerComparator.createNew(); return new RepeatedValueComparator<>(newInnerComparator); } @Override - public void attachVectors(BaseRepeatedValueVector vector1, BaseRepeatedValueVector vector2) { + public void attachVectors(RepeatedValueVector vector1, RepeatedValueVector vector2) { + this.vector1 = vector1; + this.vector2 = vector2; + + innerComparator.attachVectors((T) vector1.getDataVector(), (T) vector2.getDataVector()); + } + } + + /** + * Default comparator for {@link RepeatedValueVector}. + * It works by comparing the underlying vector in a lexicographic order. + * @param inner vector type. + */ + public static class FixedSizeListComparator + extends VectorValueComparator { + + private final VectorValueComparator innerComparator; + + public FixedSizeListComparator(VectorValueComparator innerComparator) { + this.innerComparator = innerComparator; + } + + @Override + public int compareNotNull(int index1, int index2) { + int length1 = vector1.getListSize(); + int length2 = vector2.getListSize(); + + int length = Math.min(length1, length2); + int startIdx1 = vector1.getElementStartIndex(index1); + int startIdx2 = vector2.getElementStartIndex(index2); + + for (int i = 0; i < length; i++) { + int result = innerComparator.compare(startIdx1 + i, startIdx2 + i); + if (result != 0) { + return result; + } + } + return length1 - length2; + } + + @Override + public VectorValueComparator createNew() { + VectorValueComparator newInnerComparator = innerComparator.createNew(); + return new FixedSizeListComparator<>(newInnerComparator); + } + + @Override + public void attachVectors(FixedSizeListVector vector1, FixedSizeListVector vector2) { this.vector1 = vector1; this.vector2 = vector2; diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java index bdae85110aa62..43c634b7647fb 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java @@ -31,12 +31,14 @@ import org.apache.arrow.vector.Decimal256Vector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DurationVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.IntervalDayVector; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; +import org.apache.arrow.vector.NullVector; import org.apache.arrow.vector.SmallIntVector; import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeMilliVector; @@ -52,6 +54,8 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.LargeListVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; import org.apache.arrow.vector.types.TimeUnit; @@ -158,6 +162,61 @@ public void testCopiedComparatorForLists() { } } + private FixedSizeListVector createFixedSizeListVector(int count) { + FixedSizeListVector listVector = FixedSizeListVector.empty("list vector", count, allocator); + Types.MinorType type = Types.MinorType.INT; + listVector.addOrGetVector(FieldType.nullable(type.getType())); + listVector.allocateNew(); + + IntVector dataVector = (IntVector) listVector.getDataVector(); + + for (int i = 0; i < count; i++) { + dataVector.set(i, i); + } + dataVector.setValueCount(count); + + listVector.setNotNull(0); + listVector.setValueCount(1); + + return listVector; + } + + @Test + public void testCompareFixedSizeLists() { + try (FixedSizeListVector listVector1 = createFixedSizeListVector(10); + FixedSizeListVector listVector2 = createFixedSizeListVector(11)) { + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(listVector1); + comparator.attachVectors(listVector1, listVector2); + + // prefix is smaller + assertTrue(comparator.compare(0, 0) < 0); + } + + try (FixedSizeListVector listVector1 = createFixedSizeListVector(11); + FixedSizeListVector listVector2 = createFixedSizeListVector(11)) { + ((IntVector) listVector2.getDataVector()).set(10, 110); + + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(listVector1); + comparator.attachVectors(listVector1, listVector2); + + // breaking tie by the last element + assertTrue(comparator.compare(0, 0) < 0); + } + + try (FixedSizeListVector listVector1 = createFixedSizeListVector(10); + FixedSizeListVector listVector2 = createFixedSizeListVector(10)) { + + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(listVector1); + comparator.attachVectors(listVector1, listVector2); + + // list vector elements equal + assertTrue(comparator.compare(0, 0) == 0); + } + } + @Test public void testCompareUInt1() { try (UInt1Vector vec = new UInt1Vector("", allocator)) { @@ -845,6 +904,65 @@ public void testCompareTimeStamp() { } } + @Test + public void testCompareFixedSizeBinary() { + try (FixedSizeBinaryVector vector1 = new FixedSizeBinaryVector("test1", allocator, 2); + FixedSizeBinaryVector vector2 = new FixedSizeBinaryVector("test1", allocator, 3)) { + vector1.allocateNew(); + vector2.allocateNew(); + vector1.set(0, new byte[] {1, 1}); + vector2.set(0, new byte[] {1, 1, 0}); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vector1); + comparator.attachVectors(vector1, vector2); + + // prefix is smaller + assertTrue(comparator.compare(0, 0) < 0); + } + + try (FixedSizeBinaryVector vector1 = new FixedSizeBinaryVector("test1", allocator, 3); + FixedSizeBinaryVector vector2 = new FixedSizeBinaryVector("test1", allocator, 3)) { + vector1.allocateNew(); + vector2.allocateNew(); + vector1.set(0, new byte[] {1, 1, 0}); + vector2.set(0, new byte[] {1, 1, 1}); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vector1); + comparator.attachVectors(vector1, vector2); + + // breaking tie by the last element + assertTrue(comparator.compare(0, 0) < 0); + } + + try (FixedSizeBinaryVector vector1 = new FixedSizeBinaryVector("test1", allocator, 3); + FixedSizeBinaryVector vector2 = new FixedSizeBinaryVector("test1", allocator, 3)) { + vector1.allocateNew(); + vector2.allocateNew(); + vector1.set(0, new byte[] {1, 1, 1}); + vector2.set(0, new byte[] {1, 1, 1}); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vector1); + comparator.attachVectors(vector1, vector2); + + // list vector elements equal + assertTrue(comparator.compare(0, 0) == 0); + } + } + + @Test + public void testCompareNull() { + try (NullVector vec = new NullVector("test", + FieldType.notNullable(new ArrowType.Int(32, false)))) { + vec.setValueCount(2); + + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); + comparator.attachVector(vec); + assertEquals(DefaultVectorComparators.NullComparator.class, comparator.getClass()); + assertEquals(0, comparator.compare(0, 1)); + } + } + @Test public void testCheckNullsOnCompareIsFalseForNonNullableVector() { try (IntVector vec = new IntVector("not nullable", @@ -937,4 +1055,18 @@ private static void verifyVariableWidthComparatorReturne VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); assertEquals(DefaultVectorComparators.VariableWidthComparator.class, comparator.getClass()); } + + @Test + public void testRepeatedDefaultComparators() { + final FieldType type = FieldType.nullable(Types.MinorType.INT.getType()); + try (final LargeListVector vector = new LargeListVector("list", allocator, type, null)) { + vector.addOrGetVector(FieldType.nullable(type.getType())); + verifyRepeatedComparatorReturned(vector); + } + } + + private static void verifyRepeatedComparatorReturned(V vec) { + VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + assertEquals(DefaultVectorComparators.RepeatedValueComparator.class, comparator.getClass()); + } } From f3ec224ab6ace14f630509c79dfbba2ec32d881a Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 14 Nov 2023 14:25:29 +0100 Subject: [PATCH 48/57] GH-38626: [Python] Fix segfault when PyArrow is imported at shutdown (#38637) ### Rationale for this change Some C++ destructors may be called after the Python interpreter has ceased to exist. If such a destructor tries to call back in the Python interpreter, for example by calling `Py_DECREF`, we get a crash. ### What changes are included in this PR? Protect `OwnedRef` and `OwneRefNoGIL` destructors against decref'ing a Python object after Python finalization. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * Closes: #38626 Authored-by: Antoine Pitrou Signed-off-by: Joris Van den Bossche --- python/pyarrow/src/arrow/python/common.h | 17 ++++++++++------- python/pyarrow/tests/test_misc.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/python/pyarrow/src/arrow/python/common.h b/python/pyarrow/src/arrow/python/common.h index e36c0834fd424..bc567ef78e83a 100644 --- a/python/pyarrow/src/arrow/python/common.h +++ b/python/pyarrow/src/arrow/python/common.h @@ -188,7 +188,12 @@ class ARROW_PYTHON_EXPORT OwnedRef { return *this; } - ~OwnedRef() { reset(); } + ~OwnedRef() { + // GH-38626: destructor may be called after the Python interpreter is finalized. + if (Py_IsInitialized()) { + reset(); + } + } void reset(PyObject* obj) { Py_XDECREF(obj_); @@ -225,13 +230,11 @@ class ARROW_PYTHON_EXPORT OwnedRefNoGIL : public OwnedRef { explicit OwnedRefNoGIL(PyObject* obj) : OwnedRef(obj) {} ~OwnedRefNoGIL() { - // This destructor may be called after the Python interpreter is finalized. - // At least avoid spurious attempts to take the GIL when not necessary. - if (obj() == NULLPTR) { - return; + // GH-38626: destructor may be called after the Python interpreter is finalized. + if (Py_IsInitialized() && obj() != NULLPTR) { + PyAcquireGIL lock; + reset(); } - PyAcquireGIL lock; - reset(); } }; diff --git a/python/pyarrow/tests/test_misc.py b/python/pyarrow/tests/test_misc.py index 9b9dfdd554806..a48ac0c3cd81a 100644 --- a/python/pyarrow/tests/test_misc.py +++ b/python/pyarrow/tests/test_misc.py @@ -117,6 +117,19 @@ def test_runtime_info(): subprocess.check_call([sys.executable, "-c", code], env=env) +def test_import_at_shutdown(): + # GH-38626: importing PyArrow at interpreter shutdown would crash + code = """if 1: + import atexit + + def import_arrow(): + import pyarrow + + atexit.register(import_arrow) + """ + subprocess.check_call([sys.executable, "-c", code]) + + @pytest.mark.skipif(sys.platform == "win32", reason="Path to timezone database is not configurable " "on non-Windows platforms") From bb7ffaf0bd0661baba872c3fe1500369f26241bd Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Tue, 14 Nov 2023 10:43:28 -0500 Subject: [PATCH 49/57] GH-38477: [Go] Fixing decimal 128 rounding issue (#38478) ### Rationale for this change Fixing an off-by-one rounding issue with decimal128 by ensuring proper precision handling. ### Are these changes tested? The test case which reproduced the rounding issue has been added as a unit test. * Closes: #38477 Authored-by: Matt Topol Signed-off-by: Matt Topol --- go/arrow/decimal128/decimal128.go | 4 ++-- go/arrow/decimal128/decimal128_test.go | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/go/arrow/decimal128/decimal128.go b/go/arrow/decimal128/decimal128.go index 7bde39d327417..3b88dce1fa809 100644 --- a/go/arrow/decimal128/decimal128.go +++ b/go/arrow/decimal128/decimal128.go @@ -261,7 +261,7 @@ func FromString(v string, prec, scale int32) (n Num, err error) { var precInBits = uint(math.Round(float64(prec+scale+1)/math.Log10(2))) + 1 var out *big.Float - out, _, err = big.ParseFloat(v, 10, 127, big.ToNearestEven) + out, _, err = big.ParseFloat(v, 10, 128, big.ToNearestEven) if err != nil { return } @@ -280,7 +280,7 @@ func FromString(v string, prec, scale int32) (n Num, err error) { // (e.g. C++) handles Decimal values. So if we're negative we'll subtract 0.5 and if // we're positive we'll add 0.5. p := (&big.Float{}).SetInt(scaleMultipliers[scale].BigInt()) - out.Mul(out, p).SetPrec(precInBits) + out.SetPrec(precInBits).Mul(out, p) if out.Signbit() { out.Sub(out, pt5) } else { diff --git a/go/arrow/decimal128/decimal128_test.go b/go/arrow/decimal128/decimal128_test.go index 879f2849028f8..4cfd7db20db08 100644 --- a/go/arrow/decimal128/decimal128_test.go +++ b/go/arrow/decimal128/decimal128_test.go @@ -24,6 +24,7 @@ import ( "github.com/apache/arrow/go/v15/arrow/decimal128" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestFromU64(t *testing.T) { @@ -698,3 +699,11 @@ func TestBitLen(t *testing.T) { _, err = decimal128.FromString(b.String(), decimal128.MaxPrecision, -1) assert.ErrorContains(t, err, "bitlen too large for decimal128") } + +func TestFromStringDecimal128b(t *testing.T) { + const decStr = "9323406071781562130.6457232358109488923" + + num, err := decimal128.FromString(decStr, 38, 19) + require.NoError(t, err) + assert.Equal(t, decStr, num.ToString(19)) +} From cd0d7f53b3ab7dfac7a3477751a87586d4da3782 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 14 Nov 2023 17:31:08 +0100 Subject: [PATCH 50/57] MINOR: [Python] Fix name of new keyword in the concat_tables future warning (#38710) We renamed the new keyword in a final iteration of the PR, but apparently forgot to update the warning message. Authored-by: Joris Van den Bossche Signed-off-by: Joris Van den Bossche --- python/pyarrow/table.pxi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index bbf60416de995..e55a0d1dd54cb 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5226,7 +5226,8 @@ def concat_tables(tables, MemoryPool memory_pool=None, str promote_options="none if "promote" in kwargs: warnings.warn( - "promote has been superseded by mode='default'.", FutureWarning, stacklevel=2) + "promote has been superseded by promote_options='default'.", + FutureWarning, stacklevel=2) if kwargs['promote'] is True: promote_options = "default" From 26149d9fab0360e6d4d9a295f934100470c4bc37 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Tue, 14 Nov 2023 11:44:19 -0500 Subject: [PATCH 51/57] GH-38718: [Go][Format][Integration] Add StringView/BinaryView to Go implementation (#35769) ### Rationale for this change See #35628 for the rationale and description of the StringView/BinaryView array types. This change is adding Go as a second implementation of it. ### What changes are included in this PR? Add Array Types for `StringView` and `BinaryView` along with `StringViewType` and `BinaryViewType` and necessary enums and builders. These arrays can be round tripped through JSON and IPC. ### Are these changes tested? Yes, unit tests have been added and integration tests run * Closes: [#38718](https://github.com/apache/arrow/issues/38718) * Closes: #38718 Lead-authored-by: Matt Topol Co-authored-by: Alex Shcherbakov Signed-off-by: Benjamin Kietzman --- .gitattributes | 3 + docs/source/status.rst | 4 + format/Schema.fbs | 2 +- go/arrow/array/array.go | 3 +- go/arrow/array/binary.go | 121 +++++++ go/arrow/array/binary_test.go | 24 ++ go/arrow/array/binarybuilder.go | 329 ++++++++++++++++++ go/arrow/array/bufferbuilder.go | 108 ++++++ go/arrow/array/builder.go | 4 + go/arrow/array/compare.go | 12 + go/arrow/array/concat.go | 30 +- go/arrow/array/concat_test.go | 3 + go/arrow/array/string.go | 196 ++++++++++- go/arrow/array/string_test.go | 173 +++++++++ go/arrow/compute/executor.go | 5 +- go/arrow/datatype.go | 7 + go/arrow/datatype_binary.go | 41 +++ go/arrow/datatype_binary_test.go | 30 ++ go/arrow/datatype_viewheader.go | 141 ++++++++ go/arrow/datatype_viewheader_inline.go | 31 ++ go/arrow/datatype_viewheader_inline_go1.19.go | 35 ++ go/arrow/datatype_viewheader_inline_tinygo.go | 35 ++ go/arrow/internal/arrdata/arrdata.go | 81 +++++ go/arrow/internal/arrjson/arrjson.go | 150 ++++++++ go/arrow/internal/arrjson/arrjson_test.go | 259 ++++++++++++++ go/arrow/internal/flatbuf/MetadataVersion.go | 2 +- .../internal/testing/gen/random_array_gen.go | 34 ++ go/arrow/ipc/endian_swap.go | 4 + go/arrow/ipc/file_reader.go | 38 +- go/arrow/ipc/message.go | 10 +- go/arrow/ipc/metadata.go | 40 ++- go/arrow/ipc/writer.go | 36 +- go/arrow/type_traits_view.go | 53 +++ 33 files changed, 2011 insertions(+), 33 deletions(-) create mode 100644 go/arrow/datatype_viewheader.go create mode 100644 go/arrow/datatype_viewheader_inline.go create mode 100644 go/arrow/datatype_viewheader_inline_go1.19.go create mode 100644 go/arrow/datatype_viewheader_inline_tinygo.go create mode 100644 go/arrow/type_traits_view.go diff --git a/.gitattributes b/.gitattributes index 69f4139c4e4f4..70007c26c8b9b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,6 +3,9 @@ cpp/src/generated/*.cpp linguist-generated=true cpp/src/generated/*.h linguist-generated=true go/**/*.s linguist-generated=true go/arrow/unionmode_string.go linguist-generated=true +go/arrow/internal/flatbuf/*.go linguist-generated=true +go/**/*.pb.go linguist-generated=true +go/parquet/internal/gen-go/parquet/*.go linguist-generated=true r/R/RcppExports.R linguist-generated=true r/R/arrowExports.R linguist-generated=true r/src/RcppExports.cpp linguist-generated=true diff --git a/docs/source/status.rst b/docs/source/status.rst index c8c0e6dfc1dfe..c059ab3cef971 100644 --- a/docs/source/status.rst +++ b/docs/source/status.rst @@ -68,6 +68,10 @@ Data Types +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Large Utf8 | ✓ | ✓ | ✓ | | | ✓ | ✓ | | +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ +| Binary View | ✓ | | ✓ | | | | | | ++-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ +| String View | ✓ | | ✓ | | | | | | ++-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ +-------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Data type | C++ | Java | Go | JavaScript | C# | Rust | Julia | Swift | diff --git a/format/Schema.fbs b/format/Schema.fbs index 6adbcb115cde3..dbf482e6cc786 100644 --- a/format/Schema.fbs +++ b/format/Schema.fbs @@ -40,7 +40,7 @@ enum MetadataVersion:short { /// >= 0.8.0 (December 2017). Non-backwards compatible with V3. V4, - /// >= 1.0.0 (July 2020. Backwards compatible with V4 (V5 readers can read V4 + /// >= 1.0.0 (July 2020). Backwards compatible with V4 (V5 readers can read V4 /// metadata and IPC messages). Implementations are recommended to provide a /// V4 compatibility mode with V5 format changes disabled. /// diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go index bbe301ee661f3..5aacc8f99a4ee 100644 --- a/go/arrow/array/array.go +++ b/go/arrow/array/array.go @@ -178,7 +178,8 @@ func init() { arrow.RUN_END_ENCODED: func(data arrow.ArrayData) arrow.Array { return NewRunEndEncodedData(data) }, arrow.LIST_VIEW: func(data arrow.ArrayData) arrow.Array { return NewListViewData(data) }, arrow.LARGE_LIST_VIEW: func(data arrow.ArrayData) arrow.Array { return NewLargeListViewData(data) }, - + arrow.BINARY_VIEW: func(data arrow.ArrayData) arrow.Array { return NewBinaryViewData(data) }, + arrow.STRING_VIEW: func(data arrow.ArrayData) arrow.Array { return NewStringViewData(data) }, // invalid data types to fill out array to size 2^6 - 1 63: invalidDataType, } diff --git a/go/arrow/array/binary.go b/go/arrow/array/binary.go index bf27139fddbaa..c226297da04c6 100644 --- a/go/arrow/array/binary.go +++ b/go/arrow/array/binary.go @@ -24,6 +24,7 @@ import ( "unsafe" "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/apache/arrow/go/v15/internal/json" ) @@ -318,6 +319,126 @@ func arrayEqualLargeBinary(left, right *LargeBinary) bool { return true } +type ViewLike interface { + arrow.Array + ValueHeader(int) *arrow.ViewHeader +} + +type BinaryView struct { + array + values []arrow.ViewHeader + dataBuffers []*memory.Buffer +} + +func NewBinaryViewData(data arrow.ArrayData) *BinaryView { + a := &BinaryView{} + a.refCount = 1 + a.setData(data.(*Data)) + return a +} + +func (a *BinaryView) setData(data *Data) { + if len(data.buffers) < 2 { + panic("len(data.buffers) < 2") + } + a.array.setData(data) + + if valueData := data.buffers[1]; valueData != nil { + a.values = arrow.ViewHeaderTraits.CastFromBytes(valueData.Bytes()) + } + + a.dataBuffers = data.buffers[2:] +} + +func (a *BinaryView) ValueHeader(i int) *arrow.ViewHeader { + if i < 0 || i >= a.array.data.length { + panic("arrow/array: index out of range") + } + return &a.values[a.array.data.offset+i] +} + +func (a *BinaryView) Value(i int) []byte { + s := a.ValueHeader(i) + if s.IsInline() { + return s.InlineBytes() + } + start := s.BufferOffset() + buf := a.dataBuffers[s.BufferIndex()] + return buf.Bytes()[start : start+int32(s.Len())] +} + +// ValueString returns the value at index i as a string instead of +// a byte slice, without copying the underlying data. +func (a *BinaryView) ValueString(i int) string { + b := a.Value(i) + return *(*string)(unsafe.Pointer(&b)) +} + +func (a *BinaryView) String() string { + var o strings.Builder + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + switch { + case a.IsNull(i): + o.WriteString(NullValueStr) + default: + fmt.Fprintf(&o, "%q", a.ValueString(i)) + } + } + o.WriteString("]") + return o.String() +} + +// ValueStr is paired with AppendValueFromString in that it returns +// the value at index i as a string: Semantically this means that for +// a null value it will return the string "(null)", otherwise it will +// return the value as a base64 encoded string suitable for CSV/JSON. +// +// This is always going to be less performant than just using ValueString +// and exists to fulfill the Array interface to provide a method which +// can produce a human readable string for a given index. +func (a *BinaryView) ValueStr(i int) string { + if a.IsNull(i) { + return NullValueStr + } + return base64.StdEncoding.EncodeToString(a.Value(i)) +} + +func (a *BinaryView) GetOneForMarshal(i int) interface{} { + if a.IsNull(i) { + return nil + } + return a.Value(i) +} + +func (a *BinaryView) MarshalJSON() ([]byte, error) { + vals := make([]interface{}, a.Len()) + for i := 0; i < a.Len(); i++ { + vals[i] = a.GetOneForMarshal(i) + } + // golang marshal standard says that []byte will be marshalled + // as a base64-encoded string + return json.Marshal(vals) +} + +func arrayEqualBinaryView(left, right *BinaryView) bool { + leftBufs, rightBufs := left.dataBuffers, right.dataBuffers + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if !left.ValueHeader(i).Equals(leftBufs, right.ValueHeader(i), rightBufs) { + return false + } + } + return true +} + var ( _ arrow.Array = (*Binary)(nil) + _ arrow.Array = (*LargeBinary)(nil) + _ arrow.Array = (*BinaryView)(nil) ) diff --git a/go/arrow/array/binary_test.go b/go/arrow/array/binary_test.go index 9c1770950a8b5..c9e165515225b 100644 --- a/go/arrow/array/binary_test.go +++ b/go/arrow/array/binary_test.go @@ -700,3 +700,27 @@ func TestBinaryStringRoundTrip(t *testing.T) { assert.True(t, Equal(arr, arr1)) } + +func TestBinaryViewStringRoundTrip(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + values := []string{"a", "bc", "", "", "supercalifragilistic", "", "expeallodocious"} + valid := []bool{true, true, false, false, true, true, true} + + b := NewBinaryViewBuilder(mem) + defer b.Release() + + b.AppendStringValues(values, valid) + arr := b.NewArray().(*BinaryView) + defer arr.Release() + + for i := 0; i < arr.Len(); i++ { + assert.NoError(t, b.AppendValueFromString(arr.ValueStr(i))) + } + + arr1 := b.NewArray().(*BinaryView) + defer arr1.Release() + + assert.True(t, Equal(arr, arr1)) +} diff --git a/go/arrow/array/binarybuilder.go b/go/arrow/array/binarybuilder.go index a51bc799e4965..21ad576508e9e 100644 --- a/go/arrow/array/binarybuilder.go +++ b/go/arrow/array/binarybuilder.go @@ -23,6 +23,7 @@ import ( "math" "reflect" "sync/atomic" + "unsafe" "github.com/apache/arrow/go/v15/arrow" "github.com/apache/arrow/go/v15/arrow/internal/debug" @@ -370,6 +371,334 @@ func (b *BinaryBuilder) UnmarshalJSON(data []byte) error { return b.Unmarshal(dec) } +const ( + dfltBlockSize = 32 << 10 // 32 KB + viewValueSizeLimit int32 = math.MaxInt32 +) + +type BinaryViewBuilder struct { + builder + dtype arrow.BinaryDataType + + data *memory.Buffer + rawData []arrow.ViewHeader + + blockBuilder multiBufferBuilder +} + +func NewBinaryViewBuilder(mem memory.Allocator) *BinaryViewBuilder { + return &BinaryViewBuilder{ + dtype: arrow.BinaryTypes.BinaryView, + builder: builder{ + refCount: 1, + mem: mem, + }, + blockBuilder: multiBufferBuilder{ + refCount: 1, + blockSize: dfltBlockSize, + mem: mem, + }, + } +} + +func (b *BinaryViewBuilder) SetBlockSize(sz uint) { + b.blockBuilder.blockSize = int(sz) +} + +func (b *BinaryViewBuilder) Type() arrow.DataType { return b.dtype } + +func (b *BinaryViewBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) != 0 { + return + } + + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } +} + +func (b *BinaryViewBuilder) init(capacity int) { + b.builder.init(capacity) + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.ViewHeaderTraits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.ViewHeaderTraits.CastFromBytes(b.data.Bytes()) +} + +func (b *BinaryViewBuilder) Resize(n int) { + nbuild := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + return + } + + b.builder.resize(nbuild, b.init) + b.data.Resize(arrow.ViewHeaderTraits.BytesRequired(n)) + b.rawData = arrow.ViewHeaderTraits.CastFromBytes(b.data.Bytes()) +} + +func (b *BinaryViewBuilder) ReserveData(length int) { + if int32(length) > viewValueSizeLimit { + panic(fmt.Errorf("%w: BinaryView or StringView elements cannot reference strings larger than 2GB", + arrow.ErrInvalid)) + } + b.blockBuilder.Reserve(int(length)) +} + +func (b *BinaryViewBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +func (b *BinaryViewBuilder) Append(v []byte) { + if int32(len(v)) > viewValueSizeLimit { + panic(fmt.Errorf("%w: BinaryView or StringView elements cannot reference strings larger than 2GB", arrow.ErrInvalid)) + } + + if !arrow.IsViewInline(len(v)) { + b.ReserveData(len(v)) + } + + b.Reserve(1) + b.UnsafeAppend(v) +} + +// AppendString is identical to Append, only accepting a string instead +// of a byte slice, avoiding the extra copy that would occur if you simply +// did []byte(v). +// +// This is different than AppendValueFromString which exists for the +// Builder interface, in that this expects raw binary data which is +// appended unmodified. AppendValueFromString expects base64 encoded binary +// data instead. +func (b *BinaryViewBuilder) AppendString(v string) { + // create a []byte without copying the bytes + // in go1.20 this would be unsafe.StringData + val := *(*[]byte)(unsafe.Pointer(&struct { + string + int + }{v, len(v)})) + b.Append(val) +} + +func (b *BinaryViewBuilder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *BinaryViewBuilder) AppendNulls(n int) { + b.Reserve(n) + for i := 0; i < n; i++ { + b.UnsafeAppendBoolToBitmap(false) + } +} + +func (b *BinaryViewBuilder) AppendEmptyValue() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(true) +} + +func (b *BinaryViewBuilder) AppendEmptyValues(n int) { + b.Reserve(n) + b.unsafeAppendBoolsToBitmap(nil, n) +} + +func (b *BinaryViewBuilder) UnsafeAppend(v []byte) { + hdr := &b.rawData[b.length] + hdr.SetBytes(v) + if !hdr.IsInline() { + b.blockBuilder.UnsafeAppend(hdr, v) + } + b.UnsafeAppendBoolToBitmap(true) +} + +func (b *BinaryViewBuilder) AppendValues(v [][]byte, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + outOfLineTotal := 0 + for i, vv := range v { + if len(valid) == 0 || valid[i] { + if !arrow.IsViewInline(len(vv)) { + outOfLineTotal += len(vv) + } + } + } + + b.ReserveData(outOfLineTotal) + for i, vv := range v { + if len(valid) == 0 || valid[i] { + hdr := &b.rawData[b.length+i] + hdr.SetBytes(vv) + if !hdr.IsInline() { + b.blockBuilder.UnsafeAppend(hdr, vv) + } + } + } + + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *BinaryViewBuilder) AppendStringValues(v []string, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + outOfLineTotal := 0 + for i, vv := range v { + if len(valid) == 0 || valid[i] { + if !arrow.IsViewInline(len(vv)) { + outOfLineTotal += len(vv) + } + } + } + + b.ReserveData(outOfLineTotal) + for i, vv := range v { + if len(valid) == 0 || valid[i] { + hdr := &b.rawData[b.length+i] + hdr.SetString(vv) + if !hdr.IsInline() { + b.blockBuilder.UnsafeAppendString(hdr, vv) + } + } + } + + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +// AppendValueFromString is paired with ValueStr for fulfilling the +// base Builder interface. This is intended to read in a human-readable +// string such as from CSV or JSON and append it to the array. +// +// For Binary values are expected to be base64 encoded (and will be +// decoded as such before being appended). +func (b *BinaryViewBuilder) AppendValueFromString(s string) error { + if s == NullValueStr { + b.AppendNull() + return nil + } + + if b.dtype.IsUtf8() { + b.Append([]byte(s)) + return nil + } + + decodedVal, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return fmt.Errorf("could not decode base64 string: %w", err) + } + b.Append(decodedVal) + return nil +} + +func (b *BinaryViewBuilder) UnmarshalOne(dec *json.Decoder) error { + t, err := dec.Token() + if err != nil { + return err + } + + switch v := t.(type) { + case string: + data, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return err + } + b.Append(data) + case []byte: + b.Append(v) + case nil: + b.AppendNull() + default: + return &json.UnmarshalTypeError{ + Value: fmt.Sprint(t), + Type: reflect.TypeOf([]byte{}), + Offset: dec.InputOffset(), + } + } + return nil +} + +func (b *BinaryViewBuilder) Unmarshal(dec *json.Decoder) error { + for dec.More() { + if err := b.UnmarshalOne(dec); err != nil { + return err + } + } + return nil +} + +func (b *BinaryViewBuilder) UnmarshalJSON(data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + t, err := dec.Token() + if err != nil { + return err + } + + if delim, ok := t.(json.Delim); !ok || delim != '[' { + return fmt.Errorf("binary view builder must unpack from json array, found %s", delim) + } + + return b.Unmarshal(dec) +} + +func (b *BinaryViewBuilder) newData() (data *Data) { + bytesRequired := arrow.ViewHeaderTraits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + + dataBuffers := b.blockBuilder.Finish() + data = NewData(b.dtype, b.length, append([]*memory.Buffer{ + b.nullBitmap, b.data}, dataBuffers...), nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + for _, buf := range dataBuffers { + buf.Release() + } + } + return +} + +func (b *BinaryViewBuilder) NewBinaryViewArray() (a *BinaryView) { + data := b.newData() + a = NewBinaryViewData(data) + data.Release() + return +} + +func (b *BinaryViewBuilder) NewArray() arrow.Array { + return b.NewBinaryViewArray() +} + var ( _ Builder = (*BinaryBuilder)(nil) + _ Builder = (*BinaryViewBuilder)(nil) ) diff --git a/go/arrow/array/bufferbuilder.go b/go/arrow/array/bufferbuilder.go index cb381e25b32a2..13741ba8926ac 100644 --- a/go/arrow/array/bufferbuilder.go +++ b/go/arrow/array/bufferbuilder.go @@ -18,7 +18,9 @@ package array import ( "sync/atomic" + "unsafe" + "github.com/apache/arrow/go/v15/arrow" "github.com/apache/arrow/go/v15/arrow/bitutil" "github.com/apache/arrow/go/v15/arrow/internal/debug" "github.com/apache/arrow/go/v15/arrow/memory" @@ -151,3 +153,109 @@ func (b *bufferBuilder) unsafeAppend(data []byte) { copy(b.bytes[b.length:], data) b.length += len(data) } + +type multiBufferBuilder struct { + refCount int64 + blockSize int + + mem memory.Allocator + blocks []*memory.Buffer + currentOutBuffer int +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (b *multiBufferBuilder) Retain() { + atomic.AddInt64(&b.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (b *multiBufferBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + b.Reset() + } +} + +func (b *multiBufferBuilder) Reserve(nbytes int) { + if len(b.blocks) == 0 { + out := memory.NewResizableBuffer(b.mem) + if nbytes < b.blockSize { + nbytes = b.blockSize + } + out.Reserve(nbytes) + b.currentOutBuffer = 0 + b.blocks = []*memory.Buffer{out} + return + } + + curBuf := b.blocks[b.currentOutBuffer] + remain := curBuf.Cap() - curBuf.Len() + if nbytes <= remain { + return + } + + // search for underfull block that has enough bytes + for i, block := range b.blocks { + remaining := block.Cap() - block.Len() + if nbytes <= remaining { + b.currentOutBuffer = i + return + } + } + + // current buffer doesn't have enough space, no underfull buffers + // make new buffer and set that as our current. + newBuf := memory.NewResizableBuffer(b.mem) + if nbytes < b.blockSize { + nbytes = b.blockSize + } + + newBuf.Reserve(nbytes) + b.currentOutBuffer = len(b.blocks) + b.blocks = append(b.blocks, newBuf) +} + +func (b *multiBufferBuilder) RemainingBytes() int { + if len(b.blocks) == 0 { + return 0 + } + + buf := b.blocks[b.currentOutBuffer] + return buf.Cap() - buf.Len() +} + +func (b *multiBufferBuilder) Reset() { + b.currentOutBuffer = 0 + for _, block := range b.Finish() { + block.Release() + } +} + +func (b *multiBufferBuilder) UnsafeAppend(hdr *arrow.ViewHeader, val []byte) { + buf := b.blocks[b.currentOutBuffer] + idx, offset := b.currentOutBuffer, buf.Len() + hdr.SetIndexOffset(int32(idx), int32(offset)) + + n := copy(buf.Buf()[offset:], val) + buf.ResizeNoShrink(offset + n) +} + +func (b *multiBufferBuilder) UnsafeAppendString(hdr *arrow.ViewHeader, val string) { + // create a byte slice with zero-copies + // in go1.20 this would be equivalent to unsafe.StringData + v := *(*[]byte)(unsafe.Pointer(&struct { + string + int + }{val, len(val)})) + b.UnsafeAppend(hdr, v) +} + +func (b *multiBufferBuilder) Finish() (out []*memory.Buffer) { + b.currentOutBuffer = 0 + out, b.blocks = b.blocks, nil + return +} diff --git a/go/arrow/array/builder.go b/go/arrow/array/builder.go index bb15298e03ccf..279804a1cdb9f 100644 --- a/go/arrow/array/builder.go +++ b/go/arrow/array/builder.go @@ -364,6 +364,10 @@ func NewBuilder(mem memory.Allocator, dtype arrow.DataType) Builder { case arrow.RUN_END_ENCODED: typ := dtype.(*arrow.RunEndEncodedType) return NewRunEndEncodedBuilder(mem, typ.RunEnds(), typ.Encoded()) + case arrow.BINARY_VIEW: + return NewBinaryViewBuilder(mem) + case arrow.STRING_VIEW: + return NewStringViewBuilder(mem) } panic(fmt.Errorf("arrow/array: unsupported builder for %T", dtype)) } diff --git a/go/arrow/array/compare.go b/go/arrow/array/compare.go index 778de41e32c67..372293a61d6cb 100644 --- a/go/arrow/array/compare.go +++ b/go/arrow/array/compare.go @@ -232,6 +232,12 @@ func Equal(left, right arrow.Array) bool { case *LargeString: r := right.(*LargeString) return arrayEqualLargeString(l, r) + case *BinaryView: + r := right.(*BinaryView) + return arrayEqualBinaryView(l, r) + case *StringView: + r := right.(*StringView) + return arrayEqualStringView(l, r) case *Int8: r := right.(*Int8) return arrayEqualInt8(l, r) @@ -482,6 +488,12 @@ func arrayApproxEqual(left, right arrow.Array, opt equalOption) bool { case *LargeString: r := right.(*LargeString) return arrayEqualLargeString(l, r) + case *BinaryView: + r := right.(*BinaryView) + return arrayEqualBinaryView(l, r) + case *StringView: + r := right.(*StringView) + return arrayEqualStringView(l, r) case *Int8: r := right.(*Int8) return arrayEqualInt8(l, r) diff --git a/go/arrow/array/concat.go b/go/arrow/array/concat.go index 53c5be06895b9..fa3554c1c0555 100644 --- a/go/arrow/array/concat.go +++ b/go/arrow/array/concat.go @@ -600,6 +600,35 @@ func concat(data []arrow.ArrayData, mem memory.Allocator) (arr arrow.ArrayData, } case arrow.FixedWidthDataType: out.buffers[1] = concatBuffers(gatherBuffersFixedWidthType(data, 1, dt), mem) + case arrow.BinaryViewDataType: + out.buffers = out.buffers[:2] + for _, d := range data { + for _, buf := range d.Buffers()[2:] { + buf.Retain() + out.buffers = append(out.buffers, buf) + } + } + + out.buffers[1] = concatBuffers(gatherFixedBuffers(data, 1, arrow.ViewHeaderSizeBytes), mem) + + var ( + s = arrow.ViewHeaderTraits.CastFromBytes(out.buffers[1].Bytes()) + i = data[0].Len() + precedingBufsCount int + ) + + for idx := 1; idx < len(data); idx++ { + precedingBufsCount += len(data[idx-1].Buffers()) - 2 + + for end := i + data[idx].Len(); i < end; i++ { + if s[i].IsInline() { + continue + } + + bufIndex := s[i].BufferIndex() + int32(precedingBufsCount) + s[i].SetIndexOffset(bufIndex, s[i].BufferOffset()) + } + } case arrow.BinaryDataType: offsetWidth := dt.Layout().Buffers[1].ByteWidth offsetBuffer, valueRanges, err := concatOffsets(gatherFixedBuffers(data, 1, offsetWidth), offsetWidth, mem) @@ -739,7 +768,6 @@ func concat(data []arrow.ArrayData, mem memory.Allocator) (arr arrow.ArrayData, out.childData[0].Release() return nil, err } - default: return nil, fmt.Errorf("concatenate not implemented for type %s", dt) } diff --git a/go/arrow/array/concat_test.go b/go/arrow/array/concat_test.go index 1cc484ad1a923..7b22d97a41e00 100644 --- a/go/arrow/array/concat_test.go +++ b/go/arrow/array/concat_test.go @@ -84,6 +84,7 @@ func TestConcatenate(t *testing.T) { {arrow.StructOf()}, {arrow.MapOf(arrow.PrimitiveTypes.Uint16, arrow.PrimitiveTypes.Int8)}, {&arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int32, ValueType: arrow.PrimitiveTypes.Float64}}, + {arrow.BinaryTypes.StringView}, } for _, tt := range tests { @@ -150,6 +151,8 @@ func (cts *ConcatTestSuite) generateArr(size int64, nullprob float64) arrow.Arra return cts.rng.String(size, 0, 15, nullprob) case arrow.LARGE_STRING: return cts.rng.LargeString(size, 0, 15, nullprob) + case arrow.STRING_VIEW: + return cts.rng.StringView(size, 0, 20, nullprob) case arrow.LIST: valuesSize := size * 4 values := cts.rng.Int8(valuesSize, 0, 127, nullprob).(*array.Int8) diff --git a/go/arrow/array/string.go b/go/arrow/array/string.go index 9ab7c938ef5d8..90a4628f0d0fb 100644 --- a/go/arrow/array/string.go +++ b/go/arrow/array/string.go @@ -28,6 +28,11 @@ import ( "github.com/apache/arrow/go/v15/internal/json" ) +type StringLike interface { + arrow.Array + Value(int) string +} + // String represents an immutable sequence of variable-length UTF-8 strings. type String struct { array @@ -310,6 +315,108 @@ func arrayEqualLargeString(left, right *LargeString) bool { return true } +type StringView struct { + array + values []arrow.ViewHeader + dataBuffers []*memory.Buffer +} + +func NewStringViewData(data arrow.ArrayData) *StringView { + a := &StringView{} + a.refCount = 1 + a.setData(data.(*Data)) + return a +} + +// Reset resets the String with a different set of Data. +func (a *StringView) Reset(data arrow.ArrayData) { + a.setData(data.(*Data)) +} + +func (a *StringView) setData(data *Data) { + if len(data.buffers) < 2 { + panic("len(data.buffers) < 2") + } + a.array.setData(data) + + if valueData := data.buffers[1]; valueData != nil { + a.values = arrow.ViewHeaderTraits.CastFromBytes(valueData.Bytes()) + } + + a.dataBuffers = data.buffers[2:] +} + +func (a *StringView) ValueHeader(i int) *arrow.ViewHeader { + if i < 0 || i >= a.array.data.length { + panic("arrow/array: index out of range") + } + return &a.values[a.array.data.offset+i] +} + +func (a *StringView) Value(i int) string { + s := a.ValueHeader(i) + if s.IsInline() { + return s.InlineString() + } + start := s.BufferOffset() + buf := a.dataBuffers[s.BufferIndex()] + value := buf.Bytes()[start : start+int32(s.Len())] + return *(*string)(unsafe.Pointer(&value)) +} + +func (a *StringView) String() string { + var o strings.Builder + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + switch { + case a.IsNull(i): + o.WriteString(NullValueStr) + default: + fmt.Fprintf(&o, "%q", a.Value(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *StringView) ValueStr(i int) string { + if a.IsNull(i) { + return NullValueStr + } + return a.Value(i) +} + +func (a *StringView) GetOneForMarshal(i int) interface{} { + if a.IsNull(i) { + return nil + } + return a.Value(i) +} + +func (a *StringView) MarshalJSON() ([]byte, error) { + vals := make([]interface{}, a.Len()) + for i := 0; i < a.Len(); i++ { + vals[i] = a.GetOneForMarshal(i) + } + return json.Marshal(vals) +} + +func arrayEqualStringView(left, right *StringView) bool { + leftBufs, rightBufs := left.dataBuffers, right.dataBuffers + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if !left.ValueHeader(i).Equals(leftBufs, right.ValueHeader(i), rightBufs) { + return false + } + } + return true +} + // A StringBuilder is used to build a String array using the Append methods. type StringBuilder struct { *BinaryBuilder @@ -344,10 +451,6 @@ func (b *StringBuilder) Value(i int) string { return string(b.BinaryBuilder.Value(i)) } -// func (b *StringBuilder) UnsafeAppend(v string) { -// b.BinaryBuilder.UnsafeAppend([]byte(v)) -// } - // NewArray creates a String array from the memory buffers used by the builder and resets the StringBuilder // so it can be used to build a new array. func (b *StringBuilder) NewArray() arrow.Array { @@ -441,10 +544,6 @@ func (b *LargeStringBuilder) Value(i int) string { return string(b.BinaryBuilder.Value(i)) } -// func (b *LargeStringBuilder) UnsafeAppend(v string) { -// b.BinaryBuilder.UnsafeAppend([]byte(v)) -// } - // NewArray creates a String array from the memory buffers used by the builder and resets the StringBuilder // so it can be used to build a new array. func (b *LargeStringBuilder) NewArray() arrow.Array { @@ -504,9 +603,87 @@ func (b *LargeStringBuilder) UnmarshalJSON(data []byte) error { return b.Unmarshal(dec) } +type StringViewBuilder struct { + *BinaryViewBuilder +} + +func NewStringViewBuilder(mem memory.Allocator) *StringViewBuilder { + bldr := &StringViewBuilder{ + BinaryViewBuilder: NewBinaryViewBuilder(mem), + } + bldr.dtype = arrow.BinaryTypes.StringView + return bldr +} + +func (b *StringViewBuilder) Append(v string) { + b.BinaryViewBuilder.AppendString(v) +} + +func (b *StringViewBuilder) AppendValues(v []string, valid []bool) { + b.BinaryViewBuilder.AppendStringValues(v, valid) +} + +func (b *StringViewBuilder) UnmarshalOne(dec *json.Decoder) error { + t, err := dec.Token() + if err != nil { + return err + } + + switch v := t.(type) { + case string: + b.Append(v) + case []byte: + b.BinaryViewBuilder.Append(v) + case nil: + b.AppendNull() + default: + return &json.UnmarshalTypeError{ + Value: fmt.Sprint(t), + Type: reflect.TypeOf([]byte{}), + Offset: dec.InputOffset(), + } + } + return nil +} + +func (b *StringViewBuilder) Unmarshal(dec *json.Decoder) error { + for dec.More() { + if err := b.UnmarshalOne(dec); err != nil { + return err + } + } + return nil +} + +func (b *StringViewBuilder) UnmarshalJSON(data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + t, err := dec.Token() + if err != nil { + return err + } + + if delim, ok := t.(json.Delim); !ok || delim != '[' { + return fmt.Errorf("binary view builder must unpack from json array, found %s", delim) + } + + return b.Unmarshal(dec) +} + +func (b *StringViewBuilder) NewArray() arrow.Array { + return b.NewStringViewArray() +} + +func (b *StringViewBuilder) NewStringViewArray() (a *StringView) { + data := b.newData() + a = NewStringViewData(data) + data.Release() + return +} + type StringLikeBuilder interface { Builder Append(string) + AppendValues([]string, []bool) UnsafeAppend([]byte) ReserveData(int) } @@ -514,8 +691,11 @@ type StringLikeBuilder interface { var ( _ arrow.Array = (*String)(nil) _ arrow.Array = (*LargeString)(nil) + _ arrow.Array = (*StringView)(nil) _ Builder = (*StringBuilder)(nil) _ Builder = (*LargeStringBuilder)(nil) + _ Builder = (*StringViewBuilder)(nil) _ StringLikeBuilder = (*StringBuilder)(nil) _ StringLikeBuilder = (*LargeStringBuilder)(nil) + _ StringLikeBuilder = (*StringViewBuilder)(nil) ) diff --git a/go/arrow/array/string_test.go b/go/arrow/array/string_test.go index d743a3ec7f37f..803fae51347c1 100644 --- a/go/arrow/array/string_test.go +++ b/go/arrow/array/string_test.go @@ -619,3 +619,176 @@ func TestStringValueLen(t *testing.T) { assert.Equal(t, len(v), slice.ValueLen(i)) } } +func TestStringViewArray(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + var ( + // only the last string is long enough to not get inlined + want = []string{"hello", "世界", "", "say goodbye daffy"} + valids = []bool{true, true, false, true} + ) + + sb := array.NewStringViewBuilder(mem) + defer sb.Release() + + sb.Retain() + sb.Release() + + assert.NoError(t, sb.AppendValueFromString(want[0])) + sb.AppendValues(want[1:2], nil) + + sb.AppendNull() + sb.Append(want[3]) + + if got, want := sb.Len(), len(want); got != want { + t.Fatalf("invalid len: got=%d, want=%d", got, want) + } + + if got, want := sb.NullN(), 1; got != want { + t.Fatalf("invalid nulls: got=%d, want=%d", got, want) + } + + arr := sb.NewStringViewArray() + defer arr.Release() + + arr.Retain() + arr.Release() + + assert.Equal(t, "hello", arr.ValueStr(0)) + + if got, want := arr.Len(), len(want); got != want { + t.Fatalf("invalid len: got=%d, want=%d", got, want) + } + + if got, want := arr.NullN(), 1; got != want { + t.Fatalf("invalid nulls: got=%d, want=%d", got, want) + } + + for i := range want { + if arr.IsNull(i) != !valids[i] { + t.Fatalf("arr[%d]-validity: got=%v want=%v", i, !arr.IsNull(i), valids[i]) + } + switch { + case arr.IsNull(i): + default: + got := arr.Value(i) + if got != want[i] { + t.Fatalf("arr[%d]: got=%q, want=%q", i, got, want[i]) + } + } + } + + sub := array.MakeFromData(arr.Data()) + defer sub.Release() + + if sub.DataType().ID() != arrow.STRING_VIEW { + t.Fatalf("invalid type: got=%q, want=string view", sub.DataType().Name()) + } + + if _, ok := sub.(*array.StringView); !ok { + t.Fatalf("could not type-assert to array.String") + } + + if got, want := arr.String(), `["hello" "世界" (null) "say goodbye daffy"]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + // only the last string gets stuck into a buffer the rest are inlined + // in the headers. + if !bytes.Equal([]byte(`say goodbye daffy`), arr.Data().Buffers()[2].Bytes()) { + t.Fatalf("got=%q, want=%q", string(arr.Data().Buffers()[2].Bytes()), `say goodbye daffy`) + } + + // check the prefix for the non-inlined value + if [4]byte{'s', 'a', 'y', ' '} != arr.ValueHeader(3).Prefix() { + t.Fatalf("got=%q, want=%q", arr.ValueHeader(3).Prefix(), `say `) + } + + slice := array.NewSliceData(arr.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.StringView) + if !ok { + t.Fatalf("could not type-assert to array.StringView") + } + + if got, want := v.String(), `[(null) "say goodbye daffy"]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + if !bytes.Equal([]byte(`say goodbye daffy`), v.Data().Buffers()[2].Bytes()) { + t.Fatalf("got=%q, want=%q", string(v.Data().Buffers()[2].Bytes()), `say goodbye daffy`) + } + + // check the prefix for the non-inlined value + if [4]byte{'s', 'a', 'y', ' '} != v.ValueHeader(1).Prefix() { + t.Fatalf("got=%q, want=%q", v.ValueHeader(1).Prefix(), `say `) + } +} + +func TestStringViewBuilder_Empty(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + want := []string{"hello", "世界", "", "say goodbye daffy"} + + ab := array.NewStringViewBuilder(mem) + defer ab.Release() + + stringValues := func(a *array.StringView) []string { + vs := make([]string, a.Len()) + for i := range vs { + vs[i] = a.Value(i) + } + return vs + } + + ab.AppendValues([]string{}, nil) + a := ab.NewStringViewArray() + assert.Zero(t, a.Len()) + a.Release() + + ab.AppendValues(nil, nil) + a = ab.NewStringViewArray() + assert.Zero(t, a.Len()) + a.Release() + + ab.AppendValues([]string{}, nil) + ab.AppendValues(want, nil) + a = ab.NewStringViewArray() + assert.Equal(t, want, stringValues(a)) + a.Release() + + ab.AppendValues(want, nil) + ab.AppendValues([]string{}, nil) + a = ab.NewStringViewArray() + assert.Equal(t, want, stringValues(a)) + a.Release() +} + +// TestStringReset tests the Reset() method on the String type by creating two different Strings and then +// reseting the contents of string2 with the values from string1. +func TestStringViewReset(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + sb1 := array.NewStringViewBuilder(mem) + sb2 := array.NewStringViewBuilder(mem) + defer sb1.Release() + defer sb2.Release() + + sb1.Append("string1") + sb1.AppendNull() + + var ( + string1 = sb1.NewStringViewArray() + string2 = sb2.NewStringViewArray() + + string1Data = string1.Data() + ) + string2.Reset(string1Data) + + assert.Equal(t, "string1", string2.Value(0)) +} diff --git a/go/arrow/compute/executor.go b/go/arrow/compute/executor.go index 1cba0b1e19f69..db89b206daf5f 100644 --- a/go/arrow/compute/executor.go +++ b/go/arrow/compute/executor.go @@ -171,6 +171,8 @@ func addComputeDataPrealloc(dt arrow.DataType, widths []bufferPrealloc) []buffer return append(widths, bufferPrealloc{bitWidth: 32, addLen: 1}) case arrow.LARGE_BINARY, arrow.LARGE_STRING, arrow.LARGE_LIST: return append(widths, bufferPrealloc{bitWidth: 64, addLen: 1}) + case arrow.STRING_VIEW, arrow.BINARY_VIEW: + return append(widths, bufferPrealloc{bitWidth: arrow.ViewHeaderSizeBytes * 8}) } return widths } @@ -1007,9 +1009,10 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC case <-ctx.Done(): return nil case output = <-out: - if output == nil { + if output == nil || ctx.Err() != nil { return nil } + // if the inputs contained at least one chunked array // then we want to return chunked output if hasChunked { diff --git a/go/arrow/datatype.go b/go/arrow/datatype.go index 24113b55899dc..1e5d8fb98aa59 100644 --- a/go/arrow/datatype.go +++ b/go/arrow/datatype.go @@ -210,6 +210,11 @@ type BinaryDataType interface { binary() } +type BinaryViewDataType interface { + BinaryDataType + view() +} + type OffsetsDataType interface { DataType OffsetTypeTraits() OffsetTraits @@ -272,6 +277,8 @@ func (b BufferSpec) Equals(other BufferSpec) bool { type DataTypeLayout struct { Buffers []BufferSpec HasDict bool + // VariadicSpec is what the buffers beyond len(Buffers) are expected to conform to. + VariadicSpec *BufferSpec } func SpecFixedWidth(w int) BufferSpec { return BufferSpec{KindFixedWidth, w} } diff --git a/go/arrow/datatype_binary.go b/go/arrow/datatype_binary.go index a3a8568645052..f3e601f08ec79 100644 --- a/go/arrow/datatype_binary.go +++ b/go/arrow/datatype_binary.go @@ -83,16 +83,57 @@ func (t *LargeStringType) Layout() DataTypeLayout { func (t *LargeStringType) OffsetTypeTraits() OffsetTraits { return Int64Traits } func (LargeStringType) IsUtf8() bool { return true } +type BinaryViewType struct{} + +func (*BinaryViewType) ID() Type { return BINARY_VIEW } +func (*BinaryViewType) Name() string { return "binary_view" } +func (*BinaryViewType) String() string { return "binary_view" } +func (*BinaryViewType) IsUtf8() bool { return false } +func (*BinaryViewType) binary() {} +func (*BinaryViewType) view() {} +func (t *BinaryViewType) Fingerprint() string { return typeFingerprint(t) } +func (*BinaryViewType) Layout() DataTypeLayout { + variadic := SpecVariableWidth() + return DataTypeLayout{ + Buffers: []BufferSpec{SpecBitmap(), SpecFixedWidth(ViewHeaderSizeBytes)}, + VariadicSpec: &variadic, + } +} + +type StringViewType struct{} + +func (*StringViewType) ID() Type { return STRING_VIEW } +func (*StringViewType) Name() string { return "string_view" } +func (*StringViewType) String() string { return "string_view" } +func (*StringViewType) IsUtf8() bool { return true } +func (*StringViewType) binary() {} +func (*StringViewType) view() {} +func (t *StringViewType) Fingerprint() string { return typeFingerprint(t) } +func (*StringViewType) Layout() DataTypeLayout { + variadic := SpecVariableWidth() + return DataTypeLayout{ + Buffers: []BufferSpec{SpecBitmap(), SpecFixedWidth(ViewHeaderSizeBytes)}, + VariadicSpec: &variadic, + } +} + var ( BinaryTypes = struct { Binary BinaryDataType String BinaryDataType LargeBinary BinaryDataType LargeString BinaryDataType + BinaryView BinaryDataType + StringView BinaryDataType }{ Binary: &BinaryType{}, String: &StringType{}, LargeBinary: &LargeBinaryType{}, LargeString: &LargeStringType{}, + BinaryView: &BinaryViewType{}, + StringView: &StringViewType{}, } + + _ BinaryViewDataType = (*StringViewType)(nil) + _ BinaryViewDataType = (*BinaryViewType)(nil) ) diff --git a/go/arrow/datatype_binary_test.go b/go/arrow/datatype_binary_test.go index 25ba6e8db4ba4..083d69ee3e5d4 100644 --- a/go/arrow/datatype_binary_test.go +++ b/go/arrow/datatype_binary_test.go @@ -81,3 +81,33 @@ func TestLargeStringType(t *testing.T) { t.Fatalf("invalid string type stringer. got=%v, want=%v", got, want) } } + +func TestBinaryViewType(t *testing.T) { + var nt *arrow.BinaryViewType + if got, want := nt.ID(), arrow.BINARY_VIEW; got != want { + t.Fatalf("invalid string type id. got=%v, want=%v", got, want) + } + + if got, want := nt.Name(), "binary_view"; got != want { + t.Fatalf("invalid string type name. got=%v, want=%v", got, want) + } + + if got, want := nt.String(), "binary_view"; got != want { + t.Fatalf("invalid string type stringer. got=%v, want=%v", got, want) + } +} + +func TestStringViewType(t *testing.T) { + var nt *arrow.StringViewType + if got, want := nt.ID(), arrow.STRING_VIEW; got != want { + t.Fatalf("invalid string type id. got=%v, want=%v", got, want) + } + + if got, want := nt.Name(), "string_view"; got != want { + t.Fatalf("invalid string type name. got=%v, want=%v", got, want) + } + + if got, want := nt.String(), "string_view"; got != want { + t.Fatalf("invalid string type stringer. got=%v, want=%v", got, want) + } +} diff --git a/go/arrow/datatype_viewheader.go b/go/arrow/datatype_viewheader.go new file mode 100644 index 0000000000000..54b9256b34604 --- /dev/null +++ b/go/arrow/datatype_viewheader.go @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "bytes" + "unsafe" + + "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v15/arrow/memory" +) + +const ( + ViewPrefixLen = 4 + viewInlineSize = 12 +) + +func IsViewInline(length int) bool { + return length < viewInlineSize +} + +// ViewHeader is a variable length string (utf8) or byte slice with +// a 4 byte prefix and inline optimization for small values (12 bytes +// or fewer). This is similar to Go's standard string but limited by +// a length of Uint32Max and up to the first four bytes of the string +// are copied into the struct. This prefix allows failing comparisons +// early and can reduce CPU cache working set when dealing with short +// strings. +// +// There are two situations: +// +// Entirely inlined string data +// |----|------------| +// ^ ^ +// | | +// size inline string data, zero padded +// +// Reference into buffer +// |----|----|----|----| +// ^ ^ ^ ^ +// | | | | +// size prefix buffer index and offset to out-of-line portion +// +// Adapted from TU Munich's UmbraDB [1], Velox, DuckDB. +// +// [1]: https://db.in.tum.de/~freitag/papers/p29-neumann-cidr20.pdf +type ViewHeader struct { + size int32 + // the first 4 bytes of this are the prefix for the string + // if size <= StringHeaderInlineSize, then the entire string + // is in the data array and is zero padded. + // if size > StringHeaderInlineSize, the next 8 bytes are 2 uint32 + // values which are the buffer index and offset in that buffer + // containing the full string. + data [viewInlineSize]byte +} + +func (sh *ViewHeader) IsInline() bool { + return sh.size <= int32(viewInlineSize) +} + +func (sh *ViewHeader) Len() int { return int(sh.size) } +func (sh *ViewHeader) Prefix() [ViewPrefixLen]byte { + return *(*[4]byte)(unsafe.Pointer(&sh.data)) +} + +func (sh *ViewHeader) BufferIndex() int32 { + return int32(endian.Native.Uint32(sh.data[ViewPrefixLen:])) +} + +func (sh *ViewHeader) BufferOffset() int32 { + return int32(endian.Native.Uint32(sh.data[ViewPrefixLen+4:])) +} + +func (sh *ViewHeader) InlineBytes() (data []byte) { + debug.Assert(sh.IsInline(), "calling InlineBytes on non-inline ViewHeader") + return sh.data[:sh.size] +} + +func (sh *ViewHeader) SetBytes(data []byte) int { + sh.size = int32(len(data)) + if sh.IsInline() { + return copy(sh.data[:], data) + } + return copy(sh.data[:4], data) +} + +func (sh *ViewHeader) SetString(data string) int { + sh.size = int32(len(data)) + if sh.IsInline() { + return copy(sh.data[:], data) + } + return copy(sh.data[:4], data) +} + +func (sh *ViewHeader) SetIndexOffset(bufferIndex, offset int32) { + endian.Native.PutUint32(sh.data[ViewPrefixLen:], uint32(bufferIndex)) + endian.Native.PutUint32(sh.data[ViewPrefixLen+4:], uint32(offset)) +} + +func (sh *ViewHeader) Equals(buffers []*memory.Buffer, other *ViewHeader, otherBuffers []*memory.Buffer) bool { + if sh.sizeAndPrefixAsInt64() != other.sizeAndPrefixAsInt64() { + return false + } + + if sh.IsInline() { + return sh.inlinedAsInt64() == other.inlinedAsInt64() + } + + return bytes.Equal(sh.getBufferBytes(buffers), other.getBufferBytes(otherBuffers)) +} + +func (sh *ViewHeader) getBufferBytes(buffers []*memory.Buffer) []byte { + offset := sh.BufferOffset() + return buffers[sh.BufferIndex()].Bytes()[offset : offset+sh.size] +} + +func (sh *ViewHeader) inlinedAsInt64() int64 { + s := unsafe.Slice((*int64)(unsafe.Pointer(sh)), 2) + return s[1] +} + +func (sh *ViewHeader) sizeAndPrefixAsInt64() int64 { + s := unsafe.Slice((*int64)(unsafe.Pointer(sh)), 2) + return s[0] +} diff --git a/go/arrow/datatype_viewheader_inline.go b/go/arrow/datatype_viewheader_inline.go new file mode 100644 index 0000000000000..89ac1d06adcdf --- /dev/null +++ b/go/arrow/datatype_viewheader_inline.go @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.20 + +package arrow + +import ( + "unsafe" + + "github.com/apache/arrow/go/v15/arrow/internal/debug" +) + +func (sh *ViewHeader) InlineString() (data string) { + debug.Assert(sh.IsInline(), "calling InlineString on non-inline ViewHeader") + + return unsafe.String((*byte)(unsafe.Pointer(&sh.data)), sh.size) +} diff --git a/go/arrow/datatype_viewheader_inline_go1.19.go b/go/arrow/datatype_viewheader_inline_go1.19.go new file mode 100644 index 0000000000000..aec66009d9492 --- /dev/null +++ b/go/arrow/datatype_viewheader_inline_go1.19.go @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.20 && !tinygo + +package arrow + +import ( + "reflect" + "unsafe" + + "github.com/apache/arrow/go/v15/arrow/internal/debug" +) + +func (sh *ViewHeader) InlineString() (data string) { + debug.Assert(sh.IsInline(), "calling InlineString on non-inline ViewHeader") + + h := (*reflect.StringHeader)(unsafe.Pointer(&data)) + h.Data = uintptr(unsafe.Pointer(&sh.data)) + h.Len = int(sh.size) + return +} diff --git a/go/arrow/datatype_viewheader_inline_tinygo.go b/go/arrow/datatype_viewheader_inline_tinygo.go new file mode 100644 index 0000000000000..bff63a273a722 --- /dev/null +++ b/go/arrow/datatype_viewheader_inline_tinygo.go @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.20 && tinygo + +package arrow + +import ( + "reflect" + "unsafe" + + "github.com/apache/arrow/go/v15/arrow/internal/debug" +) + +func (sh *ViewHeader) InlineString() (data string) { + debug.Assert(sh.IsInline(), "calling InlineString on non-inline ViewHeader") + + h := (*reflect.StringHeader)(unsafe.Pointer(&data)) + h.Data = uintptr(unsafe.Pointer(&sh.data)) + h.Len = uintptr(sh.size) + return +} diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go index 6631e4245c19d..985388094eb51 100644 --- a/go/arrow/internal/arrdata/arrdata.go +++ b/go/arrow/internal/arrdata/arrdata.go @@ -54,6 +54,7 @@ func init() { Records["extension"] = makeExtensionRecords() Records["union"] = makeUnionRecords() Records["run_end_encoded"] = makeRunEndEncodedRecords() + Records["view_types"] = makeStringViewRecords() for k := range Records { RecordNames = append(RecordNames, k) @@ -1155,6 +1156,65 @@ func makeRunEndEncodedRecords() []arrow.Record { return recs } +func makeStringViewRecords() []arrow.Record { + mem := memory.NewGoAllocator() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "binary_view", Type: arrow.BinaryTypes.BinaryView, Nullable: true}, + {Name: "string_view", Type: arrow.BinaryTypes.StringView, Nullable: true}, + }, nil) + + mask := []bool{true, false, false, true, true} + chunks := [][]arrow.Array{ + { + viewTypeArrayOf(mem, [][]byte{[]byte("1é"), []byte("2"), []byte("3"), []byte("4"), []byte("5")}, mask), + viewTypeArrayOf(mem, []string{"1é", "2", "3", "4", "5"}, mask), + }, + { + viewTypeArrayOf(mem, [][]byte{[]byte("1é"), []byte("22222222222222"), []byte("33333333333333"), []byte("4444"), []byte("5555")}, mask), + viewTypeArrayOf(mem, []string{"1é", "22222222222222", "33333333333333", "4444", "5555"}, nil), + }, + { + viewTypeArrayOf(mem, [][]byte{[]byte("1é1é"), []byte("22222222222222"), []byte("33333333333333"), []byte("44"), []byte("55")}, nil), + viewTypeArrayOf(mem, []string{"1é1é", "22222222222222", "33333333333333", "44", "55"}, mask), + }, + } + + defer func() { + for _, chunk := range chunks { + for _, col := range chunk { + col.Release() + } + } + }() + + recs := make([]arrow.Record, len(chunks)) + for i, chunk := range chunks { + recs[i] = array.NewRecord(schema, chunk, -1) + } + + return recs +} + +func viewTypeArrayOf(mem memory.Allocator, a interface{}, valids []bool) arrow.Array { + if mem == nil { + mem = memory.NewGoAllocator() + } + + switch a := a.(type) { + case []string: + bldr := array.NewStringViewBuilder(mem) + defer bldr.Release() + bldr.AppendValues(a, valids) + return bldr.NewArray() + case [][]byte: + bldr := array.NewBinaryViewBuilder(mem) + defer bldr.Release() + bldr.AppendValues(a, valids) + return bldr.NewArray() + } + return nil +} + func extArray(mem memory.Allocator, dt arrow.ExtensionType, a interface{}, valids []bool) arrow.Array { var storage arrow.Array switch st := dt.StorageType().(type) { @@ -1750,5 +1810,26 @@ func buildArray(bldr array.Builder, data arrow.Array) { bldr.AppendNull() } } + + case *array.BinaryViewBuilder: + data := data.(*array.BinaryView) + for i := 0; i < data.Len(); i++ { + switch { + case data.IsValid(i): + bldr.Append(data.Value(i)) + default: + bldr.AppendNull() + } + } + case *array.StringViewBuilder: + data := data.(*array.StringView) + for i := 0; i < data.Len(); i++ { + switch { + case data.IsValid(i): + bldr.Append(data.Value(i)) + default: + bldr.AppendNull() + } + } } } diff --git a/go/arrow/internal/arrjson/arrjson.go b/go/arrow/internal/arrjson/arrjson.go index 87bdc1f44d875..f74b615362642 100644 --- a/go/arrow/internal/arrjson/arrjson.go +++ b/go/arrow/internal/arrjson/arrjson.go @@ -158,6 +158,10 @@ func typeToJSON(arrowType arrow.DataType) (json.RawMessage, error) { typ = nameJSON{"utf8"} case *arrow.LargeStringType: typ = nameJSON{"largeutf8"} + case *arrow.BinaryViewType: + typ = nameJSON{"binaryview"} + case *arrow.StringViewType: + typ = nameJSON{"utf8view"} case *arrow.Date32Type: typ = unitZoneJSON{Name: "date", Unit: "DAY"} case *arrow.Date64Type: @@ -342,6 +346,10 @@ func typeFromJSON(typ json.RawMessage, children []FieldWrapper) (arrowType arrow arrowType = arrow.BinaryTypes.String case "largeutf8": arrowType = arrow.BinaryTypes.LargeString + case "binaryview": + arrowType = arrow.BinaryTypes.BinaryView + case "utf8view": + arrowType = arrow.BinaryTypes.StringView case "date": t := unitZoneJSON{} if err = json.Unmarshal(typ, &t); err != nil { @@ -818,6 +826,7 @@ type Array struct { Offset interface{} `json:"OFFSET,omitempty"` Size interface{} `json:"SIZE,omitempty"` Children []Array `json:"children,omitempty"` + Variadic []string `json:"VARIADIC_BUFFERS,omitempty"` } func (a *Array) MarshalJSON() ([]byte, error) { @@ -1078,6 +1087,18 @@ func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) arrow.Arr bldr.AppendValues(data, valids) return returnNewArrayData(bldr) + case arrow.BinaryViewDataType: + valids := validsToBitmap(validsFromJSON(arr.Valids), mem) + nulls := arr.Count - bitutil.CountSetBits(valids.Bytes(), 0, arr.Count) + headers := stringHeadersFromJSON(mem, !dt.IsUtf8(), arr.Data) + extraBufs := variadicBuffersFromJSON(arr.Variadic) + defer valids.Release() + defer headers.Release() + + return array.NewData(dt, arr.Count, + append([]*memory.Buffer{valids, headers}, extraBufs...), + nil, nulls, 0) + case *arrow.ListType: valids := validsFromJSON(arr.Valids) elems := arrayFromJSON(mem, dt.Elem(), arr.Children[0]) @@ -1486,6 +1507,24 @@ func arrayToJSON(field arrow.Field, arr arrow.Array) Array { Offset: strOffsets, } + case *array.StringView: + variadic := variadicBuffersToJSON(arr.Data().Buffers()[2:]) + return Array{ + Name: field.Name, + Count: arr.Len(), + Valids: validsToJSON(arr), + Data: stringHeadersToJSON(arr, false), + Variadic: variadic, + } + case *array.BinaryView: + variadic := variadicBuffersToJSON(arr.Data().Buffers()[2:]) + return Array{ + Name: field.Name, + Count: arr.Len(), + Valids: validsToJSON(arr), + Data: stringHeadersToJSON(arr, true), + Variadic: variadic, + } case *array.List: o := Array{ Name: field.Name, @@ -2309,3 +2348,114 @@ func durationToJSON(arr *array.Duration) []interface{} { } return o } + +func variadicBuffersFromJSON(bufs []string) []*memory.Buffer { + out := make([]*memory.Buffer, len(bufs)) + for i, data := range bufs { + rawData, err := hex.DecodeString(data) + if err != nil { + panic(err) + } + + out[i] = memory.NewBufferBytes(rawData) + } + return out +} + +func variadicBuffersToJSON(bufs []*memory.Buffer) []string { + out := make([]string, len(bufs)) + for i, data := range bufs { + out[i] = strings.ToUpper(hex.EncodeToString(data.Bytes())) + } + return out +} + +func stringHeadersFromJSON(mem memory.Allocator, isBinary bool, data []interface{}) *memory.Buffer { + buf := memory.NewResizableBuffer(mem) + buf.Resize(arrow.ViewHeaderTraits.BytesRequired(len(data))) + + values := arrow.ViewHeaderTraits.CastFromBytes(buf.Bytes()) + + for i, d := range data { + switch v := d.(type) { + case nil: + continue + case map[string]interface{}: + if inlined, ok := v["INLINED"]; ok { + if isBinary { + val, err := hex.DecodeString(inlined.(string)) + if err != nil { + panic(fmt.Errorf("could not decode %v: %v", inlined, err)) + } + values[i].SetBytes(val) + } else { + values[i].SetString(inlined.(string)) + } + continue + } + + idx, offset := v["BUFFER_INDEX"].(json.Number), v["OFFSET"].(json.Number) + bufIdx, err := idx.Int64() + if err != nil { + panic(err) + } + + bufOffset, err := offset.Int64() + if err != nil { + panic(err) + } + + values[i].SetIndexOffset(int32(bufIdx), int32(bufOffset)) + prefix, err := hex.DecodeString(v["PREFIX"].(string)) + if err != nil { + panic(err) + } + sz, err := v["SIZE"].(json.Number).Int64() + if err != nil { + panic(err) + } + + rawData := make([]byte, sz) + copy(rawData, prefix) + values[i].SetBytes(rawData) + } + } + return buf +} + +func stringHeadersToJSON(arr array.ViewLike, isBinary bool) []interface{} { + type StringHeader struct { + Size int `json:"SIZE"` + Prefix *string `json:"PREFIX,omitempty"` + BufferIdx *int `json:"BUFFER_INDEX,omitempty"` + BufferOff *int `json:"OFFSET,omitempty"` + Inlined *string `json:"INLINED,omitempty"` + } + + o := make([]interface{}, arr.Len()) + for i := range o { + hdr := arr.ValueHeader(i) + if hdr.IsInline() { + data := hdr.InlineString() + if isBinary { + data = strings.ToUpper(hex.EncodeToString(hdr.InlineBytes())) + } + o[i] = StringHeader{ + Size: hdr.Len(), + Inlined: &data, + } + continue + } + + idx, off := int(hdr.BufferIndex()), int(hdr.BufferOffset()) + prefix := hdr.Prefix() + encodedPrefix := strings.ToUpper(hex.EncodeToString(prefix[:])) + o[i] = StringHeader{ + Size: hdr.Len(), + Prefix: &encodedPrefix, + BufferIdx: &idx, + BufferOff: &off, + } + } + return o +} diff --git a/go/arrow/internal/arrjson/arrjson_test.go b/go/arrow/internal/arrjson/arrjson_test.go index 7beadee370edb..31f3cb238ec16 100644 --- a/go/arrow/internal/arrjson/arrjson_test.go +++ b/go/arrow/internal/arrjson/arrjson_test.go @@ -48,6 +48,7 @@ func TestReadWrite(t *testing.T) { wantJSONs["dictionary"] = makeDictionaryWantJSONs() wantJSONs["union"] = makeUnionWantJSONs() wantJSONs["run_end_encoded"] = makeRunEndEncodedWantJSONs() + wantJSONs["view_types"] = makeViewTypesWantJSONs() tempDir := t.TempDir() for name, recs := range arrdata.Records { @@ -6127,3 +6128,261 @@ func makeRunEndEncodedWantJSONs() string { ] }` } + +func makeViewTypesWantJSONs() string { + return `{ + "schema": { + "fields": [ + { + "name": "binary_view", + "type": { + "name": "binaryview" + }, + "nullable": true, + "children": [] + }, + { + "name": "string_view", + "type": { + "name": "utf8view" + }, + "nullable": true, + "children": [] + } + ] + }, + "batches": [ + { + "count": 5, + "columns": [ + { + "name": "binary_view", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + { + "SIZE": 3, + "INLINED": "31C3A9" + }, + { + "SIZE": 0, + "INLINED": "" + }, + { + "SIZE": 0, + "INLINED": "" + }, + { + "SIZE": 1, + "INLINED": "34" + }, + { + "SIZE": 1, + "INLINED": "35" + } + ], + "VARIADIC_BUFFERS": [""] + }, + { + "name": "string_view", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + { + "SIZE": 3, + "INLINED": "1é" + }, + { + "SIZE": 0, + "INLINED": "" + }, + { + "SIZE": 0, + "INLINED": "" + }, + { + "SIZE": 1, + "INLINED": "4" + }, + { + "SIZE": 1, + "INLINED": "5" + } + ], + "VARIADIC_BUFFERS": [""] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "binary_view", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + { + "SIZE": 3, + "INLINED": "31C3A9" + }, + { + "SIZE": 0, + "INLINED": "" + }, + { + "SIZE": 0, + "INLINED": "" + }, + { + "SIZE": 4, + "INLINED": "34343434" + }, + { + "SIZE": 4, + "INLINED": "35353535" + } + ], + "VARIADIC_BUFFERS": [""] + }, + { + "name": "string_view", + "count": 5, + "VALIDITY": [ + 1, + 1, + 1, + 1, + 1 + ], + "DATA": [ + { + "SIZE": 3, + "INLINED": "1é" + }, + { + "SIZE": 14, + "PREFIX": "32323232", + "BUFFER_INDEX": 0, + "OFFSET": 0 + }, + { + "SIZE": 14, + "PREFIX": "33333333", + "BUFFER_INDEX": 0, + "OFFSET": 14 + }, + { + "SIZE": 4, + "INLINED": "4444" + }, + { + "SIZE": 4, + "INLINED": "5555" + } + ], + "VARIADIC_BUFFERS": [ + "32323232323232323232323232323333333333333333333333333333" + ] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "binary_view", + "count": 5, + "VALIDITY": [ + 1, + 1, + 1, + 1, + 1 + ], + "DATA": [ + { + "SIZE": 6, + "INLINED": "31C3A931C3A9" + }, + { + "SIZE": 14, + "PREFIX": "32323232", + "BUFFER_INDEX": 0, + "OFFSET": 0 + }, + { + "SIZE": 14, + "PREFIX": "33333333", + "BUFFER_INDEX": 0, + "OFFSET": 14 + }, + { + "SIZE": 2, + "INLINED": "3434" + }, + { + "SIZE": 2, + "INLINED": "3535" + } + ], + "VARIADIC_BUFFERS": [ + "32323232323232323232323232323333333333333333333333333333" + ] + }, + { + "name": "string_view", + "count": 5, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1 + ], + "DATA": [ + { + "SIZE": 6, + "INLINED": "1é1é" + }, + { + "SIZE": 0, + "INLINED": "" + }, + { + "SIZE": 0, + "INLINED": "" + }, + { + "SIZE": 2, + "INLINED": "44" + }, + { + "SIZE": 2, + "INLINED": "55" + } + ], + "VARIADIC_BUFFERS": [""] + } + ] + } + ] +}` +} diff --git a/go/arrow/internal/flatbuf/MetadataVersion.go b/go/arrow/internal/flatbuf/MetadataVersion.go index 21b234f9c2b21..bb5e99dd588ad 100644 --- a/go/arrow/internal/flatbuf/MetadataVersion.go +++ b/go/arrow/internal/flatbuf/MetadataVersion.go @@ -31,7 +31,7 @@ const ( MetadataVersionV3 MetadataVersion = 2 /// >= 0.8.0 (December 2017). Non-backwards compatible with V3. MetadataVersionV4 MetadataVersion = 3 - /// >= 1.0.0 (July 2020. Backwards compatible with V4 (V5 readers can read V4 + /// >= 1.0.0 (July 2020). Backwards compatible with V4 (V5 readers can read V4 /// metadata and IPC messages). Implementations are recommended to provide a /// V4 compatibility mode with V5 format changes disabled. /// diff --git a/go/arrow/internal/testing/gen/random_array_gen.go b/go/arrow/internal/testing/gen/random_array_gen.go index b42273ff93fac..57b417bd2b878 100644 --- a/go/arrow/internal/testing/gen/random_array_gen.go +++ b/go/arrow/internal/testing/gen/random_array_gen.go @@ -351,6 +351,40 @@ func (r *RandomArrayGenerator) LargeString(size int64, minLength, maxLength int6 return bldr.NewArray() } +func (r *RandomArrayGenerator) StringView(size int64, minLength, maxLength int64, nullProb float64) arrow.Array { + return r.generateBinaryView(arrow.BinaryTypes.StringView, size, minLength, maxLength, nullProb) +} + +func (r *RandomArrayGenerator) generateBinaryView(dt arrow.DataType, size int64, minLength, maxLength int64, nullProb float64) arrow.Array { + lengths := r.Int32(size, int32(minLength), int32(maxLength), nullProb).(*array.Int32) + defer lengths.Release() + + bldr := array.NewBuilder(r.mem, dt).(array.StringLikeBuilder) + defer bldr.Release() + + r.extra++ + dist := rand.New(rand.NewSource(r.seed + r.extra)) + + buf := make([]byte, 0, maxLength) + gen := func(n int32) string { + out := buf[:n] + for i := range out { + out[i] = uint8(dist.Int31n(int32('z')-int32('A')+1) + int32('A')) + } + return string(out) + } + + for i := 0; i < lengths.Len(); i++ { + if lengths.IsNull(i) { + bldr.AppendNull() + continue + } + bldr.Append(gen(lengths.Value(i))) + } + + return bldr.NewArray() +} + func (r *RandomArrayGenerator) Numeric(dt arrow.Type, size int64, min, max int64, nullprob float64) arrow.Array { switch dt { case arrow.INT8: diff --git a/go/arrow/ipc/endian_swap.go b/go/arrow/ipc/endian_swap.go index d2e0948434abc..35ba0e4e764f9 100644 --- a/go/arrow/ipc/endian_swap.go +++ b/go/arrow/ipc/endian_swap.go @@ -18,6 +18,7 @@ package ipc import ( "errors" + "fmt" "math/bits" "github.com/apache/arrow/go/v15/arrow" @@ -119,7 +120,10 @@ func swapType(dt arrow.DataType, data *array.Data) (err error) { return swapType(dt.IndexType, data) case arrow.FixedWidthDataType: byteSwapBuffer(dt.BitWidth(), data.Buffers()[1]) + default: + err = fmt.Errorf("%w: swapping endianness of %s", arrow.ErrNotImplemented, dt) } + return } diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go index 330355d3a60c3..1c7eb31799cfa 100644 --- a/go/arrow/ipc/file_reader.go +++ b/go/arrow/ipc/file_reader.go @@ -430,13 +430,18 @@ func (src *ipcSource) fieldMetadata(i int) *flatbuf.FieldNode { return &node } +func (src *ipcSource) variadicCount(i int) int64 { + return src.meta.VariadicBufferCounts(i) +} + type arrayLoaderContext struct { - src ipcSource - ifield int - ibuffer int - max int - memo *dictutils.Memo - version MetadataVersion + src ipcSource + ifield int + ibuffer int + ivariadic int + max int + memo *dictutils.Memo + version MetadataVersion } func (ctx *arrayLoaderContext) field() *flatbuf.FieldNode { @@ -451,6 +456,12 @@ func (ctx *arrayLoaderContext) buffer() *memory.Buffer { return buf } +func (ctx *arrayLoaderContext) variadic() int64 { + v := ctx.src.variadicCount(ctx.ivariadic) + ctx.ivariadic++ + return v +} + func (ctx *arrayLoaderContext) loadArray(dt arrow.DataType) arrow.ArrayData { switch dt := dt.(type) { case *arrow.NullType: @@ -476,6 +487,9 @@ func (ctx *arrayLoaderContext) loadArray(dt arrow.DataType) arrow.ArrayData { case *arrow.BinaryType, *arrow.StringType, *arrow.LargeStringType, *arrow.LargeBinaryType: return ctx.loadBinary(dt) + case arrow.BinaryViewDataType: + return ctx.loadBinaryView(dt) + case *arrow.FixedSizeBinaryType: return ctx.loadFixedSizeBinary(dt) @@ -582,6 +596,18 @@ func (ctx *arrayLoaderContext) loadBinary(dt arrow.DataType) arrow.ArrayData { return array.NewData(dt, int(field.Length()), buffers, nil, int(field.NullCount()), 0) } +func (ctx *arrayLoaderContext) loadBinaryView(dt arrow.DataType) arrow.ArrayData { + nVariadicBufs := ctx.variadic() + field, buffers := ctx.loadCommon(dt.ID(), 2+int(nVariadicBufs)) + buffers = append(buffers, ctx.buffer()) + for i := 0; i < int(nVariadicBufs); i++ { + buffers = append(buffers, ctx.buffer()) + } + defer releaseBuffers(buffers) + + return array.NewData(dt, int(field.Length()), buffers, nil, int(field.NullCount()), 0) +} + func (ctx *arrayLoaderContext) loadFixedSizeBinary(dt *arrow.FixedSizeBinaryType) arrow.ArrayData { field, buffers := ctx.loadCommon(dt.ID(), 2) buffers = append(buffers, ctx.buffer()) diff --git a/go/arrow/ipc/message.go b/go/arrow/ipc/message.go index 709aa5aa2dba4..5295c5df30137 100644 --- a/go/arrow/ipc/message.go +++ b/go/arrow/ipc/message.go @@ -31,11 +31,11 @@ import ( type MetadataVersion flatbuf.MetadataVersion const ( - MetadataV1 = MetadataVersion(flatbuf.MetadataVersionV1) // version for Arrow-0.1.0 - MetadataV2 = MetadataVersion(flatbuf.MetadataVersionV2) // version for Arrow-0.2.0 - MetadataV3 = MetadataVersion(flatbuf.MetadataVersionV3) // version for Arrow-0.3.0 to 0.7.1 - MetadataV4 = MetadataVersion(flatbuf.MetadataVersionV4) // version for >= Arrow-0.8.0 - MetadataV5 = MetadataVersion(flatbuf.MetadataVersionV5) // version for >= Arrow-1.0.0, backward compatible with v4 + MetadataV1 = MetadataVersion(flatbuf.MetadataVersionV1) // version for Arrow Format-0.1.0 + MetadataV2 = MetadataVersion(flatbuf.MetadataVersionV2) // version for Arrow Format-0.2.0 + MetadataV3 = MetadataVersion(flatbuf.MetadataVersionV3) // version for Arrow Format-0.3.0 to 0.7.1 + MetadataV4 = MetadataVersion(flatbuf.MetadataVersionV4) // version for >= Arrow Format-0.8.0 + MetadataV5 = MetadataVersion(flatbuf.MetadataVersionV5) // version for >= Arrow Format-1.0.0, backward compatible with v4 ) func (m MetadataVersion) String() string { diff --git a/go/arrow/ipc/metadata.go b/go/arrow/ipc/metadata.go index bd437834c3d06..54ef58753a173 100644 --- a/go/arrow/ipc/metadata.go +++ b/go/arrow/ipc/metadata.go @@ -323,6 +323,16 @@ func (fv *fieldVisitor) visit(field arrow.Field) { flatbuf.LargeUtf8Start(fv.b) fv.offset = flatbuf.LargeUtf8End(fv.b) + case *arrow.BinaryViewType: + fv.dtype = flatbuf.TypeBinaryView + flatbuf.BinaryViewStart(fv.b) + fv.offset = flatbuf.BinaryViewEnd(fv.b) + + case *arrow.StringViewType: + fv.dtype = flatbuf.TypeUtf8View + flatbuf.Utf8ViewStart(fv.b) + fv.offset = flatbuf.Utf8ViewEnd(fv.b) + case *arrow.Date32Type: fv.dtype = flatbuf.TypeDate flatbuf.DateStart(fv.b) @@ -713,6 +723,12 @@ func concreteTypeFromFB(typ flatbuf.Type, data flatbuffers.Table, children []arr case flatbuf.TypeLargeUtf8: return arrow.BinaryTypes.LargeString, nil + case flatbuf.TypeUtf8View: + return arrow.BinaryTypes.StringView, nil + + case flatbuf.TypeBinaryView: + return arrow.BinaryTypes.BinaryView, nil + case flatbuf.TypeBool: return arrow.FixedWidthTypes.Boolean, nil @@ -1168,15 +1184,15 @@ func writeFileFooter(schema *arrow.Schema, dicts, recs []fileBlock, w io.Writer) return err } -func writeRecordMessage(mem memory.Allocator, size, bodyLength int64, fields []fieldMetadata, meta []bufferMetadata, codec flatbuf.CompressionType) *memory.Buffer { +func writeRecordMessage(mem memory.Allocator, size, bodyLength int64, fields []fieldMetadata, meta []bufferMetadata, codec flatbuf.CompressionType, variadicCounts []int64) *memory.Buffer { b := flatbuffers.NewBuilder(0) - recFB := recordToFB(b, size, bodyLength, fields, meta, codec) + recFB := recordToFB(b, size, bodyLength, fields, meta, codec, variadicCounts) return writeMessageFB(b, mem, flatbuf.MessageHeaderRecordBatch, recFB, bodyLength) } -func writeDictionaryMessage(mem memory.Allocator, id int64, isDelta bool, size, bodyLength int64, fields []fieldMetadata, meta []bufferMetadata, codec flatbuf.CompressionType) *memory.Buffer { +func writeDictionaryMessage(mem memory.Allocator, id int64, isDelta bool, size, bodyLength int64, fields []fieldMetadata, meta []bufferMetadata, codec flatbuf.CompressionType, variadicCounts []int64) *memory.Buffer { b := flatbuffers.NewBuilder(0) - recFB := recordToFB(b, size, bodyLength, fields, meta, codec) + recFB := recordToFB(b, size, bodyLength, fields, meta, codec, variadicCounts) flatbuf.DictionaryBatchStart(b) flatbuf.DictionaryBatchAddId(b, id) @@ -1186,7 +1202,7 @@ func writeDictionaryMessage(mem memory.Allocator, id int64, isDelta bool, size, return writeMessageFB(b, mem, flatbuf.MessageHeaderDictionaryBatch, dictFB, bodyLength) } -func recordToFB(b *flatbuffers.Builder, size, bodyLength int64, fields []fieldMetadata, meta []bufferMetadata, codec flatbuf.CompressionType) flatbuffers.UOffsetT { +func recordToFB(b *flatbuffers.Builder, size, bodyLength int64, fields []fieldMetadata, meta []bufferMetadata, codec flatbuf.CompressionType, variadicCounts []int64) flatbuffers.UOffsetT { fieldsFB := writeFieldNodes(b, fields, flatbuf.RecordBatchStartNodesVector) metaFB := writeBuffers(b, meta, flatbuf.RecordBatchStartBuffersVector) var bodyCompressFB flatbuffers.UOffsetT @@ -1194,10 +1210,24 @@ func recordToFB(b *flatbuffers.Builder, size, bodyLength int64, fields []fieldMe bodyCompressFB = writeBodyCompression(b, codec) } + var vcFB *flatbuffers.UOffsetT + if len(variadicCounts) > 0 { + flatbuf.RecordBatchStartVariadicBufferCountsVector(b, len(variadicCounts)) + for i := len(variadicCounts) - 1; i >= 0; i-- { + b.PrependInt64(variadicCounts[i]) + } + vcFBVal := b.EndVector(len(variadicCounts)) + vcFB = &vcFBVal + } + flatbuf.RecordBatchStart(b) flatbuf.RecordBatchAddLength(b, size) flatbuf.RecordBatchAddNodes(b, fieldsFB) flatbuf.RecordBatchAddBuffers(b, metaFB) + if vcFB != nil { + flatbuf.RecordBatchAddVariadicBufferCounts(b, *vcFB) + } + if codec != -1 { flatbuf.RecordBatchAddCompression(b, bodyCompressFB) } diff --git a/go/arrow/ipc/writer.go b/go/arrow/ipc/writer.go index 58c56d2d16ccf..e9d59f0e35e00 100644 --- a/go/arrow/ipc/writer.go +++ b/go/arrow/ipc/writer.go @@ -277,7 +277,7 @@ type dictEncoder struct { } func (d *dictEncoder) encodeMetadata(p *Payload, isDelta bool, id, nrows int64) error { - p.meta = writeDictionaryMessage(d.mem, id, isDelta, nrows, p.size, d.fields, d.meta, d.codec) + p.meta = writeDictionaryMessage(d.mem, id, isDelta, nrows, p.size, d.fields, d.meta, d.codec, d.variadicCounts) return nil } @@ -300,8 +300,9 @@ func (d *dictEncoder) Encode(p *Payload, id int64, isDelta bool, dict arrow.Arra type recordEncoder struct { mem memory.Allocator - fields []fieldMetadata - meta []bufferMetadata + fields []fieldMetadata + meta []bufferMetadata + variadicCounts []int64 depth int64 start int64 @@ -602,6 +603,33 @@ func (w *recordEncoder) visit(p *Payload, arr arrow.Array) error { p.body = append(p.body, voffsets) p.body = append(p.body, values) + case arrow.BinaryViewDataType: + data := arr.Data() + values := data.Buffers()[1] + arrLen := int64(arr.Len()) + typeWidth := int64(arrow.ViewHeaderSizeBytes) + minLength := paddedLength(arrLen*typeWidth, kArrowAlignment) + + switch { + case needTruncate(int64(data.Offset()), values, minLength): + // non-zero offset: slice the buffer + offset := data.Offset() * int(typeWidth) + // send padding if available + len := int(minI64(bitutil.CeilByte64(arrLen*typeWidth), int64(values.Len()-offset))) + values = memory.SliceBuffer(values, offset, len) + default: + if values != nil { + values.Retain() + } + } + p.body = append(p.body, values) + + w.variadicCounts = append(w.variadicCounts, int64(len(data.Buffers())-2)) + for _, b := range data.Buffers()[2:] { + b.Retain() + p.body = append(p.body, b) + } + case *arrow.StructType: w.depth-- arr := arr.(*array.Struct) @@ -946,7 +974,7 @@ func (w *recordEncoder) Encode(p *Payload, rec arrow.Record) error { } func (w *recordEncoder) encodeMetadata(p *Payload, nrows int64) error { - p.meta = writeRecordMessage(w.mem, nrows, p.size, w.fields, w.meta, w.codec) + p.meta = writeRecordMessage(w.mem, nrows, p.size, w.fields, w.meta, w.codec, w.variadicCounts) return nil } diff --git a/go/arrow/type_traits_view.go b/go/arrow/type_traits_view.go new file mode 100644 index 0000000000000..c3846db294681 --- /dev/null +++ b/go/arrow/type_traits_view.go @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "reflect" + "unsafe" + + "github.com/apache/arrow/go/v15/arrow/endian" +) + +var ViewHeaderTraits viewHeaderTraits + +const ( + ViewHeaderSizeBytes = int(unsafe.Sizeof(ViewHeader{})) +) + +type viewHeaderTraits struct{} + +func (viewHeaderTraits) BytesRequired(n int) int { return ViewHeaderSizeBytes * n } + +func (viewHeaderTraits) PutValue(b []byte, v ViewHeader) { + endian.Native.PutUint32(b, uint32(v.size)) + copy(b[4:], v.data[:]) +} + +func (viewHeaderTraits) CastFromBytes(b []byte) (res []ViewHeader) { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + return unsafe.Slice((*ViewHeader)(unsafe.Pointer(h.Data)), cap(b)/ViewHeaderSizeBytes)[:len(b)/ViewHeaderSizeBytes] +} + +func (viewHeaderTraits) CastToBytes(b []ViewHeader) (res []byte) { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + return unsafe.Slice((*byte)(unsafe.Pointer(h.Data)), cap(b)*ViewHeaderSizeBytes)[:len(b)*ViewHeaderSizeBytes] +} + +func (viewHeaderTraits) Copy(dst, src []ViewHeader) { copy(dst, src) } From a886fdaa2d80a2e7f56cf8a3cf94b367443b6e8e Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 14 Nov 2023 22:19:48 -0400 Subject: [PATCH 52/57] GH-38715: [R] Fix possible bashism in configure script (#38716) ### Rationale for this change The CRAN incoming check for 14.0.0 is failing with a NOTE about a possible bashism ### What changes are included in this PR? One `test -a` usage was replaced with `&&`. ### Are these changes tested? Yes (via crossbow, below) ### Are there any user-facing changes? No * Closes: #38715 Authored-by: Dewey Dunnington Signed-off-by: Dewey Dunnington --- r/configure | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/r/configure b/r/configure index 5a7f0c81a947b..4f09cfdc4419b 100755 --- a/r/configure +++ b/r/configure @@ -126,7 +126,7 @@ fi # but doing it now allows us to catch it in # nixlibs.R and activate S3 and GCS support for the source build. -# macOS ships with libressl. openssl is installable with brew, but it is +# macOS ships with libressl. openssl is installable with brew, but it is # generally not linked. We can over-ride this and find # openssl by setting OPENSSL_ROOT_DIR (which cmake will pick up later in # the installation process). @@ -135,7 +135,7 @@ if [ "${OPENSSL_ROOT_DIR}" = "" ] && brew --prefix openssl >/dev/null 2>&1; then export PKG_CONFIG_PATH="${OPENSSL_ROOT_DIR}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" fi # Look for openssl with pkg-config for non-brew sources(e.g. CRAN) and Linux -if [ "${OPENSSL_ROOT_DIR}" = "" -a "${PKG_CONFIG_AVAILABLE}" = "true" ]; then +if [ "${OPENSSL_ROOT_DIR}" = "" ] && [ "${PKG_CONFIG_AVAILABLE}" = "true" ]; then if ${PKG_CONFIG} --exists openssl; then export OPENSSL_ROOT_DIR="`${PKG_CONFIG} --variable=prefix openssl`" fi @@ -282,7 +282,7 @@ set_pkg_vars () { PKG_CFLAGS="$PKG_CFLAGS $ARROW_R_CXXFLAGS" fi - # We use expr because the product version returns more than just 10.13 and we want to + # We use expr because the product version returns more than just 10.13 and we want to # match the substring. However, expr always outputs the number of matched characters # to stdout, to avoid noise in the log we redirect the output to /dev/null if [ "$UNAME" = "Darwin" ] && expr $(sw_vers -productVersion) : '10\.13' >/dev/null 2>&1; then From e49d8ae15583ceff03237571569099a6ad62be32 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Wed, 15 Nov 2023 14:17:42 +0900 Subject: [PATCH 53/57] GH-38711: [CI] Rollback aws-cli for preview documentation (#38723) ### Rationale for this change Restored the Runner image to not delete `aws-cli` for the execution of the `preview-docs` command. ### Are these changes tested? No ### Are there any user-facing changes? No * Closes: #38711 Authored-by: Hyunseok Seo Signed-off-by: Sutou Kouhei --- ci/scripts/util_free_space.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/scripts/util_free_space.sh b/ci/scripts/util_free_space.sh index 0518869d06993..dd6ba2c4600a9 100755 --- a/ci/scripts/util_free_space.sh +++ b/ci/scripts/util_free_space.sh @@ -25,7 +25,6 @@ du -hsc /usr/local/* echo "::endgroup::" # ~1GB sudo rm -rf \ - /usr/local/aws-cli \ /usr/local/aws-sam-cil \ /usr/local/julia* || : echo "::group::/usr/local/bin/*" @@ -34,8 +33,6 @@ echo "::endgroup::" # ~1GB (From 1.2GB to 214MB) sudo rm -rf \ /usr/local/bin/aliyun \ - /usr/local/bin/aws \ - /usr/local/bin/aws_completer \ /usr/local/bin/azcopy \ /usr/local/bin/bicep \ /usr/local/bin/cmake-gui \ From 5b17b8402e0444f1a9b9ce1fb4dc2b7b92e9aede Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 15 Nov 2023 09:21:22 +0100 Subject: [PATCH 54/57] GH-38712: [Python] Remove dead code in _reconstruct_block (#38714) ### Rationale for this change It seems the object case in `_reconstruct_block` is a dead code and is not needed anymore so therefore could be removed. ### What changes are included in this PR? Removal of the object case in `_reconstruct_block` code. Was also looking at the `arrow_to_pandas.cc` code to see if there is any dead code present and I couldn't find any. ### Are these changes tested? The change in this PR should not make any of the existing tests fail. ### Are there any user-facing changes? There shouldn't be. * Closes: #38712 Authored-by: AlenkaF Signed-off-by: Joris Van den Bossche --- python/pyarrow/pandas_compat.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index e232603ba45ac..be29f68a13d5f 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -26,7 +26,6 @@ from itertools import zip_longest import json import operator -import pickle import re import warnings @@ -721,9 +720,6 @@ def _reconstruct_block(item, columns=None, extension_columns=None): block = _int.make_block(block_arr, placement=placement, klass=_int.DatetimeTZBlock, dtype=dtype) - elif 'object' in item: - block = _int.make_block(pickle.loads(block_arr), - placement=placement) elif 'py_array' in item: # create ExtensionBlock arr = item['py_array'] From cc627ee7e35807a98717603d0a2520685919e17c Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Wed, 15 Nov 2023 19:01:43 +0900 Subject: [PATCH 55/57] GH-38599: [Docs] Update Headers (#38696) ### Rationale for this change Noticed wrong section headings on the web doc and proposing a fix. ### Are these changes tested? Yes. Built and verified the documentation locally. image ### Are there any user-facing changes? No. * Closes: #38599 Authored-by: Hyunseok Seo Signed-off-by: Antoine Pitrou --- docs/source/format/CDeviceDataInterface.rst | 29 ++++++++++----------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/docs/source/format/CDeviceDataInterface.rst b/docs/source/format/CDeviceDataInterface.rst index b54e6eabe0b2d..a584852df87eb 100644 --- a/docs/source/format/CDeviceDataInterface.rst +++ b/docs/source/format/CDeviceDataInterface.rst @@ -277,7 +277,7 @@ has the following fields: to access the memory in the buffers. If an event is provided, then the producer MUST ensure that the exported - data is available on the device before the event is triggered. The + data is available on the device before the event is triggered. The consumer SHOULD wait on the event before trying to access the exported data. @@ -290,7 +290,7 @@ has the following fields: As non-CPU development expands, there may be a need to expand this structure. In order to do so without potentially breaking ABI changes, we reserve 24 bytes at the end of the object. These bytes MUST be zero'd - out after initialization by the producer in order to ensure safe + out after initialization by the producer in order to ensure safe evolution of the ABI in the future. .. _c-device-data-interface-event-types: @@ -300,7 +300,7 @@ Synchronization event types The table below lists the expected event types for each device type. If no event type is supported ("N/A"), then the ``sync_event`` member -should always be null. +should always be null. Remember that the event *CAN* be null if synchronization is not needed to access the data. @@ -352,7 +352,7 @@ Memory management ----------------- First and foremost: Out of everything in this interface, it is *only* the -data buffers themselves which reside in device memory (i.e. the ``buffers`` +data buffers themselves which reside in device memory (i.e. the ``buffers`` member of the ``ArrowArray`` struct). Everything else should be in CPU memory. @@ -408,7 +408,7 @@ see inconsistent data while the other is mutating it. Synchronization --------------- -If the ``sync_event`` member is non-NULL, the consumer should not attempt +If the ``sync_event`` member is non-NULL, the consumer should not attempt to access or read the data until they have synchronized on that event. If the ``sync_event`` member is NULL, then it MUST be safe to access the data without any synchronization necessary on the part of the consumer. @@ -501,7 +501,6 @@ could be used for any device: arr->array.release(&arr->array); } -======================= Device Stream Interface ======================= @@ -510,7 +509,7 @@ interface also specifies a higher-level structure for easing communication of streaming data within a single process. Semantics -========= +--------- An Arrow C device stream exposes a streaming source of data chunks, each with the same schema. Chunks are obtained by calling a blocking pull-style iteration @@ -520,7 +519,7 @@ to provide a stream of data on multiple device types, a producer should provide a separate stream object for each device type. Structure definition -==================== +-------------------- The C device stream interface is defined by a single ``struct`` definition: @@ -554,7 +553,7 @@ The C device stream interface is defined by a single ``struct`` definition: kept exactly as-is when these definitions are copied. The ArrowDeviceArrayStream structure ------------------------------------- +'''''''''''''''''''''''''''''''''''' The ``ArrowDeviceArrayStream`` provides a device type that can access the resulting data along with the required callbacks to interact with a @@ -627,20 +626,20 @@ streaming source of Arrow arrays. It has the following fields: handled by the producer, and especially by the release callback. Result lifetimes ----------------- +'''''''''''''''' The data returned by the ``get_schema`` and ``get_next`` callbacks must be released independantly. Their lifetimes are not tied to that of ``ArrowDeviceArrayStream``. Stream lifetime ---------------- +''''''''''''''' Lifetime of the C stream is managed using a release callback with similar usage as in :ref:`C data interface `. Thread safety -------------- +''''''''''''' The stream source is not assumed to be thread-safe. Consumers wanting to call ``get_next`` from several threads should ensure those calls are @@ -652,9 +651,9 @@ Interoperability with other interchange formats Other interchange APIs, such as the `CUDA Array Interface`_, include members to pass the shape and the data types of the data buffers being exported. This information is necessary to interpret the raw bytes in the -device data buffers that are being shared. Rather than store the -shape / types of the data alongside the ``ArrowDeviceArray``, users -should utilize the existing ``ArrowSchema`` structure to pass any data +device data buffers that are being shared. Rather than store the +shape / types of the data alongside the ``ArrowDeviceArray``, users +should utilize the existing ``ArrowSchema`` structure to pass any data type and shape information. Updating this specification From b55d13c16eb25f3264645e53cc03aa1f7d753b25 Mon Sep 17 00:00:00 2001 From: Ben Harkins <60872452+benibus@users.noreply.github.com> Date: Wed, 15 Nov 2023 06:05:07 -0500 Subject: [PATCH 56/57] GH-36036: [C++][Python][Parquet] Implement Float16 logical type (#36073) ### Rationale for this change There is currently an active proposal to support half-float types in Parquet. For more details/discussion, see the links in this PR's accompanying issue. ### What changes are included in this PR? This PR implements basic support for a `Float16LogicalType` in accordance with the proposed spec. More specifically, this includes: - Changes to `parquet.thrift` and regenerated `parqet_types` files - Basic `LogicalType` class definition, method impls, and enums - Support for specialized comparisons and column statistics In the interest of scope, this PR does not currently deal with arrow integration and byte split encoding - although we will want both of these features resolved before the proposal is approved. ### Are these changes tested? Yes (tests are included) ### Are there any user-facing changes? Yes * Closes: #36036 Lead-authored-by: benibus Co-authored-by: Ben Harkins <60872452+benibus@users.noreply.github.com> Co-authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/util/CMakeLists.txt | 1 + cpp/src/arrow/util/float16.cc | 226 ++ cpp/src/arrow/util/float16.h | 209 ++ cpp/src/arrow/util/float16_test.cc | 367 +++ cpp/src/generated/parquet_types.cpp | 2267 +++++++++-------- cpp/src/generated/parquet_types.h | 45 +- .../parquet/arrow/arrow_reader_writer_test.cc | 33 +- cpp/src/parquet/arrow/arrow_schema_test.cc | 7 +- cpp/src/parquet/arrow/reader_internal.cc | 25 + cpp/src/parquet/arrow/schema.cc | 5 + cpp/src/parquet/arrow/schema_internal.cc | 2 + cpp/src/parquet/arrow/test_util.h | 21 +- cpp/src/parquet/column_writer.cc | 30 + cpp/src/parquet/page_index_test.cc | 22 + cpp/src/parquet/parquet.thrift | 2 + cpp/src/parquet/schema_test.cc | 29 +- cpp/src/parquet/statistics.cc | 179 +- cpp/src/parquet/statistics_test.cc | 355 ++- cpp/src/parquet/test_util.cc | 10 + cpp/src/parquet/test_util.h | 4 + cpp/src/parquet/types.cc | 26 + cpp/src/parquet/types.h | 13 + docs/source/cpp/parquet.rst | 2 + 24 files changed, 2670 insertions(+), 1211 deletions(-) create mode 100644 cpp/src/arrow/util/float16.cc create mode 100644 cpp/src/arrow/util/float16.h create mode 100644 cpp/src/arrow/util/float16_test.cc diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 101b089ba837f..24e8eefad1523 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -223,6 +223,7 @@ set(ARROW_SRCS util/debug.cc util/decimal.cc util/delimiting.cc + util/float16.cc util/formatting.cc util/future.cc util/hashing.cc diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 3dc8eac1abf64..2e9487dcf50c8 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -48,6 +48,7 @@ add_arrow_test(utility-test checked_cast_test.cc compression_test.cc decimal_test.cc + float16_test.cc formatting_util_test.cc key_value_metadata_test.cc hashing_test.cc diff --git a/cpp/src/arrow/util/float16.cc b/cpp/src/arrow/util/float16.cc new file mode 100644 index 0000000000000..5c8b3d10ca0cd --- /dev/null +++ b/cpp/src/arrow/util/float16.cc @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "arrow/util/float16.h" +#include "arrow/util/ubsan.h" + +namespace arrow { +namespace util { + +namespace { + +// -------------------------------------------------------- +// Binary conversions +// -------------------------------------------------------- +// These routines are partially adapted from Numpy's C implementation +// +// Some useful metrics for conversions between different precisions: +// |-----------------------------------------| +// | precision | half | single | double | +// |-----------------------------------------| +// | mantissa | 10 bits | 23 bits | 52 bits | +// | exponent | 5 bits | 8 bits | 11 bits | +// | sign | 1 bit | 1 bit | 1 bit | +// | exp bias | 15 | 127 | 1023 | +// |-----------------------------------------| + +template +struct BinaryConverter { + static_assert(std::is_same_v || std::is_same_v); + + static constexpr int kNumBits = sizeof(T) * 8; + static constexpr int kMantNumBits = (kNumBits == 32) ? 23 : 52; + static constexpr int kExpNumBits = kNumBits - kMantNumBits - 1; + + static constexpr int kExpBias = (1 << (kExpNumBits - 1)) - 1; + + static constexpr T kMantMask = (T(1) << kMantNumBits) - 1; + static constexpr T kExpMask = ((T(1) << kExpNumBits) - 1) << kMantNumBits; + static constexpr T kSignMask = T(1) << (kNumBits - 1); + + static_assert(kMantNumBits + kExpNumBits + 1 == kNumBits); + static_assert(kSignMask + kExpMask + kMantMask == ~T(0)); + + static uint16_t ToBinary16(T); + static T FromBinary16(uint16_t); +}; + +// Converts a IEEE binary32/64 into a binary16. Rounds to nearest with ties to zero +template +uint16_t BinaryConverter::ToBinary16(T f_bits) { + // Sign mask for output binary16 + const uint16_t h_sign = uint16_t((f_bits >> (kNumBits - 16)) & 0x8000); + + // Exponent mask for input binary + const T f_exp = f_bits & kExpMask; + // Exponents as signed pre-shifted values for convenience. Here, we need to re-bias the + // exponent for a binary16. If, after re-biasing, the binary16 exponent falls outside of + // the range [1,30] then we need to handle the under/overflow case specially. + const int16_t f_biased_exp = int16_t(f_exp >> kMantNumBits); + const int16_t unbiased_exp = f_biased_exp - kExpBias; + const int16_t h_biased_exp = unbiased_exp + 15; + + // Mantissa mask for input + const T f_mant = f_bits & kMantMask; + + // We define a "rounding bit", which is the most significant bit to be dropped + // (e.g. for a binary32, 0x1000). + constexpr T rounding_bit = T(1) << (kMantNumBits - (10 + 1)); + + // Handle exponent overflow, NaN, and +/-Inf + if (h_biased_exp >= 0x1f) { + // The input is a NaN representation + if (f_exp == kExpMask && f_mant != 0) { + uint16_t h_mant = uint16_t(f_mant >> (kMantNumBits - 10)); + // If the mantissa bit(s) indicating NaN were shifted out, add one back. Otherwise, + // the result would be infinity. + if (h_mant == 0) { + h_mant = 0x1; + } + return uint16_t(h_sign | 0x7c00u | h_mant); + } + + // Clamp to +/-infinity + return uint16_t(h_sign | 0x7c00u); + } + + // Handle exponent underflow, subnormals, and +/-0 + if (h_biased_exp <= 0) { + // If the underflow exceeds the number of bits in a binary16 mantissa (10) then we + // can't round, so just clamp to 0. Note that this also weeds out any input values + // that are subnormal - including +/-0; + if (h_biased_exp < -10) { + return h_sign; + } + + // Convert to a rounded subnormal value starting with the mantissa. Since the input + // input is known to be normal at this point, we need to prepend its implicit leading + // bit - which also necessitates an additional right-shift. + T rounded_mant = (T(1) << kMantNumBits) | f_mant; + rounded_mant >>= (1 - h_biased_exp); + + // Here, we implement rounding to nearest (with ties to even) + // + // By now, our new mantissa has two conceptual ranges: + // - The lower 13 bits, which will be shifted out + // - The upper 10 bits, which will become the binary16's mantissa + // + // "Rounding to nearest" basically just means that we add 1 to the rounding bit. If + // it's set, then the bit will cascade upwards into the 10-bit mantissa (and + // potentially the exponent). The only time where we may NOT do this is when a "tie" + // occurs - i.e. when the rounding bit is set but all of the lower bits are 0. In that + // case, we don't add 1 if the retained mantissa is "even" (its least significant bit + // is 0). + if ((rounded_mant & ((rounding_bit << 2) - 1)) != rounding_bit || + (f_mant & 0x7ffu) != 0) { + rounded_mant += rounding_bit; + } + + const uint16_t h_mant = uint16_t(rounded_mant >> (kMantNumBits - 10)); + return h_sign + h_mant; + } + + const uint16_t h_exp = uint16_t(h_biased_exp) << 10; + + // See comment on rounding behavior above + T rounded_mant = f_mant; + if ((rounded_mant & ((rounding_bit << 2) - 1)) != rounding_bit) { + rounded_mant += rounding_bit; + } + + const uint16_t h_mant = uint16_t(rounded_mant >> (kMantNumBits - 10)); + // Note that we ADD (rather than OR) the components because we want the carryover bit + // from rounding the mantissa to cascade through the exponent (it shouldn't affect the + // sign bit though). + return h_sign + h_exp + h_mant; +} + +// Converts a IEEE binary16 into a binary32/64 +template +T BinaryConverter::FromBinary16(uint16_t h_bits) { + // Sign mask for output + const T f_sign = T(h_bits & 0x8000u) << (kNumBits - 16); + + // Exponent mask for input binary16 + const uint16_t h_exp = h_bits & 0x7c00; + // Mantissa mask for input binary16 + const uint16_t h_mant = h_bits & 0x3ffu; + + switch (h_exp) { + // Handle Inf and NaN + case 0x7c00u: + return f_sign | kExpMask | (T(h_mant) << (kMantNumBits - 10)); + // Handle zeros and subnormals + case 0x0000u: { + // Input is +/-0 + if (h_mant == 0) { + return f_sign; + } + // Subnormal binary16 to normal binary32/64 + // + // Start with an f32/64-biased exponent of 2^-15. We then decrement it until the + // most significant set bit is left-shifted out - as it doesn't get explicitly + // stored in normalized floating point values. Instead, its existence is implied by + // the new exponent. + T f_exp = kExpBias - 15; + T f_mant = T(h_mant) << 1; + while ((f_mant & 0x0400u) == 0) { + --f_exp; + f_mant <<= 1; + } + f_exp <<= kMantNumBits; + f_mant = (f_mant & 0x03ffu) << (kMantNumBits - 10); + return f_sign | f_exp | f_mant; + } break; + // Handle normals + default: + // Equivalent to rebiasing the exponent and shifting everything by the remaining + // mantissa bits. + return f_sign | + ((T(h_bits & 0x7fffu) + (T(kExpBias - 15) << 10)) << (kMantNumBits - 10)); + } +} + +} // namespace + +float Float16::ToFloat() const { + const uint32_t f_bits = BinaryConverter::FromBinary16(bits_); + return SafeCopy(f_bits); +} + +Float16 Float16::FromFloat(float f) { + const uint32_t f_bits = SafeCopy(f); + return FromBits(BinaryConverter::ToBinary16(f_bits)); +} + +double Float16::ToDouble() const { + const uint64_t d_bits = BinaryConverter::FromBinary16(bits_); + return SafeCopy(d_bits); +} + +Float16 Float16::FromDouble(double d) { + const uint64_t d_bits = SafeCopy(d); + return FromBits(BinaryConverter::ToBinary16(d_bits)); +} + +std::ostream& operator<<(std::ostream& os, Float16 arg) { return (os << arg.ToFloat()); } + +} // namespace util +} // namespace arrow diff --git a/cpp/src/arrow/util/float16.h b/cpp/src/arrow/util/float16.h new file mode 100644 index 0000000000000..0a432fee2cd31 --- /dev/null +++ b/cpp/src/arrow/util/float16.h @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/util/endian.h" +#include "arrow/util/macros.h" +#include "arrow/util/ubsan.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace util { + +/// \brief Class representing an IEEE half-precision float, encoded as a `uint16_t` +/// +/// The exact format is as follows (from LSB to MSB): +/// - bits 0-10: mantissa +/// - bits 10-15: exponent +/// - bit 15: sign +/// +class ARROW_EXPORT Float16 { + public: + Float16() = default; + explicit Float16(float f) : Float16(FromFloat(f)) {} + explicit Float16(double d) : Float16(FromDouble(d)) {} + template >* = NULLPTR> + explicit Float16(T v) : Float16(static_cast(v)) {} + + /// \brief Create a `Float16` from its exact binary representation + constexpr static Float16 FromBits(uint16_t bits) { return Float16{bits, bool{}}; } + /// \brief Create a `Float16` from a 32-bit float (may lose precision) + static Float16 FromFloat(float f); + /// \brief Create a `Float16` from a 64-bit float (may lose precision) + static Float16 FromDouble(double d); + + /// \brief Read a `Float16` from memory in native-endian byte order + static Float16 FromBytes(const uint8_t* src) { + return FromBits(SafeLoadAs(src)); + } + + /// \brief Read a `Float16` from memory in little-endian byte order + static Float16 FromLittleEndian(const uint8_t* src) { + return FromBits(::arrow::bit_util::FromLittleEndian(SafeLoadAs(src))); + } + + /// \brief Read a `Float16` from memory in big-endian byte order + static Float16 FromBigEndian(const uint8_t* src) { + return FromBits(::arrow::bit_util::FromBigEndian(SafeLoadAs(src))); + } + + /// \brief Return the value's binary representation as a `uint16_t` + constexpr uint16_t bits() const { return bits_; } + + /// \brief Return true if the value is negative (sign bit is set) + constexpr bool signbit() const { return (bits_ & 0x8000) != 0; } + + /// \brief Return true if the value is NaN + constexpr bool is_nan() const { return (bits_ & 0x7fff) > 0x7c00; } + /// \brief Return true if the value is positive/negative infinity + constexpr bool is_infinity() const { return (bits_ & 0x7fff) == 0x7c00; } + /// \brief Return true if the value is finite and not NaN + constexpr bool is_finite() const { return (bits_ & 0x7c00) != 0x7c00; } + /// \brief Return true if the value is positive/negative zero + constexpr bool is_zero() const { return (bits_ & 0x7fff) == 0; } + + /// \brief Convert to a 32-bit float + float ToFloat() const; + /// \brief Convert to a 64-bit float + double ToDouble() const; + + explicit operator float() const { return ToFloat(); } + explicit operator double() const { return ToDouble(); } + + /// \brief Copy the value's bytes in native-endian byte order + void ToBytes(uint8_t* dest) const { std::memcpy(dest, &bits_, sizeof(bits_)); } + /// \brief Return the value's bytes in native-endian byte order + constexpr std::array ToBytes() const { +#if ARROW_LITTLE_ENDIAN + return ToLittleEndian(); +#else + return ToBigEndian(); +#endif + } + + /// \brief Copy the value's bytes in little-endian byte order + void ToLittleEndian(uint8_t* dest) const { + const auto bytes = ToLittleEndian(); + std::memcpy(dest, bytes.data(), bytes.size()); + } + /// \brief Return the value's bytes in little-endian byte order + constexpr std::array ToLittleEndian() const { +#if ARROW_LITTLE_ENDIAN + return {uint8_t(bits_ & 0xff), uint8_t(bits_ >> 8)}; +#else + return {uint8_t(bits_ >> 8), uint8_t(bits_ & 0xff)}; +#endif + } + + /// \brief Copy the value's bytes in big-endian byte order + void ToBigEndian(uint8_t* dest) const { + const auto bytes = ToBigEndian(); + std::memcpy(dest, bytes.data(), bytes.size()); + } + /// \brief Return the value's bytes in big-endian byte order + constexpr std::array ToBigEndian() const { +#if ARROW_LITTLE_ENDIAN + return {uint8_t(bits_ >> 8), uint8_t(bits_ & 0xff)}; +#else + return {uint8_t(bits_ & 0xff), uint8_t(bits_ >> 8)}; +#endif + } + + constexpr Float16 operator-() const { return FromBits(bits_ ^ 0x8000); } + constexpr Float16 operator+() const { return FromBits(bits_); } + + friend constexpr bool operator==(Float16 lhs, Float16 rhs) { + if (lhs.is_nan() || rhs.is_nan()) return false; + return Float16::CompareEq(lhs, rhs); + } + friend constexpr bool operator!=(Float16 lhs, Float16 rhs) { return !(lhs == rhs); } + + friend constexpr bool operator<(Float16 lhs, Float16 rhs) { + if (lhs.is_nan() || rhs.is_nan()) return false; + return Float16::CompareLt(lhs, rhs); + } + friend constexpr bool operator>(Float16 lhs, Float16 rhs) { return rhs < lhs; } + + friend constexpr bool operator<=(Float16 lhs, Float16 rhs) { + if (lhs.is_nan() || rhs.is_nan()) return false; + return !Float16::CompareLt(rhs, lhs); + } + friend constexpr bool operator>=(Float16 lhs, Float16 rhs) { return rhs <= lhs; } + + ARROW_FRIEND_EXPORT friend std::ostream& operator<<(std::ostream& os, Float16 arg); + + protected: + uint16_t bits_; + + private: + constexpr Float16(uint16_t bits, bool) : bits_(bits) {} + + // Comparison helpers that assume neither operand is NaN + static constexpr bool CompareEq(Float16 lhs, Float16 rhs) { + return (lhs.bits() == rhs.bits()) || (lhs.is_zero() && rhs.is_zero()); + } + static constexpr bool CompareLt(Float16 lhs, Float16 rhs) { + if (lhs.signbit()) { + if (rhs.signbit()) { + // Both are negative + return lhs.bits() > rhs.bits(); + } else { + // Handle +/-0 + return !lhs.is_zero() || rhs.bits() != 0; + } + } else if (rhs.signbit()) { + return false; + } else { + // Both are positive + return lhs.bits() < rhs.bits(); + } + } +}; + +static_assert(std::is_trivial_v); + +} // namespace util +} // namespace arrow + +// TODO: Not complete +template <> +class std::numeric_limits { + using T = arrow::util::Float16; + + public: + static constexpr bool is_specialized = true; + static constexpr bool is_signed = true; + static constexpr bool has_infinity = true; + static constexpr bool has_quiet_NaN = true; + + static constexpr T min() { return T::FromBits(0b0000010000000000); } + static constexpr T max() { return T::FromBits(0b0111101111111111); } + static constexpr T lowest() { return -max(); } + + static constexpr T infinity() { return T::FromBits(0b0111110000000000); } + + static constexpr T quiet_NaN() { return T::FromBits(0b0111111111111111); } +}; diff --git a/cpp/src/arrow/util/float16_test.cc b/cpp/src/arrow/util/float16_test.cc new file mode 100644 index 0000000000000..073375882e3c2 --- /dev/null +++ b/cpp/src/arrow/util/float16_test.cc @@ -0,0 +1,367 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include + +#include "arrow/testing/gtest_util.h" +#include "arrow/util/endian.h" +#include "arrow/util/float16.h" +#include "arrow/util/span.h" +#include "arrow/util/ubsan.h" + +namespace arrow::util { +namespace { + +template +using Limits = std::numeric_limits; + +float F32(uint32_t bits) { return SafeCopy(bits); } +double F64(uint64_t bits) { return SafeCopy(bits); } + +template +class Float16ConversionTest : public ::testing::Test { + public: + struct RoundTripTestCase { + T input; + uint16_t bits; + T output; + }; + + static void TestRoundTrip(span test_cases) { + for (size_t index = 0; index < test_cases.size(); ++index) { + ARROW_SCOPED_TRACE("i=", index); + const auto& tc = test_cases[index]; + + const auto f16 = Float16(tc.input); + EXPECT_EQ(tc.bits, f16.bits()); + EXPECT_EQ(tc.output, static_cast(f16)); + + EXPECT_EQ(std::signbit(tc.output), f16.signbit()); + EXPECT_EQ(std::isnan(tc.output), f16.is_nan()); + EXPECT_EQ(std::isinf(tc.output), f16.is_infinity()); + EXPECT_EQ(std::isfinite(tc.output), f16.is_finite()); + } + } + + static void TestRoundTripFromNaN(span test_cases) { + for (size_t i = 0; i < test_cases.size(); ++i) { + ARROW_SCOPED_TRACE("i=", i); + const auto input = test_cases[i]; + + ASSERT_TRUE(std::isnan(input)); + const bool sign = std::signbit(input); + + const auto f16 = Float16(input); + EXPECT_TRUE(f16.is_nan()); + EXPECT_EQ(std::isinf(input), f16.is_infinity()); + EXPECT_EQ(std::isfinite(input), f16.is_finite()); + EXPECT_EQ(sign, f16.signbit()); + + const auto output = static_cast(f16); + EXPECT_TRUE(std::isnan(output)); + EXPECT_EQ(sign, std::signbit(output)); + } + } + + void TestRoundTripFromInf() { + const T test_cases[] = {+Limits::infinity(), -Limits::infinity()}; + + for (size_t i = 0; i < std::size(test_cases); ++i) { + ARROW_SCOPED_TRACE("i=", i); + const auto input = test_cases[i]; + + ASSERT_TRUE(std::isinf(input)); + const bool sign = std::signbit(input); + + const auto f16 = Float16(input); + EXPECT_TRUE(f16.is_infinity()); + EXPECT_EQ(std::isfinite(input), f16.is_finite()); + EXPECT_EQ(std::isnan(input), f16.is_nan()); + EXPECT_EQ(sign, f16.signbit()); + + const auto output = static_cast(f16); + EXPECT_TRUE(std::isinf(output)); + EXPECT_EQ(sign, std::signbit(output)); + } + } + + void TestRoundTrip(); + void TestRoundTripFromNaN(); +}; + +template <> +void Float16ConversionTest::TestRoundTrip() { + // Expected values were also manually validated with numpy-1.24.3 + const RoundTripTestCase test_cases[] = { + // +/-0.0f + {F32(0x80000000u), 0b1000000000000000u, -0.0f}, + {F32(0x00000000u), 0b0000000000000000u, +0.0f}, + // 32-bit exp is 102 => 2^-25. Rounding to nearest. + {F32(0xb3000001u), 0b1000000000000001u, -5.96046447754e-8f}, + // 32-bit exp is 102 => 2^-25. Rounding to even. + {F32(0xb3000000u), 0b1000000000000000u, -0.0f}, + // 32-bit exp is 101 => 2^-26. Underflow to zero. + {F32(0xb2800001u), 0b1000000000000000u, -0.0f}, + // 32-bit exp is 108 => 2^-19. + {F32(0xb61a0000u), 0b1000000000100110u, -2.26497650146e-6f}, + // 32-bit exp is 108 => 2^-19. + {F32(0xb61e0000u), 0b1000000000101000u, -2.38418579102e-6f}, + // 32-bit exp is 112 => 2^-15. Rounding to nearest. + {F32(0xb87fa001u), 0b1000001111111111u, -6.09755516052e-5f}, + // 32-bit exp is 112 => 2^-15. Rounds to 16-bit exp of 1 => 2^-14 + {F32(0xb87fe001u), 0b1000010000000000u, -6.103515625e-5f}, + // 32-bit exp is 142 => 2^15. Rounding to nearest. + {F32(0xc7001001u), 0b1111100000000001u, -32800.0f}, + // 32-bit exp is 142 => 2^15. Rounding to even. + {F32(0xc7001000u), 0b1111100000000000u, -32768.0f}, + // 65520.0f rounds to inf + {F32(0x477ff000u), 0b0111110000000000u, Limits::infinity()}, + // 65488.0039062f rounds to 65504.0 (float16 max) + {F32(0x477fd001u), 0b0111101111111111u, 65504.0f}, + // 32-bit exp is 127 => 2^0, rounds to 16-bit exp of 16 => 2^1. + {F32(0xbffff000u), 0b1100000000000000u, -2.0f}, + // Extreme values should safely clamp to +/-inf + {Limits::max(), 0b0111110000000000u, +Limits::infinity()}, + {Limits::lowest(), 0b1111110000000000u, -Limits::infinity()}, + }; + + TestRoundTrip(span(test_cases, std::size(test_cases))); +} + +template <> +void Float16ConversionTest::TestRoundTrip() { + // Expected values were also manually validated with numpy-1.24.3 + const RoundTripTestCase test_cases[] = { + // +/-0.0 + {F64(0x8000000000000000u), 0b1000000000000000u, -0.0}, + {F64(0x0000000000000000u), 0b0000000000000000u, +0.0}, + // 64-bit exp is 998 => 2^-25. Rounding to nearest. + {F64(0xbe60000000000001u), 0b1000000000000001u, -5.9604644775390625e-8}, + // 64-bit exp is 998 => 2^-25. Rounding to even. + {F64(0xbe60000000000000u), 0b1000000000000000u, -0.0}, + // 64-bit exp is 997 => 2^-26. Underflow to zero. + {F64(0xbe50000000000001u), 0b1000000000000000u, -0.0}, + // 64-bit exp is 1004 => 2^-19. + {F64(0xbec3400000000000u), 0b1000000000100110u, -2.2649765014648438e-6}, + // 64-bit exp is 1004 => 2^-19. + {F64(0xbec3c00000000000u), 0b1000000000101000u, -2.3841857910156250e-6}, + // 64-bit exp is 1008 => 2^-15. Rounding to nearest. + {F64(0xbf0ff40000000001u), 0b1000001111111111u, -6.0975551605224609e-5}, + // 64-bit exp is 1008 => 2^-15. Rounds to 16-bit exp of 1 => 2^-14 + {F64(0xbf0ffc0000000001u), 0b1000010000000000u, -6.1035156250000000e-5}, + // 64-bit exp is 1038 => 2^15. Rounding to nearest. + {F64(0xc0e0020000000001u), 0b1111100000000001u, -32800.0}, + // 64-bit exp is 1038 => 2^15. Rounding to even. + {F64(0xc0e0020000000000u), 0b1111100000000000u, -32768.0}, + // 65520.0 rounds to inf + {F64(0x40effe0000000000u), 0b0111110000000000u, Limits::infinity()}, + // 65488.00000000001 rounds to 65504.0 (float16 max) + {F64(0x40effa0000000001u), 0b0111101111111111u, 65504.0}, + // 64-bit exp is 1023 => 2^0, rounds to 16-bit exp of 16 => 2^1. + {F64(0xbffffe0000000000u), 0b1100000000000000u, -2.0}, + // Extreme values should safely clamp to +/-inf + {Limits::max(), 0b0111110000000000u, +Limits::infinity()}, + {Limits::lowest(), 0b1111110000000000u, -Limits::infinity()}, + }; + + TestRoundTrip(span(test_cases, std::size(test_cases))); +} + +template <> +void Float16ConversionTest::TestRoundTripFromNaN() { + const float test_cases[] = { + Limits::quiet_NaN(), F32(0x7f800001u), F32(0xff800001u), F32(0x7fc00000u), + F32(0xffc00000u), F32(0x7fffffffu), F32(0xffffffffu)}; + TestRoundTripFromNaN(span(test_cases, std::size(test_cases))); +} + +template <> +void Float16ConversionTest::TestRoundTripFromNaN() { + const double test_cases[] = {Limits::quiet_NaN(), F64(0x7ff0000000000001u), + F64(0xfff0000000000001u), F64(0x7ff8000000000000u), + F64(0xfff8000000000000u), F64(0x7fffffffffffffffu), + F64(0xffffffffffffffffu)}; + TestRoundTripFromNaN(span(test_cases, std::size(test_cases))); +} + +using NativeFloatTypes = ::testing::Types; + +TYPED_TEST_SUITE(Float16ConversionTest, NativeFloatTypes); + +TYPED_TEST(Float16ConversionTest, RoundTrip) { this->TestRoundTrip(); } +TYPED_TEST(Float16ConversionTest, RoundTripFromNaN) { this->TestRoundTripFromNaN(); } +TYPED_TEST(Float16ConversionTest, RoundTripFromInf) { this->TestRoundTripFromInf(); } + +TEST(Float16Test, ConstexprFunctions) { + constexpr auto a = Float16::FromBits(0xbc00); // -1.0 + constexpr auto b = Float16::FromBits(0x3c00); // +1.0 + + static_assert(a.bits() == 0xbc00); + static_assert(a.signbit() == true); + static_assert(a.is_nan() == false); + static_assert(a.is_infinity() == false); + static_assert(a.is_finite() == true); + static_assert(a.is_zero() == false); + + static_assert((a == b) == false); + static_assert((a != b) == true); + static_assert((a < b) == true); + static_assert((a > b) == false); + static_assert((a <= b) == true); + static_assert((a >= b) == false); + static_assert(-a == +b); + + constexpr auto v = Float16::FromBits(0xffff); + static_assert(v.ToBytes()[0] == 0xff); + static_assert(v.ToLittleEndian()[0] == 0xff); + static_assert(v.ToBigEndian()[0] == 0xff); +} + +TEST(Float16Test, Constructors) { + // Construction from exact bits + ASSERT_EQ(1, Float16::FromBits(1).bits()); + // Construction from floating point (including implicit conversions) + int i = 0; + for (auto f16 : {Float16(1.0f), Float16(1.0), Float16(1)}) { + ARROW_SCOPED_TRACE("i=", i++); + ASSERT_EQ(0x3c00, f16.bits()); + } +} + +TEST(Float16Test, Compare) { + constexpr float f32_inf = Limits::infinity(); + constexpr float f32_nan = Limits::quiet_NaN(); + + const struct { + Float16 f16; + float f32; + } test_values[] = { + {Limits::min(), +6.103515625e-05f}, + {Limits::max(), +65504.0f}, + {Limits::lowest(), -65504.0f}, + {+Limits::infinity(), +f32_inf}, + {-Limits::infinity(), -f32_inf}, + // Multiple (semantically equivalent) NaN representations + {Float16::FromBits(0x7e00), f32_nan}, + {Float16::FromBits(0xfe00), f32_nan}, + {Float16::FromBits(0x7fff), f32_nan}, + {Float16::FromBits(0xffff), f32_nan}, + // Positive/negative zeros + {Float16::FromBits(0x0000), +0.0f}, + {Float16::FromBits(0x8000), -0.0f}, + // Miscellaneous values. In general, they're chosen to test the sign/exponent and + // exponent/mantissa boundaries + {Float16::FromBits(0x101c), +0.00050163269043f}, + {Float16::FromBits(0x901c), -0.00050163269043f}, + {Float16::FromBits(0x101d), +0.000502109527588f}, + {Float16::FromBits(0x901d), -0.000502109527588f}, + {Float16::FromBits(0x121c), +0.00074577331543f}, + {Float16::FromBits(0x921c), -0.00074577331543f}, + {Float16::FromBits(0x141c), +0.00100326538086f}, + {Float16::FromBits(0x941c), -0.00100326538086f}, + {Float16::FromBits(0x501c), +32.875f}, + {Float16::FromBits(0xd01c), -32.875f}, + // A few subnormals for good measure + {Float16::FromBits(0x001c), +1.66893005371e-06f}, + {Float16::FromBits(0x801c), -1.66893005371e-06f}, + {Float16::FromBits(0x021c), +3.21865081787e-05f}, + {Float16::FromBits(0x821c), -3.21865081787e-05f}, + }; + + auto expect_op = [&](std::string op_name, auto op) { + ARROW_SCOPED_TRACE(op_name); + const auto num_values = static_cast(std::size(test_values)); + + // Check all combinations of operands in both directions + for (int i = 0; i < num_values; ++i) { + for (int j = 0; j < num_values; ++j) { + auto [a16, a32] = test_values[i]; + auto [b16, b32] = test_values[j]; + ARROW_SCOPED_TRACE("[", i, ",", j, "] = ", a16, ",", b16); + + // Results for float16 and float32 should be the same + ASSERT_EQ(op(a16, b16), op(a32, b32)); + } + } + }; + + // Verify that our "equivalent" 16/32-bit values actually are + for (const auto& v : test_values) { + if (std::isnan(v.f32)) { + ASSERT_TRUE(std::isnan(v.f16.ToFloat())); + } else { + ASSERT_EQ(v.f32, v.f16.ToFloat()); + } + } + + expect_op("equal", [](auto l, auto r) { return l == r; }); + expect_op("not_equal", [](auto l, auto r) { return l != r; }); + expect_op("less", [](auto l, auto r) { return l < r; }); + expect_op("greater", [](auto l, auto r) { return l > r; }); + expect_op("less_equal", [](auto l, auto r) { return l <= r; }); + expect_op("greater_equal", [](auto l, auto r) { return l >= r; }); +} + +TEST(Float16Test, ToBytes) { + constexpr auto f16 = Float16::FromBits(0xd01c); + std::array bytes; + auto load = [&bytes]() { return SafeLoadAs(bytes.data()); }; + + // Test native-endian + f16.ToBytes(bytes.data()); + ASSERT_EQ(load(), 0xd01c); + bytes = f16.ToBytes(); + ASSERT_EQ(load(), 0xd01c); + +#if ARROW_LITTLE_ENDIAN + constexpr uint16_t expected_le = 0xd01c; + constexpr uint16_t expected_be = 0x1cd0; +#else + constexpr uint16_t expected_le = 0x1cd0; + constexpr uint16_t expected_be = 0xd01c; +#endif + // Test little-endian + f16.ToLittleEndian(bytes.data()); + ASSERT_EQ(load(), expected_le); + bytes = f16.ToLittleEndian(); + ASSERT_EQ(load(), expected_le); + // Test big-endian + f16.ToBigEndian(bytes.data()); + ASSERT_EQ(load(), expected_be); + bytes = f16.ToBigEndian(); + ASSERT_EQ(load(), expected_be); +} + +TEST(Float16Test, FromBytes) { + constexpr uint16_t u16 = 0xd01c; + const auto* data = reinterpret_cast(&u16); + ASSERT_EQ(Float16::FromBytes(data), Float16::FromBits(0xd01c)); +#if ARROW_LITTLE_ENDIAN + ASSERT_EQ(Float16::FromLittleEndian(data), Float16::FromBits(0xd01c)); + ASSERT_EQ(Float16::FromBigEndian(data), Float16::FromBits(0x1cd0)); +#else + ASSERT_EQ(Float16::FromLittleEndian(data), Float16(0x1cd0)); + ASSERT_EQ(Float16::FromBigEndian(data), Float16(0xd01c)); +#endif +} + +} // namespace +} // namespace arrow::util diff --git a/cpp/src/generated/parquet_types.cpp b/cpp/src/generated/parquet_types.cpp index f4e378fd3822a..86188581e0c42 100644 --- a/cpp/src/generated/parquet_types.cpp +++ b/cpp/src/generated/parquet_types.cpp @@ -1288,6 +1288,81 @@ void DateType::printTo(std::ostream& out) const { } +Float16Type::~Float16Type() noexcept { +} + +std::ostream& operator<<(std::ostream& out, const Float16Type& obj) +{ + obj.printTo(out); + return out; +} + + +uint32_t Float16Type::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t Float16Type::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("Float16Type"); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(Float16Type &a, Float16Type &b) { + using ::std::swap; + (void) a; + (void) b; +} + +Float16Type::Float16Type(const Float16Type& other28) noexcept { + (void) other28; +} +Float16Type::Float16Type(Float16Type&& other29) noexcept { + (void) other29; +} +Float16Type& Float16Type::operator=(const Float16Type& other30) noexcept { + (void) other30; + return *this; +} +Float16Type& Float16Type::operator=(Float16Type&& other31) noexcept { + (void) other31; + return *this; +} +void Float16Type::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "Float16Type("; + out << ")"; +} + + NullType::~NullType() noexcept { } @@ -1342,18 +1417,18 @@ void swap(NullType &a, NullType &b) { (void) b; } -NullType::NullType(const NullType& other28) noexcept { - (void) other28; +NullType::NullType(const NullType& other32) noexcept { + (void) other32; } -NullType::NullType(NullType&& other29) noexcept { - (void) other29; +NullType::NullType(NullType&& other33) noexcept { + (void) other33; } -NullType& NullType::operator=(const NullType& other30) noexcept { - (void) other30; +NullType& NullType::operator=(const NullType& other34) noexcept { + (void) other34; return *this; } -NullType& NullType::operator=(NullType&& other31) noexcept { - (void) other31; +NullType& NullType::operator=(NullType&& other35) noexcept { + (void) other35; return *this; } void NullType::printTo(std::ostream& out) const { @@ -1460,22 +1535,22 @@ void swap(DecimalType &a, DecimalType &b) { swap(a.precision, b.precision); } -DecimalType::DecimalType(const DecimalType& other32) noexcept { - scale = other32.scale; - precision = other32.precision; +DecimalType::DecimalType(const DecimalType& other36) noexcept { + scale = other36.scale; + precision = other36.precision; } -DecimalType::DecimalType(DecimalType&& other33) noexcept { - scale = other33.scale; - precision = other33.precision; +DecimalType::DecimalType(DecimalType&& other37) noexcept { + scale = other37.scale; + precision = other37.precision; } -DecimalType& DecimalType::operator=(const DecimalType& other34) noexcept { - scale = other34.scale; - precision = other34.precision; +DecimalType& DecimalType::operator=(const DecimalType& other38) noexcept { + scale = other38.scale; + precision = other38.precision; return *this; } -DecimalType& DecimalType::operator=(DecimalType&& other35) noexcept { - scale = other35.scale; - precision = other35.precision; +DecimalType& DecimalType::operator=(DecimalType&& other39) noexcept { + scale = other39.scale; + precision = other39.precision; return *this; } void DecimalType::printTo(std::ostream& out) const { @@ -1541,18 +1616,18 @@ void swap(MilliSeconds &a, MilliSeconds &b) { (void) b; } -MilliSeconds::MilliSeconds(const MilliSeconds& other36) noexcept { - (void) other36; +MilliSeconds::MilliSeconds(const MilliSeconds& other40) noexcept { + (void) other40; } -MilliSeconds::MilliSeconds(MilliSeconds&& other37) noexcept { - (void) other37; +MilliSeconds::MilliSeconds(MilliSeconds&& other41) noexcept { + (void) other41; } -MilliSeconds& MilliSeconds::operator=(const MilliSeconds& other38) noexcept { - (void) other38; +MilliSeconds& MilliSeconds::operator=(const MilliSeconds& other42) noexcept { + (void) other42; return *this; } -MilliSeconds& MilliSeconds::operator=(MilliSeconds&& other39) noexcept { - (void) other39; +MilliSeconds& MilliSeconds::operator=(MilliSeconds&& other43) noexcept { + (void) other43; return *this; } void MilliSeconds::printTo(std::ostream& out) const { @@ -1616,18 +1691,18 @@ void swap(MicroSeconds &a, MicroSeconds &b) { (void) b; } -MicroSeconds::MicroSeconds(const MicroSeconds& other40) noexcept { - (void) other40; +MicroSeconds::MicroSeconds(const MicroSeconds& other44) noexcept { + (void) other44; } -MicroSeconds::MicroSeconds(MicroSeconds&& other41) noexcept { - (void) other41; +MicroSeconds::MicroSeconds(MicroSeconds&& other45) noexcept { + (void) other45; } -MicroSeconds& MicroSeconds::operator=(const MicroSeconds& other42) noexcept { - (void) other42; +MicroSeconds& MicroSeconds::operator=(const MicroSeconds& other46) noexcept { + (void) other46; return *this; } -MicroSeconds& MicroSeconds::operator=(MicroSeconds&& other43) noexcept { - (void) other43; +MicroSeconds& MicroSeconds::operator=(MicroSeconds&& other47) noexcept { + (void) other47; return *this; } void MicroSeconds::printTo(std::ostream& out) const { @@ -1691,18 +1766,18 @@ void swap(NanoSeconds &a, NanoSeconds &b) { (void) b; } -NanoSeconds::NanoSeconds(const NanoSeconds& other44) noexcept { - (void) other44; +NanoSeconds::NanoSeconds(const NanoSeconds& other48) noexcept { + (void) other48; } -NanoSeconds::NanoSeconds(NanoSeconds&& other45) noexcept { - (void) other45; +NanoSeconds::NanoSeconds(NanoSeconds&& other49) noexcept { + (void) other49; } -NanoSeconds& NanoSeconds::operator=(const NanoSeconds& other46) noexcept { - (void) other46; +NanoSeconds& NanoSeconds::operator=(const NanoSeconds& other50) noexcept { + (void) other50; return *this; } -NanoSeconds& NanoSeconds::operator=(NanoSeconds&& other47) noexcept { - (void) other47; +NanoSeconds& NanoSeconds::operator=(NanoSeconds&& other51) noexcept { + (void) other51; return *this; } void NanoSeconds::printTo(std::ostream& out) const { @@ -1827,30 +1902,30 @@ void swap(TimeUnit &a, TimeUnit &b) { swap(a.__isset, b.__isset); } -TimeUnit::TimeUnit(const TimeUnit& other48) noexcept { - MILLIS = other48.MILLIS; - MICROS = other48.MICROS; - NANOS = other48.NANOS; - __isset = other48.__isset; +TimeUnit::TimeUnit(const TimeUnit& other52) noexcept { + MILLIS = other52.MILLIS; + MICROS = other52.MICROS; + NANOS = other52.NANOS; + __isset = other52.__isset; } -TimeUnit::TimeUnit(TimeUnit&& other49) noexcept { - MILLIS = std::move(other49.MILLIS); - MICROS = std::move(other49.MICROS); - NANOS = std::move(other49.NANOS); - __isset = other49.__isset; +TimeUnit::TimeUnit(TimeUnit&& other53) noexcept { + MILLIS = std::move(other53.MILLIS); + MICROS = std::move(other53.MICROS); + NANOS = std::move(other53.NANOS); + __isset = other53.__isset; } -TimeUnit& TimeUnit::operator=(const TimeUnit& other50) noexcept { - MILLIS = other50.MILLIS; - MICROS = other50.MICROS; - NANOS = other50.NANOS; - __isset = other50.__isset; +TimeUnit& TimeUnit::operator=(const TimeUnit& other54) noexcept { + MILLIS = other54.MILLIS; + MICROS = other54.MICROS; + NANOS = other54.NANOS; + __isset = other54.__isset; return *this; } -TimeUnit& TimeUnit::operator=(TimeUnit&& other51) noexcept { - MILLIS = std::move(other51.MILLIS); - MICROS = std::move(other51.MICROS); - NANOS = std::move(other51.NANOS); - __isset = other51.__isset; +TimeUnit& TimeUnit::operator=(TimeUnit&& other55) noexcept { + MILLIS = std::move(other55.MILLIS); + MICROS = std::move(other55.MICROS); + NANOS = std::move(other55.NANOS); + __isset = other55.__isset; return *this; } void TimeUnit::printTo(std::ostream& out) const { @@ -1960,22 +2035,22 @@ void swap(TimestampType &a, TimestampType &b) { swap(a.unit, b.unit); } -TimestampType::TimestampType(const TimestampType& other52) noexcept { - isAdjustedToUTC = other52.isAdjustedToUTC; - unit = other52.unit; +TimestampType::TimestampType(const TimestampType& other56) noexcept { + isAdjustedToUTC = other56.isAdjustedToUTC; + unit = other56.unit; } -TimestampType::TimestampType(TimestampType&& other53) noexcept { - isAdjustedToUTC = other53.isAdjustedToUTC; - unit = std::move(other53.unit); +TimestampType::TimestampType(TimestampType&& other57) noexcept { + isAdjustedToUTC = other57.isAdjustedToUTC; + unit = std::move(other57.unit); } -TimestampType& TimestampType::operator=(const TimestampType& other54) noexcept { - isAdjustedToUTC = other54.isAdjustedToUTC; - unit = other54.unit; +TimestampType& TimestampType::operator=(const TimestampType& other58) noexcept { + isAdjustedToUTC = other58.isAdjustedToUTC; + unit = other58.unit; return *this; } -TimestampType& TimestampType::operator=(TimestampType&& other55) noexcept { - isAdjustedToUTC = other55.isAdjustedToUTC; - unit = std::move(other55.unit); +TimestampType& TimestampType::operator=(TimestampType&& other59) noexcept { + isAdjustedToUTC = other59.isAdjustedToUTC; + unit = std::move(other59.unit); return *this; } void TimestampType::printTo(std::ostream& out) const { @@ -2084,22 +2159,22 @@ void swap(TimeType &a, TimeType &b) { swap(a.unit, b.unit); } -TimeType::TimeType(const TimeType& other56) noexcept { - isAdjustedToUTC = other56.isAdjustedToUTC; - unit = other56.unit; +TimeType::TimeType(const TimeType& other60) noexcept { + isAdjustedToUTC = other60.isAdjustedToUTC; + unit = other60.unit; } -TimeType::TimeType(TimeType&& other57) noexcept { - isAdjustedToUTC = other57.isAdjustedToUTC; - unit = std::move(other57.unit); +TimeType::TimeType(TimeType&& other61) noexcept { + isAdjustedToUTC = other61.isAdjustedToUTC; + unit = std::move(other61.unit); } -TimeType& TimeType::operator=(const TimeType& other58) noexcept { - isAdjustedToUTC = other58.isAdjustedToUTC; - unit = other58.unit; +TimeType& TimeType::operator=(const TimeType& other62) noexcept { + isAdjustedToUTC = other62.isAdjustedToUTC; + unit = other62.unit; return *this; } -TimeType& TimeType::operator=(TimeType&& other59) noexcept { - isAdjustedToUTC = other59.isAdjustedToUTC; - unit = std::move(other59.unit); +TimeType& TimeType::operator=(TimeType&& other63) noexcept { + isAdjustedToUTC = other63.isAdjustedToUTC; + unit = std::move(other63.unit); return *this; } void TimeType::printTo(std::ostream& out) const { @@ -2208,22 +2283,22 @@ void swap(IntType &a, IntType &b) { swap(a.isSigned, b.isSigned); } -IntType::IntType(const IntType& other60) noexcept { - bitWidth = other60.bitWidth; - isSigned = other60.isSigned; +IntType::IntType(const IntType& other64) noexcept { + bitWidth = other64.bitWidth; + isSigned = other64.isSigned; } -IntType::IntType(IntType&& other61) noexcept { - bitWidth = other61.bitWidth; - isSigned = other61.isSigned; +IntType::IntType(IntType&& other65) noexcept { + bitWidth = other65.bitWidth; + isSigned = other65.isSigned; } -IntType& IntType::operator=(const IntType& other62) noexcept { - bitWidth = other62.bitWidth; - isSigned = other62.isSigned; +IntType& IntType::operator=(const IntType& other66) noexcept { + bitWidth = other66.bitWidth; + isSigned = other66.isSigned; return *this; } -IntType& IntType::operator=(IntType&& other63) noexcept { - bitWidth = other63.bitWidth; - isSigned = other63.isSigned; +IntType& IntType::operator=(IntType&& other67) noexcept { + bitWidth = other67.bitWidth; + isSigned = other67.isSigned; return *this; } void IntType::printTo(std::ostream& out) const { @@ -2289,18 +2364,18 @@ void swap(JsonType &a, JsonType &b) { (void) b; } -JsonType::JsonType(const JsonType& other64) noexcept { - (void) other64; +JsonType::JsonType(const JsonType& other68) noexcept { + (void) other68; } -JsonType::JsonType(JsonType&& other65) noexcept { - (void) other65; +JsonType::JsonType(JsonType&& other69) noexcept { + (void) other69; } -JsonType& JsonType::operator=(const JsonType& other66) noexcept { - (void) other66; +JsonType& JsonType::operator=(const JsonType& other70) noexcept { + (void) other70; return *this; } -JsonType& JsonType::operator=(JsonType&& other67) noexcept { - (void) other67; +JsonType& JsonType::operator=(JsonType&& other71) noexcept { + (void) other71; return *this; } void JsonType::printTo(std::ostream& out) const { @@ -2364,18 +2439,18 @@ void swap(BsonType &a, BsonType &b) { (void) b; } -BsonType::BsonType(const BsonType& other68) noexcept { - (void) other68; +BsonType::BsonType(const BsonType& other72) noexcept { + (void) other72; } -BsonType::BsonType(BsonType&& other69) noexcept { - (void) other69; +BsonType::BsonType(BsonType&& other73) noexcept { + (void) other73; } -BsonType& BsonType::operator=(const BsonType& other70) noexcept { - (void) other70; +BsonType& BsonType::operator=(const BsonType& other74) noexcept { + (void) other74; return *this; } -BsonType& BsonType::operator=(BsonType&& other71) noexcept { - (void) other71; +BsonType& BsonType::operator=(BsonType&& other75) noexcept { + (void) other75; return *this; } void BsonType::printTo(std::ostream& out) const { @@ -2453,6 +2528,11 @@ void LogicalType::__set_UUID(const UUIDType& val) { this->UUID = val; __isset.UUID = true; } + +void LogicalType::__set_FLOAT16(const Float16Type& val) { + this->FLOAT16 = val; +__isset.FLOAT16 = true; +} std::ostream& operator<<(std::ostream& out, const LogicalType& obj) { obj.printTo(out); @@ -2585,6 +2665,14 @@ uint32_t LogicalType::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; + case 15: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->FLOAT16.read(iprot); + this->__isset.FLOAT16 = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -2667,6 +2755,11 @@ uint32_t LogicalType::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += this->UUID.write(oprot); xfer += oprot->writeFieldEnd(); } + if (this->__isset.FLOAT16) { + xfer += oprot->writeFieldBegin("FLOAT16", ::apache::thrift::protocol::T_STRUCT, 15); + xfer += this->FLOAT16.write(oprot); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -2687,73 +2780,78 @@ void swap(LogicalType &a, LogicalType &b) { swap(a.JSON, b.JSON); swap(a.BSON, b.BSON); swap(a.UUID, b.UUID); + swap(a.FLOAT16, b.FLOAT16); swap(a.__isset, b.__isset); } -LogicalType::LogicalType(const LogicalType& other72) noexcept { - STRING = other72.STRING; - MAP = other72.MAP; - LIST = other72.LIST; - ENUM = other72.ENUM; - DECIMAL = other72.DECIMAL; - DATE = other72.DATE; - TIME = other72.TIME; - TIMESTAMP = other72.TIMESTAMP; - INTEGER = other72.INTEGER; - UNKNOWN = other72.UNKNOWN; - JSON = other72.JSON; - BSON = other72.BSON; - UUID = other72.UUID; - __isset = other72.__isset; -} -LogicalType::LogicalType(LogicalType&& other73) noexcept { - STRING = std::move(other73.STRING); - MAP = std::move(other73.MAP); - LIST = std::move(other73.LIST); - ENUM = std::move(other73.ENUM); - DECIMAL = std::move(other73.DECIMAL); - DATE = std::move(other73.DATE); - TIME = std::move(other73.TIME); - TIMESTAMP = std::move(other73.TIMESTAMP); - INTEGER = std::move(other73.INTEGER); - UNKNOWN = std::move(other73.UNKNOWN); - JSON = std::move(other73.JSON); - BSON = std::move(other73.BSON); - UUID = std::move(other73.UUID); - __isset = other73.__isset; -} -LogicalType& LogicalType::operator=(const LogicalType& other74) noexcept { - STRING = other74.STRING; - MAP = other74.MAP; - LIST = other74.LIST; - ENUM = other74.ENUM; - DECIMAL = other74.DECIMAL; - DATE = other74.DATE; - TIME = other74.TIME; - TIMESTAMP = other74.TIMESTAMP; - INTEGER = other74.INTEGER; - UNKNOWN = other74.UNKNOWN; - JSON = other74.JSON; - BSON = other74.BSON; - UUID = other74.UUID; - __isset = other74.__isset; +LogicalType::LogicalType(const LogicalType& other76) noexcept { + STRING = other76.STRING; + MAP = other76.MAP; + LIST = other76.LIST; + ENUM = other76.ENUM; + DECIMAL = other76.DECIMAL; + DATE = other76.DATE; + TIME = other76.TIME; + TIMESTAMP = other76.TIMESTAMP; + INTEGER = other76.INTEGER; + UNKNOWN = other76.UNKNOWN; + JSON = other76.JSON; + BSON = other76.BSON; + UUID = other76.UUID; + FLOAT16 = other76.FLOAT16; + __isset = other76.__isset; +} +LogicalType::LogicalType(LogicalType&& other77) noexcept { + STRING = std::move(other77.STRING); + MAP = std::move(other77.MAP); + LIST = std::move(other77.LIST); + ENUM = std::move(other77.ENUM); + DECIMAL = std::move(other77.DECIMAL); + DATE = std::move(other77.DATE); + TIME = std::move(other77.TIME); + TIMESTAMP = std::move(other77.TIMESTAMP); + INTEGER = std::move(other77.INTEGER); + UNKNOWN = std::move(other77.UNKNOWN); + JSON = std::move(other77.JSON); + BSON = std::move(other77.BSON); + UUID = std::move(other77.UUID); + FLOAT16 = std::move(other77.FLOAT16); + __isset = other77.__isset; +} +LogicalType& LogicalType::operator=(const LogicalType& other78) noexcept { + STRING = other78.STRING; + MAP = other78.MAP; + LIST = other78.LIST; + ENUM = other78.ENUM; + DECIMAL = other78.DECIMAL; + DATE = other78.DATE; + TIME = other78.TIME; + TIMESTAMP = other78.TIMESTAMP; + INTEGER = other78.INTEGER; + UNKNOWN = other78.UNKNOWN; + JSON = other78.JSON; + BSON = other78.BSON; + UUID = other78.UUID; + FLOAT16 = other78.FLOAT16; + __isset = other78.__isset; return *this; } -LogicalType& LogicalType::operator=(LogicalType&& other75) noexcept { - STRING = std::move(other75.STRING); - MAP = std::move(other75.MAP); - LIST = std::move(other75.LIST); - ENUM = std::move(other75.ENUM); - DECIMAL = std::move(other75.DECIMAL); - DATE = std::move(other75.DATE); - TIME = std::move(other75.TIME); - TIMESTAMP = std::move(other75.TIMESTAMP); - INTEGER = std::move(other75.INTEGER); - UNKNOWN = std::move(other75.UNKNOWN); - JSON = std::move(other75.JSON); - BSON = std::move(other75.BSON); - UUID = std::move(other75.UUID); - __isset = other75.__isset; +LogicalType& LogicalType::operator=(LogicalType&& other79) noexcept { + STRING = std::move(other79.STRING); + MAP = std::move(other79.MAP); + LIST = std::move(other79.LIST); + ENUM = std::move(other79.ENUM); + DECIMAL = std::move(other79.DECIMAL); + DATE = std::move(other79.DATE); + TIME = std::move(other79.TIME); + TIMESTAMP = std::move(other79.TIMESTAMP); + INTEGER = std::move(other79.INTEGER); + UNKNOWN = std::move(other79.UNKNOWN); + JSON = std::move(other79.JSON); + BSON = std::move(other79.BSON); + UUID = std::move(other79.UUID); + FLOAT16 = std::move(other79.FLOAT16); + __isset = other79.__isset; return *this; } void LogicalType::printTo(std::ostream& out) const { @@ -2772,6 +2870,7 @@ void LogicalType::printTo(std::ostream& out) const { out << ", " << "JSON="; (__isset.JSON ? (out << to_string(JSON)) : (out << "")); out << ", " << "BSON="; (__isset.BSON ? (out << to_string(BSON)) : (out << "")); out << ", " << "UUID="; (__isset.UUID ? (out << to_string(UUID)) : (out << "")); + out << ", " << "FLOAT16="; (__isset.FLOAT16 ? (out << to_string(FLOAT16)) : (out << "")); out << ")"; } @@ -2859,9 +2958,9 @@ uint32_t SchemaElement::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast76; - xfer += iprot->readI32(ecast76); - this->type = static_cast(ecast76); + int32_t ecast80; + xfer += iprot->readI32(ecast80); + this->type = static_cast(ecast80); this->__isset.type = true; } else { xfer += iprot->skip(ftype); @@ -2877,9 +2976,9 @@ uint32_t SchemaElement::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 3: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast77; - xfer += iprot->readI32(ecast77); - this->repetition_type = static_cast(ecast77); + int32_t ecast81; + xfer += iprot->readI32(ecast81); + this->repetition_type = static_cast(ecast81); this->__isset.repetition_type = true; } else { xfer += iprot->skip(ftype); @@ -2903,9 +3002,9 @@ uint32_t SchemaElement::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 6: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast78; - xfer += iprot->readI32(ecast78); - this->converted_type = static_cast(ecast78); + int32_t ecast82; + xfer += iprot->readI32(ecast82); + this->converted_type = static_cast(ecast82); this->__isset.converted_type = true; } else { xfer += iprot->skip(ftype); @@ -3031,58 +3130,58 @@ void swap(SchemaElement &a, SchemaElement &b) { swap(a.__isset, b.__isset); } -SchemaElement::SchemaElement(const SchemaElement& other79) { - type = other79.type; - type_length = other79.type_length; - repetition_type = other79.repetition_type; - name = other79.name; - num_children = other79.num_children; - converted_type = other79.converted_type; - scale = other79.scale; - precision = other79.precision; - field_id = other79.field_id; - logicalType = other79.logicalType; - __isset = other79.__isset; -} -SchemaElement::SchemaElement(SchemaElement&& other80) noexcept { - type = other80.type; - type_length = other80.type_length; - repetition_type = other80.repetition_type; - name = std::move(other80.name); - num_children = other80.num_children; - converted_type = other80.converted_type; - scale = other80.scale; - precision = other80.precision; - field_id = other80.field_id; - logicalType = std::move(other80.logicalType); - __isset = other80.__isset; -} -SchemaElement& SchemaElement::operator=(const SchemaElement& other81) { - type = other81.type; - type_length = other81.type_length; - repetition_type = other81.repetition_type; - name = other81.name; - num_children = other81.num_children; - converted_type = other81.converted_type; - scale = other81.scale; - precision = other81.precision; - field_id = other81.field_id; - logicalType = other81.logicalType; - __isset = other81.__isset; +SchemaElement::SchemaElement(const SchemaElement& other83) { + type = other83.type; + type_length = other83.type_length; + repetition_type = other83.repetition_type; + name = other83.name; + num_children = other83.num_children; + converted_type = other83.converted_type; + scale = other83.scale; + precision = other83.precision; + field_id = other83.field_id; + logicalType = other83.logicalType; + __isset = other83.__isset; +} +SchemaElement::SchemaElement(SchemaElement&& other84) noexcept { + type = other84.type; + type_length = other84.type_length; + repetition_type = other84.repetition_type; + name = std::move(other84.name); + num_children = other84.num_children; + converted_type = other84.converted_type; + scale = other84.scale; + precision = other84.precision; + field_id = other84.field_id; + logicalType = std::move(other84.logicalType); + __isset = other84.__isset; +} +SchemaElement& SchemaElement::operator=(const SchemaElement& other85) { + type = other85.type; + type_length = other85.type_length; + repetition_type = other85.repetition_type; + name = other85.name; + num_children = other85.num_children; + converted_type = other85.converted_type; + scale = other85.scale; + precision = other85.precision; + field_id = other85.field_id; + logicalType = other85.logicalType; + __isset = other85.__isset; return *this; } -SchemaElement& SchemaElement::operator=(SchemaElement&& other82) noexcept { - type = other82.type; - type_length = other82.type_length; - repetition_type = other82.repetition_type; - name = std::move(other82.name); - num_children = other82.num_children; - converted_type = other82.converted_type; - scale = other82.scale; - precision = other82.precision; - field_id = other82.field_id; - logicalType = std::move(other82.logicalType); - __isset = other82.__isset; +SchemaElement& SchemaElement::operator=(SchemaElement&& other86) noexcept { + type = other86.type; + type_length = other86.type_length; + repetition_type = other86.repetition_type; + name = std::move(other86.name); + num_children = other86.num_children; + converted_type = other86.converted_type; + scale = other86.scale; + precision = other86.precision; + field_id = other86.field_id; + logicalType = std::move(other86.logicalType); + __isset = other86.__isset; return *this; } void SchemaElement::printTo(std::ostream& out) const { @@ -3168,9 +3267,9 @@ uint32_t DataPageHeader::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast83; - xfer += iprot->readI32(ecast83); - this->encoding = static_cast(ecast83); + int32_t ecast87; + xfer += iprot->readI32(ecast87); + this->encoding = static_cast(ecast87); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -3178,9 +3277,9 @@ uint32_t DataPageHeader::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 3: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast84; - xfer += iprot->readI32(ecast84); - this->definition_level_encoding = static_cast(ecast84); + int32_t ecast88; + xfer += iprot->readI32(ecast88); + this->definition_level_encoding = static_cast(ecast88); isset_definition_level_encoding = true; } else { xfer += iprot->skip(ftype); @@ -3188,9 +3287,9 @@ uint32_t DataPageHeader::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast85; - xfer += iprot->readI32(ecast85); - this->repetition_level_encoding = static_cast(ecast85); + int32_t ecast89; + xfer += iprot->readI32(ecast89); + this->repetition_level_encoding = static_cast(ecast89); isset_repetition_level_encoding = true; } else { xfer += iprot->skip(ftype); @@ -3265,38 +3364,38 @@ void swap(DataPageHeader &a, DataPageHeader &b) { swap(a.__isset, b.__isset); } -DataPageHeader::DataPageHeader(const DataPageHeader& other86) { - num_values = other86.num_values; - encoding = other86.encoding; - definition_level_encoding = other86.definition_level_encoding; - repetition_level_encoding = other86.repetition_level_encoding; - statistics = other86.statistics; - __isset = other86.__isset; -} -DataPageHeader::DataPageHeader(DataPageHeader&& other87) noexcept { - num_values = other87.num_values; - encoding = other87.encoding; - definition_level_encoding = other87.definition_level_encoding; - repetition_level_encoding = other87.repetition_level_encoding; - statistics = std::move(other87.statistics); - __isset = other87.__isset; -} -DataPageHeader& DataPageHeader::operator=(const DataPageHeader& other88) { - num_values = other88.num_values; - encoding = other88.encoding; - definition_level_encoding = other88.definition_level_encoding; - repetition_level_encoding = other88.repetition_level_encoding; - statistics = other88.statistics; - __isset = other88.__isset; +DataPageHeader::DataPageHeader(const DataPageHeader& other90) { + num_values = other90.num_values; + encoding = other90.encoding; + definition_level_encoding = other90.definition_level_encoding; + repetition_level_encoding = other90.repetition_level_encoding; + statistics = other90.statistics; + __isset = other90.__isset; +} +DataPageHeader::DataPageHeader(DataPageHeader&& other91) noexcept { + num_values = other91.num_values; + encoding = other91.encoding; + definition_level_encoding = other91.definition_level_encoding; + repetition_level_encoding = other91.repetition_level_encoding; + statistics = std::move(other91.statistics); + __isset = other91.__isset; +} +DataPageHeader& DataPageHeader::operator=(const DataPageHeader& other92) { + num_values = other92.num_values; + encoding = other92.encoding; + definition_level_encoding = other92.definition_level_encoding; + repetition_level_encoding = other92.repetition_level_encoding; + statistics = other92.statistics; + __isset = other92.__isset; return *this; } -DataPageHeader& DataPageHeader::operator=(DataPageHeader&& other89) noexcept { - num_values = other89.num_values; - encoding = other89.encoding; - definition_level_encoding = other89.definition_level_encoding; - repetition_level_encoding = other89.repetition_level_encoding; - statistics = std::move(other89.statistics); - __isset = other89.__isset; +DataPageHeader& DataPageHeader::operator=(DataPageHeader&& other93) noexcept { + num_values = other93.num_values; + encoding = other93.encoding; + definition_level_encoding = other93.definition_level_encoding; + repetition_level_encoding = other93.repetition_level_encoding; + statistics = std::move(other93.statistics); + __isset = other93.__isset; return *this; } void DataPageHeader::printTo(std::ostream& out) const { @@ -3365,18 +3464,18 @@ void swap(IndexPageHeader &a, IndexPageHeader &b) { (void) b; } -IndexPageHeader::IndexPageHeader(const IndexPageHeader& other90) noexcept { - (void) other90; +IndexPageHeader::IndexPageHeader(const IndexPageHeader& other94) noexcept { + (void) other94; } -IndexPageHeader::IndexPageHeader(IndexPageHeader&& other91) noexcept { - (void) other91; +IndexPageHeader::IndexPageHeader(IndexPageHeader&& other95) noexcept { + (void) other95; } -IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other92) noexcept { - (void) other92; +IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other96) noexcept { + (void) other96; return *this; } -IndexPageHeader& IndexPageHeader::operator=(IndexPageHeader&& other93) noexcept { - (void) other93; +IndexPageHeader& IndexPageHeader::operator=(IndexPageHeader&& other97) noexcept { + (void) other97; return *this; } void IndexPageHeader::printTo(std::ostream& out) const { @@ -3442,9 +3541,9 @@ uint32_t DictionaryPageHeader::read(::apache::thrift::protocol::TProtocol* iprot break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast94; - xfer += iprot->readI32(ecast94); - this->encoding = static_cast(ecast94); + int32_t ecast98; + xfer += iprot->readI32(ecast98); + this->encoding = static_cast(ecast98); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -3505,30 +3604,30 @@ void swap(DictionaryPageHeader &a, DictionaryPageHeader &b) { swap(a.__isset, b.__isset); } -DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader& other95) noexcept { - num_values = other95.num_values; - encoding = other95.encoding; - is_sorted = other95.is_sorted; - __isset = other95.__isset; +DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader& other99) noexcept { + num_values = other99.num_values; + encoding = other99.encoding; + is_sorted = other99.is_sorted; + __isset = other99.__isset; } -DictionaryPageHeader::DictionaryPageHeader(DictionaryPageHeader&& other96) noexcept { - num_values = other96.num_values; - encoding = other96.encoding; - is_sorted = other96.is_sorted; - __isset = other96.__isset; +DictionaryPageHeader::DictionaryPageHeader(DictionaryPageHeader&& other100) noexcept { + num_values = other100.num_values; + encoding = other100.encoding; + is_sorted = other100.is_sorted; + __isset = other100.__isset; } -DictionaryPageHeader& DictionaryPageHeader::operator=(const DictionaryPageHeader& other97) noexcept { - num_values = other97.num_values; - encoding = other97.encoding; - is_sorted = other97.is_sorted; - __isset = other97.__isset; +DictionaryPageHeader& DictionaryPageHeader::operator=(const DictionaryPageHeader& other101) noexcept { + num_values = other101.num_values; + encoding = other101.encoding; + is_sorted = other101.is_sorted; + __isset = other101.__isset; return *this; } -DictionaryPageHeader& DictionaryPageHeader::operator=(DictionaryPageHeader&& other98) noexcept { - num_values = other98.num_values; - encoding = other98.encoding; - is_sorted = other98.is_sorted; - __isset = other98.__isset; +DictionaryPageHeader& DictionaryPageHeader::operator=(DictionaryPageHeader&& other102) noexcept { + num_values = other102.num_values; + encoding = other102.encoding; + is_sorted = other102.is_sorted; + __isset = other102.__isset; return *this; } void DictionaryPageHeader::printTo(std::ostream& out) const { @@ -3638,9 +3737,9 @@ uint32_t DataPageHeaderV2::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast99; - xfer += iprot->readI32(ecast99); - this->encoding = static_cast(ecast99); + int32_t ecast103; + xfer += iprot->readI32(ecast103); + this->encoding = static_cast(ecast103); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -3759,50 +3858,50 @@ void swap(DataPageHeaderV2 &a, DataPageHeaderV2 &b) { swap(a.__isset, b.__isset); } -DataPageHeaderV2::DataPageHeaderV2(const DataPageHeaderV2& other100) { - num_values = other100.num_values; - num_nulls = other100.num_nulls; - num_rows = other100.num_rows; - encoding = other100.encoding; - definition_levels_byte_length = other100.definition_levels_byte_length; - repetition_levels_byte_length = other100.repetition_levels_byte_length; - is_compressed = other100.is_compressed; - statistics = other100.statistics; - __isset = other100.__isset; -} -DataPageHeaderV2::DataPageHeaderV2(DataPageHeaderV2&& other101) noexcept { - num_values = other101.num_values; - num_nulls = other101.num_nulls; - num_rows = other101.num_rows; - encoding = other101.encoding; - definition_levels_byte_length = other101.definition_levels_byte_length; - repetition_levels_byte_length = other101.repetition_levels_byte_length; - is_compressed = other101.is_compressed; - statistics = std::move(other101.statistics); - __isset = other101.__isset; -} -DataPageHeaderV2& DataPageHeaderV2::operator=(const DataPageHeaderV2& other102) { - num_values = other102.num_values; - num_nulls = other102.num_nulls; - num_rows = other102.num_rows; - encoding = other102.encoding; - definition_levels_byte_length = other102.definition_levels_byte_length; - repetition_levels_byte_length = other102.repetition_levels_byte_length; - is_compressed = other102.is_compressed; - statistics = other102.statistics; - __isset = other102.__isset; +DataPageHeaderV2::DataPageHeaderV2(const DataPageHeaderV2& other104) { + num_values = other104.num_values; + num_nulls = other104.num_nulls; + num_rows = other104.num_rows; + encoding = other104.encoding; + definition_levels_byte_length = other104.definition_levels_byte_length; + repetition_levels_byte_length = other104.repetition_levels_byte_length; + is_compressed = other104.is_compressed; + statistics = other104.statistics; + __isset = other104.__isset; +} +DataPageHeaderV2::DataPageHeaderV2(DataPageHeaderV2&& other105) noexcept { + num_values = other105.num_values; + num_nulls = other105.num_nulls; + num_rows = other105.num_rows; + encoding = other105.encoding; + definition_levels_byte_length = other105.definition_levels_byte_length; + repetition_levels_byte_length = other105.repetition_levels_byte_length; + is_compressed = other105.is_compressed; + statistics = std::move(other105.statistics); + __isset = other105.__isset; +} +DataPageHeaderV2& DataPageHeaderV2::operator=(const DataPageHeaderV2& other106) { + num_values = other106.num_values; + num_nulls = other106.num_nulls; + num_rows = other106.num_rows; + encoding = other106.encoding; + definition_levels_byte_length = other106.definition_levels_byte_length; + repetition_levels_byte_length = other106.repetition_levels_byte_length; + is_compressed = other106.is_compressed; + statistics = other106.statistics; + __isset = other106.__isset; return *this; } -DataPageHeaderV2& DataPageHeaderV2::operator=(DataPageHeaderV2&& other103) noexcept { - num_values = other103.num_values; - num_nulls = other103.num_nulls; - num_rows = other103.num_rows; - encoding = other103.encoding; - definition_levels_byte_length = other103.definition_levels_byte_length; - repetition_levels_byte_length = other103.repetition_levels_byte_length; - is_compressed = other103.is_compressed; - statistics = std::move(other103.statistics); - __isset = other103.__isset; +DataPageHeaderV2& DataPageHeaderV2::operator=(DataPageHeaderV2&& other107) noexcept { + num_values = other107.num_values; + num_nulls = other107.num_nulls; + num_rows = other107.num_rows; + encoding = other107.encoding; + definition_levels_byte_length = other107.definition_levels_byte_length; + repetition_levels_byte_length = other107.repetition_levels_byte_length; + is_compressed = other107.is_compressed; + statistics = std::move(other107.statistics); + __isset = other107.__isset; return *this; } void DataPageHeaderV2::printTo(std::ostream& out) const { @@ -3874,18 +3973,18 @@ void swap(SplitBlockAlgorithm &a, SplitBlockAlgorithm &b) { (void) b; } -SplitBlockAlgorithm::SplitBlockAlgorithm(const SplitBlockAlgorithm& other104) noexcept { - (void) other104; +SplitBlockAlgorithm::SplitBlockAlgorithm(const SplitBlockAlgorithm& other108) noexcept { + (void) other108; } -SplitBlockAlgorithm::SplitBlockAlgorithm(SplitBlockAlgorithm&& other105) noexcept { - (void) other105; +SplitBlockAlgorithm::SplitBlockAlgorithm(SplitBlockAlgorithm&& other109) noexcept { + (void) other109; } -SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(const SplitBlockAlgorithm& other106) noexcept { - (void) other106; +SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(const SplitBlockAlgorithm& other110) noexcept { + (void) other110; return *this; } -SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(SplitBlockAlgorithm&& other107) noexcept { - (void) other107; +SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(SplitBlockAlgorithm&& other111) noexcept { + (void) other111; return *this; } void SplitBlockAlgorithm::printTo(std::ostream& out) const { @@ -3972,22 +4071,22 @@ void swap(BloomFilterAlgorithm &a, BloomFilterAlgorithm &b) { swap(a.__isset, b.__isset); } -BloomFilterAlgorithm::BloomFilterAlgorithm(const BloomFilterAlgorithm& other108) noexcept { - BLOCK = other108.BLOCK; - __isset = other108.__isset; +BloomFilterAlgorithm::BloomFilterAlgorithm(const BloomFilterAlgorithm& other112) noexcept { + BLOCK = other112.BLOCK; + __isset = other112.__isset; } -BloomFilterAlgorithm::BloomFilterAlgorithm(BloomFilterAlgorithm&& other109) noexcept { - BLOCK = std::move(other109.BLOCK); - __isset = other109.__isset; +BloomFilterAlgorithm::BloomFilterAlgorithm(BloomFilterAlgorithm&& other113) noexcept { + BLOCK = std::move(other113.BLOCK); + __isset = other113.__isset; } -BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(const BloomFilterAlgorithm& other110) noexcept { - BLOCK = other110.BLOCK; - __isset = other110.__isset; +BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(const BloomFilterAlgorithm& other114) noexcept { + BLOCK = other114.BLOCK; + __isset = other114.__isset; return *this; } -BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(BloomFilterAlgorithm&& other111) noexcept { - BLOCK = std::move(other111.BLOCK); - __isset = other111.__isset; +BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(BloomFilterAlgorithm&& other115) noexcept { + BLOCK = std::move(other115.BLOCK); + __isset = other115.__isset; return *this; } void BloomFilterAlgorithm::printTo(std::ostream& out) const { @@ -4052,18 +4151,18 @@ void swap(XxHash &a, XxHash &b) { (void) b; } -XxHash::XxHash(const XxHash& other112) noexcept { - (void) other112; +XxHash::XxHash(const XxHash& other116) noexcept { + (void) other116; } -XxHash::XxHash(XxHash&& other113) noexcept { - (void) other113; +XxHash::XxHash(XxHash&& other117) noexcept { + (void) other117; } -XxHash& XxHash::operator=(const XxHash& other114) noexcept { - (void) other114; +XxHash& XxHash::operator=(const XxHash& other118) noexcept { + (void) other118; return *this; } -XxHash& XxHash::operator=(XxHash&& other115) noexcept { - (void) other115; +XxHash& XxHash::operator=(XxHash&& other119) noexcept { + (void) other119; return *this; } void XxHash::printTo(std::ostream& out) const { @@ -4150,22 +4249,22 @@ void swap(BloomFilterHash &a, BloomFilterHash &b) { swap(a.__isset, b.__isset); } -BloomFilterHash::BloomFilterHash(const BloomFilterHash& other116) noexcept { - XXHASH = other116.XXHASH; - __isset = other116.__isset; +BloomFilterHash::BloomFilterHash(const BloomFilterHash& other120) noexcept { + XXHASH = other120.XXHASH; + __isset = other120.__isset; } -BloomFilterHash::BloomFilterHash(BloomFilterHash&& other117) noexcept { - XXHASH = std::move(other117.XXHASH); - __isset = other117.__isset; +BloomFilterHash::BloomFilterHash(BloomFilterHash&& other121) noexcept { + XXHASH = std::move(other121.XXHASH); + __isset = other121.__isset; } -BloomFilterHash& BloomFilterHash::operator=(const BloomFilterHash& other118) noexcept { - XXHASH = other118.XXHASH; - __isset = other118.__isset; +BloomFilterHash& BloomFilterHash::operator=(const BloomFilterHash& other122) noexcept { + XXHASH = other122.XXHASH; + __isset = other122.__isset; return *this; } -BloomFilterHash& BloomFilterHash::operator=(BloomFilterHash&& other119) noexcept { - XXHASH = std::move(other119.XXHASH); - __isset = other119.__isset; +BloomFilterHash& BloomFilterHash::operator=(BloomFilterHash&& other123) noexcept { + XXHASH = std::move(other123.XXHASH); + __isset = other123.__isset; return *this; } void BloomFilterHash::printTo(std::ostream& out) const { @@ -4230,18 +4329,18 @@ void swap(Uncompressed &a, Uncompressed &b) { (void) b; } -Uncompressed::Uncompressed(const Uncompressed& other120) noexcept { - (void) other120; +Uncompressed::Uncompressed(const Uncompressed& other124) noexcept { + (void) other124; } -Uncompressed::Uncompressed(Uncompressed&& other121) noexcept { - (void) other121; +Uncompressed::Uncompressed(Uncompressed&& other125) noexcept { + (void) other125; } -Uncompressed& Uncompressed::operator=(const Uncompressed& other122) noexcept { - (void) other122; +Uncompressed& Uncompressed::operator=(const Uncompressed& other126) noexcept { + (void) other126; return *this; } -Uncompressed& Uncompressed::operator=(Uncompressed&& other123) noexcept { - (void) other123; +Uncompressed& Uncompressed::operator=(Uncompressed&& other127) noexcept { + (void) other127; return *this; } void Uncompressed::printTo(std::ostream& out) const { @@ -4328,22 +4427,22 @@ void swap(BloomFilterCompression &a, BloomFilterCompression &b) { swap(a.__isset, b.__isset); } -BloomFilterCompression::BloomFilterCompression(const BloomFilterCompression& other124) noexcept { - UNCOMPRESSED = other124.UNCOMPRESSED; - __isset = other124.__isset; +BloomFilterCompression::BloomFilterCompression(const BloomFilterCompression& other128) noexcept { + UNCOMPRESSED = other128.UNCOMPRESSED; + __isset = other128.__isset; } -BloomFilterCompression::BloomFilterCompression(BloomFilterCompression&& other125) noexcept { - UNCOMPRESSED = std::move(other125.UNCOMPRESSED); - __isset = other125.__isset; +BloomFilterCompression::BloomFilterCompression(BloomFilterCompression&& other129) noexcept { + UNCOMPRESSED = std::move(other129.UNCOMPRESSED); + __isset = other129.__isset; } -BloomFilterCompression& BloomFilterCompression::operator=(const BloomFilterCompression& other126) noexcept { - UNCOMPRESSED = other126.UNCOMPRESSED; - __isset = other126.__isset; +BloomFilterCompression& BloomFilterCompression::operator=(const BloomFilterCompression& other130) noexcept { + UNCOMPRESSED = other130.UNCOMPRESSED; + __isset = other130.__isset; return *this; } -BloomFilterCompression& BloomFilterCompression::operator=(BloomFilterCompression&& other127) noexcept { - UNCOMPRESSED = std::move(other127.UNCOMPRESSED); - __isset = other127.__isset; +BloomFilterCompression& BloomFilterCompression::operator=(BloomFilterCompression&& other131) noexcept { + UNCOMPRESSED = std::move(other131.UNCOMPRESSED); + __isset = other131.__isset; return *this; } void BloomFilterCompression::printTo(std::ostream& out) const { @@ -4491,30 +4590,30 @@ void swap(BloomFilterHeader &a, BloomFilterHeader &b) { swap(a.compression, b.compression); } -BloomFilterHeader::BloomFilterHeader(const BloomFilterHeader& other128) noexcept { - numBytes = other128.numBytes; - algorithm = other128.algorithm; - hash = other128.hash; - compression = other128.compression; +BloomFilterHeader::BloomFilterHeader(const BloomFilterHeader& other132) noexcept { + numBytes = other132.numBytes; + algorithm = other132.algorithm; + hash = other132.hash; + compression = other132.compression; } -BloomFilterHeader::BloomFilterHeader(BloomFilterHeader&& other129) noexcept { - numBytes = other129.numBytes; - algorithm = std::move(other129.algorithm); - hash = std::move(other129.hash); - compression = std::move(other129.compression); +BloomFilterHeader::BloomFilterHeader(BloomFilterHeader&& other133) noexcept { + numBytes = other133.numBytes; + algorithm = std::move(other133.algorithm); + hash = std::move(other133.hash); + compression = std::move(other133.compression); } -BloomFilterHeader& BloomFilterHeader::operator=(const BloomFilterHeader& other130) noexcept { - numBytes = other130.numBytes; - algorithm = other130.algorithm; - hash = other130.hash; - compression = other130.compression; +BloomFilterHeader& BloomFilterHeader::operator=(const BloomFilterHeader& other134) noexcept { + numBytes = other134.numBytes; + algorithm = other134.algorithm; + hash = other134.hash; + compression = other134.compression; return *this; } -BloomFilterHeader& BloomFilterHeader::operator=(BloomFilterHeader&& other131) noexcept { - numBytes = other131.numBytes; - algorithm = std::move(other131.algorithm); - hash = std::move(other131.hash); - compression = std::move(other131.compression); +BloomFilterHeader& BloomFilterHeader::operator=(BloomFilterHeader&& other135) noexcept { + numBytes = other135.numBytes; + algorithm = std::move(other135.algorithm); + hash = std::move(other135.hash); + compression = std::move(other135.compression); return *this; } void BloomFilterHeader::printTo(std::ostream& out) const { @@ -4601,9 +4700,9 @@ uint32_t PageHeader::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast132; - xfer += iprot->readI32(ecast132); - this->type = static_cast(ecast132); + int32_t ecast136; + xfer += iprot->readI32(ecast136); + this->type = static_cast(ecast136); isset_type = true; } else { xfer += iprot->skip(ftype); @@ -4743,50 +4842,50 @@ void swap(PageHeader &a, PageHeader &b) { swap(a.__isset, b.__isset); } -PageHeader::PageHeader(const PageHeader& other133) { - type = other133.type; - uncompressed_page_size = other133.uncompressed_page_size; - compressed_page_size = other133.compressed_page_size; - crc = other133.crc; - data_page_header = other133.data_page_header; - index_page_header = other133.index_page_header; - dictionary_page_header = other133.dictionary_page_header; - data_page_header_v2 = other133.data_page_header_v2; - __isset = other133.__isset; -} -PageHeader::PageHeader(PageHeader&& other134) noexcept { - type = other134.type; - uncompressed_page_size = other134.uncompressed_page_size; - compressed_page_size = other134.compressed_page_size; - crc = other134.crc; - data_page_header = std::move(other134.data_page_header); - index_page_header = std::move(other134.index_page_header); - dictionary_page_header = std::move(other134.dictionary_page_header); - data_page_header_v2 = std::move(other134.data_page_header_v2); - __isset = other134.__isset; -} -PageHeader& PageHeader::operator=(const PageHeader& other135) { - type = other135.type; - uncompressed_page_size = other135.uncompressed_page_size; - compressed_page_size = other135.compressed_page_size; - crc = other135.crc; - data_page_header = other135.data_page_header; - index_page_header = other135.index_page_header; - dictionary_page_header = other135.dictionary_page_header; - data_page_header_v2 = other135.data_page_header_v2; - __isset = other135.__isset; +PageHeader::PageHeader(const PageHeader& other137) { + type = other137.type; + uncompressed_page_size = other137.uncompressed_page_size; + compressed_page_size = other137.compressed_page_size; + crc = other137.crc; + data_page_header = other137.data_page_header; + index_page_header = other137.index_page_header; + dictionary_page_header = other137.dictionary_page_header; + data_page_header_v2 = other137.data_page_header_v2; + __isset = other137.__isset; +} +PageHeader::PageHeader(PageHeader&& other138) noexcept { + type = other138.type; + uncompressed_page_size = other138.uncompressed_page_size; + compressed_page_size = other138.compressed_page_size; + crc = other138.crc; + data_page_header = std::move(other138.data_page_header); + index_page_header = std::move(other138.index_page_header); + dictionary_page_header = std::move(other138.dictionary_page_header); + data_page_header_v2 = std::move(other138.data_page_header_v2); + __isset = other138.__isset; +} +PageHeader& PageHeader::operator=(const PageHeader& other139) { + type = other139.type; + uncompressed_page_size = other139.uncompressed_page_size; + compressed_page_size = other139.compressed_page_size; + crc = other139.crc; + data_page_header = other139.data_page_header; + index_page_header = other139.index_page_header; + dictionary_page_header = other139.dictionary_page_header; + data_page_header_v2 = other139.data_page_header_v2; + __isset = other139.__isset; return *this; } -PageHeader& PageHeader::operator=(PageHeader&& other136) noexcept { - type = other136.type; - uncompressed_page_size = other136.uncompressed_page_size; - compressed_page_size = other136.compressed_page_size; - crc = other136.crc; - data_page_header = std::move(other136.data_page_header); - index_page_header = std::move(other136.index_page_header); - dictionary_page_header = std::move(other136.dictionary_page_header); - data_page_header_v2 = std::move(other136.data_page_header_v2); - __isset = other136.__isset; +PageHeader& PageHeader::operator=(PageHeader&& other140) noexcept { + type = other140.type; + uncompressed_page_size = other140.uncompressed_page_size; + compressed_page_size = other140.compressed_page_size; + crc = other140.crc; + data_page_header = std::move(other140.data_page_header); + index_page_header = std::move(other140.index_page_header); + dictionary_page_header = std::move(other140.dictionary_page_header); + data_page_header_v2 = std::move(other140.data_page_header_v2); + __isset = other140.__isset; return *this; } void PageHeader::printTo(std::ostream& out) const { @@ -4901,26 +5000,26 @@ void swap(KeyValue &a, KeyValue &b) { swap(a.__isset, b.__isset); } -KeyValue::KeyValue(const KeyValue& other137) { - key = other137.key; - value = other137.value; - __isset = other137.__isset; +KeyValue::KeyValue(const KeyValue& other141) { + key = other141.key; + value = other141.value; + __isset = other141.__isset; } -KeyValue::KeyValue(KeyValue&& other138) noexcept { - key = std::move(other138.key); - value = std::move(other138.value); - __isset = other138.__isset; +KeyValue::KeyValue(KeyValue&& other142) noexcept { + key = std::move(other142.key); + value = std::move(other142.value); + __isset = other142.__isset; } -KeyValue& KeyValue::operator=(const KeyValue& other139) { - key = other139.key; - value = other139.value; - __isset = other139.__isset; +KeyValue& KeyValue::operator=(const KeyValue& other143) { + key = other143.key; + value = other143.value; + __isset = other143.__isset; return *this; } -KeyValue& KeyValue::operator=(KeyValue&& other140) noexcept { - key = std::move(other140.key); - value = std::move(other140.value); - __isset = other140.__isset; +KeyValue& KeyValue::operator=(KeyValue&& other144) noexcept { + key = std::move(other144.key); + value = std::move(other144.value); + __isset = other144.__isset; return *this; } void KeyValue::printTo(std::ostream& out) const { @@ -5049,26 +5148,26 @@ void swap(SortingColumn &a, SortingColumn &b) { swap(a.nulls_first, b.nulls_first); } -SortingColumn::SortingColumn(const SortingColumn& other141) noexcept { - column_idx = other141.column_idx; - descending = other141.descending; - nulls_first = other141.nulls_first; +SortingColumn::SortingColumn(const SortingColumn& other145) noexcept { + column_idx = other145.column_idx; + descending = other145.descending; + nulls_first = other145.nulls_first; } -SortingColumn::SortingColumn(SortingColumn&& other142) noexcept { - column_idx = other142.column_idx; - descending = other142.descending; - nulls_first = other142.nulls_first; +SortingColumn::SortingColumn(SortingColumn&& other146) noexcept { + column_idx = other146.column_idx; + descending = other146.descending; + nulls_first = other146.nulls_first; } -SortingColumn& SortingColumn::operator=(const SortingColumn& other143) noexcept { - column_idx = other143.column_idx; - descending = other143.descending; - nulls_first = other143.nulls_first; +SortingColumn& SortingColumn::operator=(const SortingColumn& other147) noexcept { + column_idx = other147.column_idx; + descending = other147.descending; + nulls_first = other147.nulls_first; return *this; } -SortingColumn& SortingColumn::operator=(SortingColumn&& other144) noexcept { - column_idx = other144.column_idx; - descending = other144.descending; - nulls_first = other144.nulls_first; +SortingColumn& SortingColumn::operator=(SortingColumn&& other148) noexcept { + column_idx = other148.column_idx; + descending = other148.descending; + nulls_first = other148.nulls_first; return *this; } void SortingColumn::printTo(std::ostream& out) const { @@ -5129,9 +5228,9 @@ uint32_t PageEncodingStats::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast145; - xfer += iprot->readI32(ecast145); - this->page_type = static_cast(ecast145); + int32_t ecast149; + xfer += iprot->readI32(ecast149); + this->page_type = static_cast(ecast149); isset_page_type = true; } else { xfer += iprot->skip(ftype); @@ -5139,9 +5238,9 @@ uint32_t PageEncodingStats::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast146; - xfer += iprot->readI32(ecast146); - this->encoding = static_cast(ecast146); + int32_t ecast150; + xfer += iprot->readI32(ecast150); + this->encoding = static_cast(ecast150); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -5202,26 +5301,26 @@ void swap(PageEncodingStats &a, PageEncodingStats &b) { swap(a.count, b.count); } -PageEncodingStats::PageEncodingStats(const PageEncodingStats& other147) noexcept { - page_type = other147.page_type; - encoding = other147.encoding; - count = other147.count; +PageEncodingStats::PageEncodingStats(const PageEncodingStats& other151) noexcept { + page_type = other151.page_type; + encoding = other151.encoding; + count = other151.count; } -PageEncodingStats::PageEncodingStats(PageEncodingStats&& other148) noexcept { - page_type = other148.page_type; - encoding = other148.encoding; - count = other148.count; +PageEncodingStats::PageEncodingStats(PageEncodingStats&& other152) noexcept { + page_type = other152.page_type; + encoding = other152.encoding; + count = other152.count; } -PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats& other149) noexcept { - page_type = other149.page_type; - encoding = other149.encoding; - count = other149.count; +PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats& other153) noexcept { + page_type = other153.page_type; + encoding = other153.encoding; + count = other153.count; return *this; } -PageEncodingStats& PageEncodingStats::operator=(PageEncodingStats&& other150) noexcept { - page_type = other150.page_type; - encoding = other150.encoding; - count = other150.count; +PageEncodingStats& PageEncodingStats::operator=(PageEncodingStats&& other154) noexcept { + page_type = other154.page_type; + encoding = other154.encoding; + count = other154.count; return *this; } void PageEncodingStats::printTo(std::ostream& out) const { @@ -5337,9 +5436,9 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast151; - xfer += iprot->readI32(ecast151); - this->type = static_cast(ecast151); + int32_t ecast155; + xfer += iprot->readI32(ecast155); + this->type = static_cast(ecast155); isset_type = true; } else { xfer += iprot->skip(ftype); @@ -5349,16 +5448,16 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->encodings.clear(); - uint32_t _size152; - ::apache::thrift::protocol::TType _etype155; - xfer += iprot->readListBegin(_etype155, _size152); - this->encodings.resize(_size152); - uint32_t _i156; - for (_i156 = 0; _i156 < _size152; ++_i156) + uint32_t _size156; + ::apache::thrift::protocol::TType _etype159; + xfer += iprot->readListBegin(_etype159, _size156); + this->encodings.resize(_size156); + uint32_t _i160; + for (_i160 = 0; _i160 < _size156; ++_i160) { - int32_t ecast157; - xfer += iprot->readI32(ecast157); - this->encodings[_i156] = static_cast(ecast157); + int32_t ecast161; + xfer += iprot->readI32(ecast161); + this->encodings[_i160] = static_cast(ecast161); } xfer += iprot->readListEnd(); } @@ -5371,14 +5470,14 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->path_in_schema.clear(); - uint32_t _size158; - ::apache::thrift::protocol::TType _etype161; - xfer += iprot->readListBegin(_etype161, _size158); - this->path_in_schema.resize(_size158); - uint32_t _i162; - for (_i162 = 0; _i162 < _size158; ++_i162) + uint32_t _size162; + ::apache::thrift::protocol::TType _etype165; + xfer += iprot->readListBegin(_etype165, _size162); + this->path_in_schema.resize(_size162); + uint32_t _i166; + for (_i166 = 0; _i166 < _size162; ++_i166) { - xfer += iprot->readString(this->path_in_schema[_i162]); + xfer += iprot->readString(this->path_in_schema[_i166]); } xfer += iprot->readListEnd(); } @@ -5389,9 +5488,9 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast163; - xfer += iprot->readI32(ecast163); - this->codec = static_cast(ecast163); + int32_t ecast167; + xfer += iprot->readI32(ecast167); + this->codec = static_cast(ecast167); isset_codec = true; } else { xfer += iprot->skip(ftype); @@ -5425,14 +5524,14 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->key_value_metadata.clear(); - uint32_t _size164; - ::apache::thrift::protocol::TType _etype167; - xfer += iprot->readListBegin(_etype167, _size164); - this->key_value_metadata.resize(_size164); - uint32_t _i168; - for (_i168 = 0; _i168 < _size164; ++_i168) + uint32_t _size168; + ::apache::thrift::protocol::TType _etype171; + xfer += iprot->readListBegin(_etype171, _size168); + this->key_value_metadata.resize(_size168); + uint32_t _i172; + for (_i172 = 0; _i172 < _size168; ++_i172) { - xfer += this->key_value_metadata[_i168].read(iprot); + xfer += this->key_value_metadata[_i172].read(iprot); } xfer += iprot->readListEnd(); } @@ -5477,14 +5576,14 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->encoding_stats.clear(); - uint32_t _size169; - ::apache::thrift::protocol::TType _etype172; - xfer += iprot->readListBegin(_etype172, _size169); - this->encoding_stats.resize(_size169); - uint32_t _i173; - for (_i173 = 0; _i173 < _size169; ++_i173) + uint32_t _size173; + ::apache::thrift::protocol::TType _etype176; + xfer += iprot->readListBegin(_etype176, _size173); + this->encoding_stats.resize(_size173); + uint32_t _i177; + for (_i177 = 0; _i177 < _size173; ++_i177) { - xfer += this->encoding_stats[_i173].read(iprot); + xfer += this->encoding_stats[_i177].read(iprot); } xfer += iprot->readListEnd(); } @@ -5541,10 +5640,10 @@ uint32_t ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con xfer += oprot->writeFieldBegin("encodings", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I32, static_cast(this->encodings.size())); - std::vector ::const_iterator _iter174; - for (_iter174 = this->encodings.begin(); _iter174 != this->encodings.end(); ++_iter174) + std::vector ::const_iterator _iter178; + for (_iter178 = this->encodings.begin(); _iter178 != this->encodings.end(); ++_iter178) { - xfer += oprot->writeI32(static_cast((*_iter174))); + xfer += oprot->writeI32(static_cast((*_iter178))); } xfer += oprot->writeListEnd(); } @@ -5553,10 +5652,10 @@ uint32_t ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); - std::vector ::const_iterator _iter175; - for (_iter175 = this->path_in_schema.begin(); _iter175 != this->path_in_schema.end(); ++_iter175) + std::vector ::const_iterator _iter179; + for (_iter179 = this->path_in_schema.begin(); _iter179 != this->path_in_schema.end(); ++_iter179) { - xfer += oprot->writeString((*_iter175)); + xfer += oprot->writeString((*_iter179)); } xfer += oprot->writeListEnd(); } @@ -5582,10 +5681,10 @@ uint32_t ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 8); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter176; - for (_iter176 = this->key_value_metadata.begin(); _iter176 != this->key_value_metadata.end(); ++_iter176) + std::vector ::const_iterator _iter180; + for (_iter180 = this->key_value_metadata.begin(); _iter180 != this->key_value_metadata.end(); ++_iter180) { - xfer += (*_iter176).write(oprot); + xfer += (*_iter180).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5614,10 +5713,10 @@ uint32_t ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con xfer += oprot->writeFieldBegin("encoding_stats", ::apache::thrift::protocol::T_LIST, 13); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->encoding_stats.size())); - std::vector ::const_iterator _iter177; - for (_iter177 = this->encoding_stats.begin(); _iter177 != this->encoding_stats.end(); ++_iter177) + std::vector ::const_iterator _iter181; + for (_iter181 = this->encoding_stats.begin(); _iter181 != this->encoding_stats.end(); ++_iter181) { - xfer += (*_iter177).write(oprot); + xfer += (*_iter181).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5652,74 +5751,74 @@ void swap(ColumnMetaData &a, ColumnMetaData &b) { swap(a.__isset, b.__isset); } -ColumnMetaData::ColumnMetaData(const ColumnMetaData& other178) { - type = other178.type; - encodings = other178.encodings; - path_in_schema = other178.path_in_schema; - codec = other178.codec; - num_values = other178.num_values; - total_uncompressed_size = other178.total_uncompressed_size; - total_compressed_size = other178.total_compressed_size; - key_value_metadata = other178.key_value_metadata; - data_page_offset = other178.data_page_offset; - index_page_offset = other178.index_page_offset; - dictionary_page_offset = other178.dictionary_page_offset; - statistics = other178.statistics; - encoding_stats = other178.encoding_stats; - bloom_filter_offset = other178.bloom_filter_offset; - __isset = other178.__isset; -} -ColumnMetaData::ColumnMetaData(ColumnMetaData&& other179) noexcept { - type = other179.type; - encodings = std::move(other179.encodings); - path_in_schema = std::move(other179.path_in_schema); - codec = other179.codec; - num_values = other179.num_values; - total_uncompressed_size = other179.total_uncompressed_size; - total_compressed_size = other179.total_compressed_size; - key_value_metadata = std::move(other179.key_value_metadata); - data_page_offset = other179.data_page_offset; - index_page_offset = other179.index_page_offset; - dictionary_page_offset = other179.dictionary_page_offset; - statistics = std::move(other179.statistics); - encoding_stats = std::move(other179.encoding_stats); - bloom_filter_offset = other179.bloom_filter_offset; - __isset = other179.__isset; -} -ColumnMetaData& ColumnMetaData::operator=(const ColumnMetaData& other180) { - type = other180.type; - encodings = other180.encodings; - path_in_schema = other180.path_in_schema; - codec = other180.codec; - num_values = other180.num_values; - total_uncompressed_size = other180.total_uncompressed_size; - total_compressed_size = other180.total_compressed_size; - key_value_metadata = other180.key_value_metadata; - data_page_offset = other180.data_page_offset; - index_page_offset = other180.index_page_offset; - dictionary_page_offset = other180.dictionary_page_offset; - statistics = other180.statistics; - encoding_stats = other180.encoding_stats; - bloom_filter_offset = other180.bloom_filter_offset; - __isset = other180.__isset; +ColumnMetaData::ColumnMetaData(const ColumnMetaData& other182) { + type = other182.type; + encodings = other182.encodings; + path_in_schema = other182.path_in_schema; + codec = other182.codec; + num_values = other182.num_values; + total_uncompressed_size = other182.total_uncompressed_size; + total_compressed_size = other182.total_compressed_size; + key_value_metadata = other182.key_value_metadata; + data_page_offset = other182.data_page_offset; + index_page_offset = other182.index_page_offset; + dictionary_page_offset = other182.dictionary_page_offset; + statistics = other182.statistics; + encoding_stats = other182.encoding_stats; + bloom_filter_offset = other182.bloom_filter_offset; + __isset = other182.__isset; +} +ColumnMetaData::ColumnMetaData(ColumnMetaData&& other183) noexcept { + type = other183.type; + encodings = std::move(other183.encodings); + path_in_schema = std::move(other183.path_in_schema); + codec = other183.codec; + num_values = other183.num_values; + total_uncompressed_size = other183.total_uncompressed_size; + total_compressed_size = other183.total_compressed_size; + key_value_metadata = std::move(other183.key_value_metadata); + data_page_offset = other183.data_page_offset; + index_page_offset = other183.index_page_offset; + dictionary_page_offset = other183.dictionary_page_offset; + statistics = std::move(other183.statistics); + encoding_stats = std::move(other183.encoding_stats); + bloom_filter_offset = other183.bloom_filter_offset; + __isset = other183.__isset; +} +ColumnMetaData& ColumnMetaData::operator=(const ColumnMetaData& other184) { + type = other184.type; + encodings = other184.encodings; + path_in_schema = other184.path_in_schema; + codec = other184.codec; + num_values = other184.num_values; + total_uncompressed_size = other184.total_uncompressed_size; + total_compressed_size = other184.total_compressed_size; + key_value_metadata = other184.key_value_metadata; + data_page_offset = other184.data_page_offset; + index_page_offset = other184.index_page_offset; + dictionary_page_offset = other184.dictionary_page_offset; + statistics = other184.statistics; + encoding_stats = other184.encoding_stats; + bloom_filter_offset = other184.bloom_filter_offset; + __isset = other184.__isset; return *this; } -ColumnMetaData& ColumnMetaData::operator=(ColumnMetaData&& other181) noexcept { - type = other181.type; - encodings = std::move(other181.encodings); - path_in_schema = std::move(other181.path_in_schema); - codec = other181.codec; - num_values = other181.num_values; - total_uncompressed_size = other181.total_uncompressed_size; - total_compressed_size = other181.total_compressed_size; - key_value_metadata = std::move(other181.key_value_metadata); - data_page_offset = other181.data_page_offset; - index_page_offset = other181.index_page_offset; - dictionary_page_offset = other181.dictionary_page_offset; - statistics = std::move(other181.statistics); - encoding_stats = std::move(other181.encoding_stats); - bloom_filter_offset = other181.bloom_filter_offset; - __isset = other181.__isset; +ColumnMetaData& ColumnMetaData::operator=(ColumnMetaData&& other185) noexcept { + type = other185.type; + encodings = std::move(other185.encodings); + path_in_schema = std::move(other185.path_in_schema); + codec = other185.codec; + num_values = other185.num_values; + total_uncompressed_size = other185.total_uncompressed_size; + total_compressed_size = other185.total_compressed_size; + key_value_metadata = std::move(other185.key_value_metadata); + data_page_offset = other185.data_page_offset; + index_page_offset = other185.index_page_offset; + dictionary_page_offset = other185.dictionary_page_offset; + statistics = std::move(other185.statistics); + encoding_stats = std::move(other185.encoding_stats); + bloom_filter_offset = other185.bloom_filter_offset; + __isset = other185.__isset; return *this; } void ColumnMetaData::printTo(std::ostream& out) const { @@ -5797,18 +5896,18 @@ void swap(EncryptionWithFooterKey &a, EncryptionWithFooterKey &b) { (void) b; } -EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other182) noexcept { - (void) other182; +EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other186) noexcept { + (void) other186; } -EncryptionWithFooterKey::EncryptionWithFooterKey(EncryptionWithFooterKey&& other183) noexcept { - (void) other183; +EncryptionWithFooterKey::EncryptionWithFooterKey(EncryptionWithFooterKey&& other187) noexcept { + (void) other187; } -EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other184) noexcept { - (void) other184; +EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other188) noexcept { + (void) other188; return *this; } -EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(EncryptionWithFooterKey&& other185) noexcept { - (void) other185; +EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(EncryptionWithFooterKey&& other189) noexcept { + (void) other189; return *this; } void EncryptionWithFooterKey::printTo(std::ostream& out) const { @@ -5863,14 +5962,14 @@ uint32_t EncryptionWithColumnKey::read(::apache::thrift::protocol::TProtocol* ip if (ftype == ::apache::thrift::protocol::T_LIST) { { this->path_in_schema.clear(); - uint32_t _size186; - ::apache::thrift::protocol::TType _etype189; - xfer += iprot->readListBegin(_etype189, _size186); - this->path_in_schema.resize(_size186); - uint32_t _i190; - for (_i190 = 0; _i190 < _size186; ++_i190) + uint32_t _size190; + ::apache::thrift::protocol::TType _etype193; + xfer += iprot->readListBegin(_etype193, _size190); + this->path_in_schema.resize(_size190); + uint32_t _i194; + for (_i194 = 0; _i194 < _size190; ++_i194) { - xfer += iprot->readString(this->path_in_schema[_i190]); + xfer += iprot->readString(this->path_in_schema[_i194]); } xfer += iprot->readListEnd(); } @@ -5909,10 +6008,10 @@ uint32_t EncryptionWithColumnKey::write(::apache::thrift::protocol::TProtocol* o xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); - std::vector ::const_iterator _iter191; - for (_iter191 = this->path_in_schema.begin(); _iter191 != this->path_in_schema.end(); ++_iter191) + std::vector ::const_iterator _iter195; + for (_iter195 = this->path_in_schema.begin(); _iter195 != this->path_in_schema.end(); ++_iter195) { - xfer += oprot->writeString((*_iter191)); + xfer += oprot->writeString((*_iter195)); } xfer += oprot->writeListEnd(); } @@ -5935,26 +6034,26 @@ void swap(EncryptionWithColumnKey &a, EncryptionWithColumnKey &b) { swap(a.__isset, b.__isset); } -EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other192) { - path_in_schema = other192.path_in_schema; - key_metadata = other192.key_metadata; - __isset = other192.__isset; +EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other196) { + path_in_schema = other196.path_in_schema; + key_metadata = other196.key_metadata; + __isset = other196.__isset; } -EncryptionWithColumnKey::EncryptionWithColumnKey(EncryptionWithColumnKey&& other193) noexcept { - path_in_schema = std::move(other193.path_in_schema); - key_metadata = std::move(other193.key_metadata); - __isset = other193.__isset; +EncryptionWithColumnKey::EncryptionWithColumnKey(EncryptionWithColumnKey&& other197) noexcept { + path_in_schema = std::move(other197.path_in_schema); + key_metadata = std::move(other197.key_metadata); + __isset = other197.__isset; } -EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other194) { - path_in_schema = other194.path_in_schema; - key_metadata = other194.key_metadata; - __isset = other194.__isset; +EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other198) { + path_in_schema = other198.path_in_schema; + key_metadata = other198.key_metadata; + __isset = other198.__isset; return *this; } -EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(EncryptionWithColumnKey&& other195) noexcept { - path_in_schema = std::move(other195.path_in_schema); - key_metadata = std::move(other195.key_metadata); - __isset = other195.__isset; +EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(EncryptionWithColumnKey&& other199) noexcept { + path_in_schema = std::move(other199.path_in_schema); + key_metadata = std::move(other199.key_metadata); + __isset = other199.__isset; return *this; } void EncryptionWithColumnKey::printTo(std::ostream& out) const { @@ -6062,26 +6161,26 @@ void swap(ColumnCryptoMetaData &a, ColumnCryptoMetaData &b) { swap(a.__isset, b.__isset); } -ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other196) { - ENCRYPTION_WITH_FOOTER_KEY = other196.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other196.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other196.__isset; +ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other200) { + ENCRYPTION_WITH_FOOTER_KEY = other200.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other200.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other200.__isset; } -ColumnCryptoMetaData::ColumnCryptoMetaData(ColumnCryptoMetaData&& other197) noexcept { - ENCRYPTION_WITH_FOOTER_KEY = std::move(other197.ENCRYPTION_WITH_FOOTER_KEY); - ENCRYPTION_WITH_COLUMN_KEY = std::move(other197.ENCRYPTION_WITH_COLUMN_KEY); - __isset = other197.__isset; +ColumnCryptoMetaData::ColumnCryptoMetaData(ColumnCryptoMetaData&& other201) noexcept { + ENCRYPTION_WITH_FOOTER_KEY = std::move(other201.ENCRYPTION_WITH_FOOTER_KEY); + ENCRYPTION_WITH_COLUMN_KEY = std::move(other201.ENCRYPTION_WITH_COLUMN_KEY); + __isset = other201.__isset; } -ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other198) { - ENCRYPTION_WITH_FOOTER_KEY = other198.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other198.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other198.__isset; +ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other202) { + ENCRYPTION_WITH_FOOTER_KEY = other202.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other202.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other202.__isset; return *this; } -ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(ColumnCryptoMetaData&& other199) noexcept { - ENCRYPTION_WITH_FOOTER_KEY = std::move(other199.ENCRYPTION_WITH_FOOTER_KEY); - ENCRYPTION_WITH_COLUMN_KEY = std::move(other199.ENCRYPTION_WITH_COLUMN_KEY); - __isset = other199.__isset; +ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(ColumnCryptoMetaData&& other203) noexcept { + ENCRYPTION_WITH_FOOTER_KEY = std::move(other203.ENCRYPTION_WITH_FOOTER_KEY); + ENCRYPTION_WITH_COLUMN_KEY = std::move(other203.ENCRYPTION_WITH_COLUMN_KEY); + __isset = other203.__isset; return *this; } void ColumnCryptoMetaData::printTo(std::ostream& out) const { @@ -6323,54 +6422,54 @@ void swap(ColumnChunk &a, ColumnChunk &b) { swap(a.__isset, b.__isset); } -ColumnChunk::ColumnChunk(const ColumnChunk& other200) { - file_path = other200.file_path; - file_offset = other200.file_offset; - meta_data = other200.meta_data; - offset_index_offset = other200.offset_index_offset; - offset_index_length = other200.offset_index_length; - column_index_offset = other200.column_index_offset; - column_index_length = other200.column_index_length; - crypto_metadata = other200.crypto_metadata; - encrypted_column_metadata = other200.encrypted_column_metadata; - __isset = other200.__isset; -} -ColumnChunk::ColumnChunk(ColumnChunk&& other201) noexcept { - file_path = std::move(other201.file_path); - file_offset = other201.file_offset; - meta_data = std::move(other201.meta_data); - offset_index_offset = other201.offset_index_offset; - offset_index_length = other201.offset_index_length; - column_index_offset = other201.column_index_offset; - column_index_length = other201.column_index_length; - crypto_metadata = std::move(other201.crypto_metadata); - encrypted_column_metadata = std::move(other201.encrypted_column_metadata); - __isset = other201.__isset; -} -ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other202) { - file_path = other202.file_path; - file_offset = other202.file_offset; - meta_data = other202.meta_data; - offset_index_offset = other202.offset_index_offset; - offset_index_length = other202.offset_index_length; - column_index_offset = other202.column_index_offset; - column_index_length = other202.column_index_length; - crypto_metadata = other202.crypto_metadata; - encrypted_column_metadata = other202.encrypted_column_metadata; - __isset = other202.__isset; +ColumnChunk::ColumnChunk(const ColumnChunk& other204) { + file_path = other204.file_path; + file_offset = other204.file_offset; + meta_data = other204.meta_data; + offset_index_offset = other204.offset_index_offset; + offset_index_length = other204.offset_index_length; + column_index_offset = other204.column_index_offset; + column_index_length = other204.column_index_length; + crypto_metadata = other204.crypto_metadata; + encrypted_column_metadata = other204.encrypted_column_metadata; + __isset = other204.__isset; +} +ColumnChunk::ColumnChunk(ColumnChunk&& other205) noexcept { + file_path = std::move(other205.file_path); + file_offset = other205.file_offset; + meta_data = std::move(other205.meta_data); + offset_index_offset = other205.offset_index_offset; + offset_index_length = other205.offset_index_length; + column_index_offset = other205.column_index_offset; + column_index_length = other205.column_index_length; + crypto_metadata = std::move(other205.crypto_metadata); + encrypted_column_metadata = std::move(other205.encrypted_column_metadata); + __isset = other205.__isset; +} +ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other206) { + file_path = other206.file_path; + file_offset = other206.file_offset; + meta_data = other206.meta_data; + offset_index_offset = other206.offset_index_offset; + offset_index_length = other206.offset_index_length; + column_index_offset = other206.column_index_offset; + column_index_length = other206.column_index_length; + crypto_metadata = other206.crypto_metadata; + encrypted_column_metadata = other206.encrypted_column_metadata; + __isset = other206.__isset; return *this; } -ColumnChunk& ColumnChunk::operator=(ColumnChunk&& other203) noexcept { - file_path = std::move(other203.file_path); - file_offset = other203.file_offset; - meta_data = std::move(other203.meta_data); - offset_index_offset = other203.offset_index_offset; - offset_index_length = other203.offset_index_length; - column_index_offset = other203.column_index_offset; - column_index_length = other203.column_index_length; - crypto_metadata = std::move(other203.crypto_metadata); - encrypted_column_metadata = std::move(other203.encrypted_column_metadata); - __isset = other203.__isset; +ColumnChunk& ColumnChunk::operator=(ColumnChunk&& other207) noexcept { + file_path = std::move(other207.file_path); + file_offset = other207.file_offset; + meta_data = std::move(other207.meta_data); + offset_index_offset = other207.offset_index_offset; + offset_index_length = other207.offset_index_length; + column_index_offset = other207.column_index_offset; + column_index_length = other207.column_index_length; + crypto_metadata = std::move(other207.crypto_metadata); + encrypted_column_metadata = std::move(other207.encrypted_column_metadata); + __isset = other207.__isset; return *this; } void ColumnChunk::printTo(std::ostream& out) const { @@ -6459,14 +6558,14 @@ uint32_t RowGroup::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->columns.clear(); - uint32_t _size204; - ::apache::thrift::protocol::TType _etype207; - xfer += iprot->readListBegin(_etype207, _size204); - this->columns.resize(_size204); - uint32_t _i208; - for (_i208 = 0; _i208 < _size204; ++_i208) + uint32_t _size208; + ::apache::thrift::protocol::TType _etype211; + xfer += iprot->readListBegin(_etype211, _size208); + this->columns.resize(_size208); + uint32_t _i212; + for (_i212 = 0; _i212 < _size208; ++_i212) { - xfer += this->columns[_i208].read(iprot); + xfer += this->columns[_i212].read(iprot); } xfer += iprot->readListEnd(); } @@ -6495,14 +6594,14 @@ uint32_t RowGroup::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->sorting_columns.clear(); - uint32_t _size209; - ::apache::thrift::protocol::TType _etype212; - xfer += iprot->readListBegin(_etype212, _size209); - this->sorting_columns.resize(_size209); - uint32_t _i213; - for (_i213 = 0; _i213 < _size209; ++_i213) + uint32_t _size213; + ::apache::thrift::protocol::TType _etype216; + xfer += iprot->readListBegin(_etype216, _size213); + this->sorting_columns.resize(_size213); + uint32_t _i217; + for (_i217 = 0; _i217 < _size213; ++_i217) { - xfer += this->sorting_columns[_i213].read(iprot); + xfer += this->sorting_columns[_i217].read(iprot); } xfer += iprot->readListEnd(); } @@ -6561,10 +6660,10 @@ uint32_t RowGroup::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("columns", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->columns.size())); - std::vector ::const_iterator _iter214; - for (_iter214 = this->columns.begin(); _iter214 != this->columns.end(); ++_iter214) + std::vector ::const_iterator _iter218; + for (_iter218 = this->columns.begin(); _iter218 != this->columns.end(); ++_iter218) { - xfer += (*_iter214).write(oprot); + xfer += (*_iter218).write(oprot); } xfer += oprot->writeListEnd(); } @@ -6582,10 +6681,10 @@ uint32_t RowGroup::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("sorting_columns", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->sorting_columns.size())); - std::vector ::const_iterator _iter215; - for (_iter215 = this->sorting_columns.begin(); _iter215 != this->sorting_columns.end(); ++_iter215) + std::vector ::const_iterator _iter219; + for (_iter219 = this->sorting_columns.begin(); _iter219 != this->sorting_columns.end(); ++_iter219) { - xfer += (*_iter215).write(oprot); + xfer += (*_iter219).write(oprot); } xfer += oprot->writeListEnd(); } @@ -6623,46 +6722,46 @@ void swap(RowGroup &a, RowGroup &b) { swap(a.__isset, b.__isset); } -RowGroup::RowGroup(const RowGroup& other216) { - columns = other216.columns; - total_byte_size = other216.total_byte_size; - num_rows = other216.num_rows; - sorting_columns = other216.sorting_columns; - file_offset = other216.file_offset; - total_compressed_size = other216.total_compressed_size; - ordinal = other216.ordinal; - __isset = other216.__isset; -} -RowGroup::RowGroup(RowGroup&& other217) noexcept { - columns = std::move(other217.columns); - total_byte_size = other217.total_byte_size; - num_rows = other217.num_rows; - sorting_columns = std::move(other217.sorting_columns); - file_offset = other217.file_offset; - total_compressed_size = other217.total_compressed_size; - ordinal = other217.ordinal; - __isset = other217.__isset; -} -RowGroup& RowGroup::operator=(const RowGroup& other218) { - columns = other218.columns; - total_byte_size = other218.total_byte_size; - num_rows = other218.num_rows; - sorting_columns = other218.sorting_columns; - file_offset = other218.file_offset; - total_compressed_size = other218.total_compressed_size; - ordinal = other218.ordinal; - __isset = other218.__isset; +RowGroup::RowGroup(const RowGroup& other220) { + columns = other220.columns; + total_byte_size = other220.total_byte_size; + num_rows = other220.num_rows; + sorting_columns = other220.sorting_columns; + file_offset = other220.file_offset; + total_compressed_size = other220.total_compressed_size; + ordinal = other220.ordinal; + __isset = other220.__isset; +} +RowGroup::RowGroup(RowGroup&& other221) noexcept { + columns = std::move(other221.columns); + total_byte_size = other221.total_byte_size; + num_rows = other221.num_rows; + sorting_columns = std::move(other221.sorting_columns); + file_offset = other221.file_offset; + total_compressed_size = other221.total_compressed_size; + ordinal = other221.ordinal; + __isset = other221.__isset; +} +RowGroup& RowGroup::operator=(const RowGroup& other222) { + columns = other222.columns; + total_byte_size = other222.total_byte_size; + num_rows = other222.num_rows; + sorting_columns = other222.sorting_columns; + file_offset = other222.file_offset; + total_compressed_size = other222.total_compressed_size; + ordinal = other222.ordinal; + __isset = other222.__isset; return *this; } -RowGroup& RowGroup::operator=(RowGroup&& other219) noexcept { - columns = std::move(other219.columns); - total_byte_size = other219.total_byte_size; - num_rows = other219.num_rows; - sorting_columns = std::move(other219.sorting_columns); - file_offset = other219.file_offset; - total_compressed_size = other219.total_compressed_size; - ordinal = other219.ordinal; - __isset = other219.__isset; +RowGroup& RowGroup::operator=(RowGroup&& other223) noexcept { + columns = std::move(other223.columns); + total_byte_size = other223.total_byte_size; + num_rows = other223.num_rows; + sorting_columns = std::move(other223.sorting_columns); + file_offset = other223.file_offset; + total_compressed_size = other223.total_compressed_size; + ordinal = other223.ordinal; + __isset = other223.__isset; return *this; } void RowGroup::printTo(std::ostream& out) const { @@ -6733,18 +6832,18 @@ void swap(TypeDefinedOrder &a, TypeDefinedOrder &b) { (void) b; } -TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other220) noexcept { - (void) other220; +TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other224) noexcept { + (void) other224; } -TypeDefinedOrder::TypeDefinedOrder(TypeDefinedOrder&& other221) noexcept { - (void) other221; +TypeDefinedOrder::TypeDefinedOrder(TypeDefinedOrder&& other225) noexcept { + (void) other225; } -TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other222) noexcept { - (void) other222; +TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other226) noexcept { + (void) other226; return *this; } -TypeDefinedOrder& TypeDefinedOrder::operator=(TypeDefinedOrder&& other223) noexcept { - (void) other223; +TypeDefinedOrder& TypeDefinedOrder::operator=(TypeDefinedOrder&& other227) noexcept { + (void) other227; return *this; } void TypeDefinedOrder::printTo(std::ostream& out) const { @@ -6831,22 +6930,22 @@ void swap(ColumnOrder &a, ColumnOrder &b) { swap(a.__isset, b.__isset); } -ColumnOrder::ColumnOrder(const ColumnOrder& other224) noexcept { - TYPE_ORDER = other224.TYPE_ORDER; - __isset = other224.__isset; +ColumnOrder::ColumnOrder(const ColumnOrder& other228) noexcept { + TYPE_ORDER = other228.TYPE_ORDER; + __isset = other228.__isset; } -ColumnOrder::ColumnOrder(ColumnOrder&& other225) noexcept { - TYPE_ORDER = std::move(other225.TYPE_ORDER); - __isset = other225.__isset; +ColumnOrder::ColumnOrder(ColumnOrder&& other229) noexcept { + TYPE_ORDER = std::move(other229.TYPE_ORDER); + __isset = other229.__isset; } -ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other226) noexcept { - TYPE_ORDER = other226.TYPE_ORDER; - __isset = other226.__isset; +ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other230) noexcept { + TYPE_ORDER = other230.TYPE_ORDER; + __isset = other230.__isset; return *this; } -ColumnOrder& ColumnOrder::operator=(ColumnOrder&& other227) noexcept { - TYPE_ORDER = std::move(other227.TYPE_ORDER); - __isset = other227.__isset; +ColumnOrder& ColumnOrder::operator=(ColumnOrder&& other231) noexcept { + TYPE_ORDER = std::move(other231.TYPE_ORDER); + __isset = other231.__isset; return *this; } void ColumnOrder::printTo(std::ostream& out) const { @@ -6974,26 +7073,26 @@ void swap(PageLocation &a, PageLocation &b) { swap(a.first_row_index, b.first_row_index); } -PageLocation::PageLocation(const PageLocation& other228) noexcept { - offset = other228.offset; - compressed_page_size = other228.compressed_page_size; - first_row_index = other228.first_row_index; +PageLocation::PageLocation(const PageLocation& other232) noexcept { + offset = other232.offset; + compressed_page_size = other232.compressed_page_size; + first_row_index = other232.first_row_index; } -PageLocation::PageLocation(PageLocation&& other229) noexcept { - offset = other229.offset; - compressed_page_size = other229.compressed_page_size; - first_row_index = other229.first_row_index; +PageLocation::PageLocation(PageLocation&& other233) noexcept { + offset = other233.offset; + compressed_page_size = other233.compressed_page_size; + first_row_index = other233.first_row_index; } -PageLocation& PageLocation::operator=(const PageLocation& other230) noexcept { - offset = other230.offset; - compressed_page_size = other230.compressed_page_size; - first_row_index = other230.first_row_index; +PageLocation& PageLocation::operator=(const PageLocation& other234) noexcept { + offset = other234.offset; + compressed_page_size = other234.compressed_page_size; + first_row_index = other234.first_row_index; return *this; } -PageLocation& PageLocation::operator=(PageLocation&& other231) noexcept { - offset = other231.offset; - compressed_page_size = other231.compressed_page_size; - first_row_index = other231.first_row_index; +PageLocation& PageLocation::operator=(PageLocation&& other235) noexcept { + offset = other235.offset; + compressed_page_size = other235.compressed_page_size; + first_row_index = other235.first_row_index; return *this; } void PageLocation::printTo(std::ostream& out) const { @@ -7046,14 +7145,14 @@ uint32_t OffsetIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->page_locations.clear(); - uint32_t _size232; - ::apache::thrift::protocol::TType _etype235; - xfer += iprot->readListBegin(_etype235, _size232); - this->page_locations.resize(_size232); - uint32_t _i236; - for (_i236 = 0; _i236 < _size232; ++_i236) + uint32_t _size236; + ::apache::thrift::protocol::TType _etype239; + xfer += iprot->readListBegin(_etype239, _size236); + this->page_locations.resize(_size236); + uint32_t _i240; + for (_i240 = 0; _i240 < _size236; ++_i240) { - xfer += this->page_locations[_i236].read(iprot); + xfer += this->page_locations[_i240].read(iprot); } xfer += iprot->readListEnd(); } @@ -7084,10 +7183,10 @@ uint32_t OffsetIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("page_locations", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->page_locations.size())); - std::vector ::const_iterator _iter237; - for (_iter237 = this->page_locations.begin(); _iter237 != this->page_locations.end(); ++_iter237) + std::vector ::const_iterator _iter241; + for (_iter241 = this->page_locations.begin(); _iter241 != this->page_locations.end(); ++_iter241) { - xfer += (*_iter237).write(oprot); + xfer += (*_iter241).write(oprot); } xfer += oprot->writeListEnd(); } @@ -7103,18 +7202,18 @@ void swap(OffsetIndex &a, OffsetIndex &b) { swap(a.page_locations, b.page_locations); } -OffsetIndex::OffsetIndex(const OffsetIndex& other238) { - page_locations = other238.page_locations; +OffsetIndex::OffsetIndex(const OffsetIndex& other242) { + page_locations = other242.page_locations; } -OffsetIndex::OffsetIndex(OffsetIndex&& other239) noexcept { - page_locations = std::move(other239.page_locations); +OffsetIndex::OffsetIndex(OffsetIndex&& other243) noexcept { + page_locations = std::move(other243.page_locations); } -OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other240) { - page_locations = other240.page_locations; +OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other244) { + page_locations = other244.page_locations; return *this; } -OffsetIndex& OffsetIndex::operator=(OffsetIndex&& other241) noexcept { - page_locations = std::move(other241.page_locations); +OffsetIndex& OffsetIndex::operator=(OffsetIndex&& other245) noexcept { + page_locations = std::move(other245.page_locations); return *this; } void OffsetIndex::printTo(std::ostream& out) const { @@ -7185,14 +7284,14 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_pages.clear(); - uint32_t _size242; - ::apache::thrift::protocol::TType _etype245; - xfer += iprot->readListBegin(_etype245, _size242); - this->null_pages.resize(_size242); - uint32_t _i246; - for (_i246 = 0; _i246 < _size242; ++_i246) + uint32_t _size246; + ::apache::thrift::protocol::TType _etype249; + xfer += iprot->readListBegin(_etype249, _size246); + this->null_pages.resize(_size246); + uint32_t _i250; + for (_i250 = 0; _i250 < _size246; ++_i250) { - xfer += iprot->readBool(this->null_pages[_i246]); + xfer += iprot->readBool(this->null_pages[_i250]); } xfer += iprot->readListEnd(); } @@ -7205,14 +7304,14 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->min_values.clear(); - uint32_t _size247; - ::apache::thrift::protocol::TType _etype250; - xfer += iprot->readListBegin(_etype250, _size247); - this->min_values.resize(_size247); - uint32_t _i251; - for (_i251 = 0; _i251 < _size247; ++_i251) + uint32_t _size251; + ::apache::thrift::protocol::TType _etype254; + xfer += iprot->readListBegin(_etype254, _size251); + this->min_values.resize(_size251); + uint32_t _i255; + for (_i255 = 0; _i255 < _size251; ++_i255) { - xfer += iprot->readBinary(this->min_values[_i251]); + xfer += iprot->readBinary(this->min_values[_i255]); } xfer += iprot->readListEnd(); } @@ -7225,14 +7324,14 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->max_values.clear(); - uint32_t _size252; - ::apache::thrift::protocol::TType _etype255; - xfer += iprot->readListBegin(_etype255, _size252); - this->max_values.resize(_size252); - uint32_t _i256; - for (_i256 = 0; _i256 < _size252; ++_i256) + uint32_t _size256; + ::apache::thrift::protocol::TType _etype259; + xfer += iprot->readListBegin(_etype259, _size256); + this->max_values.resize(_size256); + uint32_t _i260; + for (_i260 = 0; _i260 < _size256; ++_i260) { - xfer += iprot->readBinary(this->max_values[_i256]); + xfer += iprot->readBinary(this->max_values[_i260]); } xfer += iprot->readListEnd(); } @@ -7243,9 +7342,9 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast257; - xfer += iprot->readI32(ecast257); - this->boundary_order = static_cast(ecast257); + int32_t ecast261; + xfer += iprot->readI32(ecast261); + this->boundary_order = static_cast(ecast261); isset_boundary_order = true; } else { xfer += iprot->skip(ftype); @@ -7255,14 +7354,14 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_counts.clear(); - uint32_t _size258; - ::apache::thrift::protocol::TType _etype261; - xfer += iprot->readListBegin(_etype261, _size258); - this->null_counts.resize(_size258); - uint32_t _i262; - for (_i262 = 0; _i262 < _size258; ++_i262) + uint32_t _size262; + ::apache::thrift::protocol::TType _etype265; + xfer += iprot->readListBegin(_etype265, _size262); + this->null_counts.resize(_size262); + uint32_t _i266; + for (_i266 = 0; _i266 < _size262; ++_i266) { - xfer += iprot->readI64(this->null_counts[_i262]); + xfer += iprot->readI64(this->null_counts[_i266]); } xfer += iprot->readListEnd(); } @@ -7299,10 +7398,10 @@ uint32_t ColumnIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("null_pages", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_BOOL, static_cast(this->null_pages.size())); - std::vector ::const_iterator _iter263; - for (_iter263 = this->null_pages.begin(); _iter263 != this->null_pages.end(); ++_iter263) + std::vector ::const_iterator _iter267; + for (_iter267 = this->null_pages.begin(); _iter267 != this->null_pages.end(); ++_iter267) { - xfer += oprot->writeBool((*_iter263)); + xfer += oprot->writeBool((*_iter267)); } xfer += oprot->writeListEnd(); } @@ -7311,10 +7410,10 @@ uint32_t ColumnIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("min_values", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->min_values.size())); - std::vector ::const_iterator _iter264; - for (_iter264 = this->min_values.begin(); _iter264 != this->min_values.end(); ++_iter264) + std::vector ::const_iterator _iter268; + for (_iter268 = this->min_values.begin(); _iter268 != this->min_values.end(); ++_iter268) { - xfer += oprot->writeBinary((*_iter264)); + xfer += oprot->writeBinary((*_iter268)); } xfer += oprot->writeListEnd(); } @@ -7323,10 +7422,10 @@ uint32_t ColumnIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("max_values", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->max_values.size())); - std::vector ::const_iterator _iter265; - for (_iter265 = this->max_values.begin(); _iter265 != this->max_values.end(); ++_iter265) + std::vector ::const_iterator _iter269; + for (_iter269 = this->max_values.begin(); _iter269 != this->max_values.end(); ++_iter269) { - xfer += oprot->writeBinary((*_iter265)); + xfer += oprot->writeBinary((*_iter269)); } xfer += oprot->writeListEnd(); } @@ -7340,10 +7439,10 @@ uint32_t ColumnIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("null_counts", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->null_counts.size())); - std::vector ::const_iterator _iter266; - for (_iter266 = this->null_counts.begin(); _iter266 != this->null_counts.end(); ++_iter266) + std::vector ::const_iterator _iter270; + for (_iter270 = this->null_counts.begin(); _iter270 != this->null_counts.end(); ++_iter270) { - xfer += oprot->writeI64((*_iter266)); + xfer += oprot->writeI64((*_iter270)); } xfer += oprot->writeListEnd(); } @@ -7364,38 +7463,38 @@ void swap(ColumnIndex &a, ColumnIndex &b) { swap(a.__isset, b.__isset); } -ColumnIndex::ColumnIndex(const ColumnIndex& other267) { - null_pages = other267.null_pages; - min_values = other267.min_values; - max_values = other267.max_values; - boundary_order = other267.boundary_order; - null_counts = other267.null_counts; - __isset = other267.__isset; -} -ColumnIndex::ColumnIndex(ColumnIndex&& other268) noexcept { - null_pages = std::move(other268.null_pages); - min_values = std::move(other268.min_values); - max_values = std::move(other268.max_values); - boundary_order = other268.boundary_order; - null_counts = std::move(other268.null_counts); - __isset = other268.__isset; -} -ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other269) { - null_pages = other269.null_pages; - min_values = other269.min_values; - max_values = other269.max_values; - boundary_order = other269.boundary_order; - null_counts = other269.null_counts; - __isset = other269.__isset; +ColumnIndex::ColumnIndex(const ColumnIndex& other271) { + null_pages = other271.null_pages; + min_values = other271.min_values; + max_values = other271.max_values; + boundary_order = other271.boundary_order; + null_counts = other271.null_counts; + __isset = other271.__isset; +} +ColumnIndex::ColumnIndex(ColumnIndex&& other272) noexcept { + null_pages = std::move(other272.null_pages); + min_values = std::move(other272.min_values); + max_values = std::move(other272.max_values); + boundary_order = other272.boundary_order; + null_counts = std::move(other272.null_counts); + __isset = other272.__isset; +} +ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other273) { + null_pages = other273.null_pages; + min_values = other273.min_values; + max_values = other273.max_values; + boundary_order = other273.boundary_order; + null_counts = other273.null_counts; + __isset = other273.__isset; return *this; } -ColumnIndex& ColumnIndex::operator=(ColumnIndex&& other270) noexcept { - null_pages = std::move(other270.null_pages); - min_values = std::move(other270.min_values); - max_values = std::move(other270.max_values); - boundary_order = other270.boundary_order; - null_counts = std::move(other270.null_counts); - __isset = other270.__isset; +ColumnIndex& ColumnIndex::operator=(ColumnIndex&& other274) noexcept { + null_pages = std::move(other274.null_pages); + min_values = std::move(other274.min_values); + max_values = std::move(other274.max_values); + boundary_order = other274.boundary_order; + null_counts = std::move(other274.null_counts); + __isset = other274.__isset; return *this; } void ColumnIndex::printTo(std::ostream& out) const { @@ -7525,30 +7624,30 @@ void swap(AesGcmV1 &a, AesGcmV1 &b) { swap(a.__isset, b.__isset); } -AesGcmV1::AesGcmV1(const AesGcmV1& other271) { - aad_prefix = other271.aad_prefix; - aad_file_unique = other271.aad_file_unique; - supply_aad_prefix = other271.supply_aad_prefix; - __isset = other271.__isset; +AesGcmV1::AesGcmV1(const AesGcmV1& other275) { + aad_prefix = other275.aad_prefix; + aad_file_unique = other275.aad_file_unique; + supply_aad_prefix = other275.supply_aad_prefix; + __isset = other275.__isset; } -AesGcmV1::AesGcmV1(AesGcmV1&& other272) noexcept { - aad_prefix = std::move(other272.aad_prefix); - aad_file_unique = std::move(other272.aad_file_unique); - supply_aad_prefix = other272.supply_aad_prefix; - __isset = other272.__isset; +AesGcmV1::AesGcmV1(AesGcmV1&& other276) noexcept { + aad_prefix = std::move(other276.aad_prefix); + aad_file_unique = std::move(other276.aad_file_unique); + supply_aad_prefix = other276.supply_aad_prefix; + __isset = other276.__isset; } -AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other273) { - aad_prefix = other273.aad_prefix; - aad_file_unique = other273.aad_file_unique; - supply_aad_prefix = other273.supply_aad_prefix; - __isset = other273.__isset; +AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other277) { + aad_prefix = other277.aad_prefix; + aad_file_unique = other277.aad_file_unique; + supply_aad_prefix = other277.supply_aad_prefix; + __isset = other277.__isset; return *this; } -AesGcmV1& AesGcmV1::operator=(AesGcmV1&& other274) noexcept { - aad_prefix = std::move(other274.aad_prefix); - aad_file_unique = std::move(other274.aad_file_unique); - supply_aad_prefix = other274.supply_aad_prefix; - __isset = other274.__isset; +AesGcmV1& AesGcmV1::operator=(AesGcmV1&& other278) noexcept { + aad_prefix = std::move(other278.aad_prefix); + aad_file_unique = std::move(other278.aad_file_unique); + supply_aad_prefix = other278.supply_aad_prefix; + __isset = other278.__isset; return *this; } void AesGcmV1::printTo(std::ostream& out) const { @@ -7676,30 +7775,30 @@ void swap(AesGcmCtrV1 &a, AesGcmCtrV1 &b) { swap(a.__isset, b.__isset); } -AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other275) { - aad_prefix = other275.aad_prefix; - aad_file_unique = other275.aad_file_unique; - supply_aad_prefix = other275.supply_aad_prefix; - __isset = other275.__isset; +AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other279) { + aad_prefix = other279.aad_prefix; + aad_file_unique = other279.aad_file_unique; + supply_aad_prefix = other279.supply_aad_prefix; + __isset = other279.__isset; } -AesGcmCtrV1::AesGcmCtrV1(AesGcmCtrV1&& other276) noexcept { - aad_prefix = std::move(other276.aad_prefix); - aad_file_unique = std::move(other276.aad_file_unique); - supply_aad_prefix = other276.supply_aad_prefix; - __isset = other276.__isset; +AesGcmCtrV1::AesGcmCtrV1(AesGcmCtrV1&& other280) noexcept { + aad_prefix = std::move(other280.aad_prefix); + aad_file_unique = std::move(other280.aad_file_unique); + supply_aad_prefix = other280.supply_aad_prefix; + __isset = other280.__isset; } -AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other277) { - aad_prefix = other277.aad_prefix; - aad_file_unique = other277.aad_file_unique; - supply_aad_prefix = other277.supply_aad_prefix; - __isset = other277.__isset; +AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other281) { + aad_prefix = other281.aad_prefix; + aad_file_unique = other281.aad_file_unique; + supply_aad_prefix = other281.supply_aad_prefix; + __isset = other281.__isset; return *this; } -AesGcmCtrV1& AesGcmCtrV1::operator=(AesGcmCtrV1&& other278) noexcept { - aad_prefix = std::move(other278.aad_prefix); - aad_file_unique = std::move(other278.aad_file_unique); - supply_aad_prefix = other278.supply_aad_prefix; - __isset = other278.__isset; +AesGcmCtrV1& AesGcmCtrV1::operator=(AesGcmCtrV1&& other282) noexcept { + aad_prefix = std::move(other282.aad_prefix); + aad_file_unique = std::move(other282.aad_file_unique); + supply_aad_prefix = other282.supply_aad_prefix; + __isset = other282.__isset; return *this; } void AesGcmCtrV1::printTo(std::ostream& out) const { @@ -7808,26 +7907,26 @@ void swap(EncryptionAlgorithm &a, EncryptionAlgorithm &b) { swap(a.__isset, b.__isset); } -EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other279) { - AES_GCM_V1 = other279.AES_GCM_V1; - AES_GCM_CTR_V1 = other279.AES_GCM_CTR_V1; - __isset = other279.__isset; +EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other283) { + AES_GCM_V1 = other283.AES_GCM_V1; + AES_GCM_CTR_V1 = other283.AES_GCM_CTR_V1; + __isset = other283.__isset; } -EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other280) noexcept { - AES_GCM_V1 = std::move(other280.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other280.AES_GCM_CTR_V1); - __isset = other280.__isset; +EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other284) noexcept { + AES_GCM_V1 = std::move(other284.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other284.AES_GCM_CTR_V1); + __isset = other284.__isset; } -EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other281) { - AES_GCM_V1 = other281.AES_GCM_V1; - AES_GCM_CTR_V1 = other281.AES_GCM_CTR_V1; - __isset = other281.__isset; +EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other285) { + AES_GCM_V1 = other285.AES_GCM_V1; + AES_GCM_CTR_V1 = other285.AES_GCM_CTR_V1; + __isset = other285.__isset; return *this; } -EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other282) noexcept { - AES_GCM_V1 = std::move(other282.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other282.AES_GCM_CTR_V1); - __isset = other282.__isset; +EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other286) noexcept { + AES_GCM_V1 = std::move(other286.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other286.AES_GCM_CTR_V1); + __isset = other286.__isset; return *this; } void EncryptionAlgorithm::printTo(std::ostream& out) const { @@ -7927,14 +8026,14 @@ uint32_t FileMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->schema.clear(); - uint32_t _size283; - ::apache::thrift::protocol::TType _etype286; - xfer += iprot->readListBegin(_etype286, _size283); - this->schema.resize(_size283); - uint32_t _i287; - for (_i287 = 0; _i287 < _size283; ++_i287) + uint32_t _size287; + ::apache::thrift::protocol::TType _etype290; + xfer += iprot->readListBegin(_etype290, _size287); + this->schema.resize(_size287); + uint32_t _i291; + for (_i291 = 0; _i291 < _size287; ++_i291) { - xfer += this->schema[_i287].read(iprot); + xfer += this->schema[_i291].read(iprot); } xfer += iprot->readListEnd(); } @@ -7955,14 +8054,14 @@ uint32_t FileMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->row_groups.clear(); - uint32_t _size288; - ::apache::thrift::protocol::TType _etype291; - xfer += iprot->readListBegin(_etype291, _size288); - this->row_groups.resize(_size288); - uint32_t _i292; - for (_i292 = 0; _i292 < _size288; ++_i292) + uint32_t _size292; + ::apache::thrift::protocol::TType _etype295; + xfer += iprot->readListBegin(_etype295, _size292); + this->row_groups.resize(_size292); + uint32_t _i296; + for (_i296 = 0; _i296 < _size292; ++_i296) { - xfer += this->row_groups[_i292].read(iprot); + xfer += this->row_groups[_i296].read(iprot); } xfer += iprot->readListEnd(); } @@ -7975,14 +8074,14 @@ uint32_t FileMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->key_value_metadata.clear(); - uint32_t _size293; - ::apache::thrift::protocol::TType _etype296; - xfer += iprot->readListBegin(_etype296, _size293); - this->key_value_metadata.resize(_size293); - uint32_t _i297; - for (_i297 = 0; _i297 < _size293; ++_i297) + uint32_t _size297; + ::apache::thrift::protocol::TType _etype300; + xfer += iprot->readListBegin(_etype300, _size297); + this->key_value_metadata.resize(_size297); + uint32_t _i301; + for (_i301 = 0; _i301 < _size297; ++_i301) { - xfer += this->key_value_metadata[_i297].read(iprot); + xfer += this->key_value_metadata[_i301].read(iprot); } xfer += iprot->readListEnd(); } @@ -8003,14 +8102,14 @@ uint32_t FileMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->column_orders.clear(); - uint32_t _size298; - ::apache::thrift::protocol::TType _etype301; - xfer += iprot->readListBegin(_etype301, _size298); - this->column_orders.resize(_size298); - uint32_t _i302; - for (_i302 = 0; _i302 < _size298; ++_i302) + uint32_t _size302; + ::apache::thrift::protocol::TType _etype305; + xfer += iprot->readListBegin(_etype305, _size302); + this->column_orders.resize(_size302); + uint32_t _i306; + for (_i306 = 0; _i306 < _size302; ++_i306) { - xfer += this->column_orders[_i302].read(iprot); + xfer += this->column_orders[_i306].read(iprot); } xfer += iprot->readListEnd(); } @@ -8067,10 +8166,10 @@ uint32_t FileMetaData::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("schema", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->schema.size())); - std::vector ::const_iterator _iter303; - for (_iter303 = this->schema.begin(); _iter303 != this->schema.end(); ++_iter303) + std::vector ::const_iterator _iter307; + for (_iter307 = this->schema.begin(); _iter307 != this->schema.end(); ++_iter307) { - xfer += (*_iter303).write(oprot); + xfer += (*_iter307).write(oprot); } xfer += oprot->writeListEnd(); } @@ -8083,10 +8182,10 @@ uint32_t FileMetaData::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("row_groups", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->row_groups.size())); - std::vector ::const_iterator _iter304; - for (_iter304 = this->row_groups.begin(); _iter304 != this->row_groups.end(); ++_iter304) + std::vector ::const_iterator _iter308; + for (_iter308 = this->row_groups.begin(); _iter308 != this->row_groups.end(); ++_iter308) { - xfer += (*_iter304).write(oprot); + xfer += (*_iter308).write(oprot); } xfer += oprot->writeListEnd(); } @@ -8096,10 +8195,10 @@ uint32_t FileMetaData::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter305; - for (_iter305 = this->key_value_metadata.begin(); _iter305 != this->key_value_metadata.end(); ++_iter305) + std::vector ::const_iterator _iter309; + for (_iter309 = this->key_value_metadata.begin(); _iter309 != this->key_value_metadata.end(); ++_iter309) { - xfer += (*_iter305).write(oprot); + xfer += (*_iter309).write(oprot); } xfer += oprot->writeListEnd(); } @@ -8114,10 +8213,10 @@ uint32_t FileMetaData::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("column_orders", ::apache::thrift::protocol::T_LIST, 7); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->column_orders.size())); - std::vector ::const_iterator _iter306; - for (_iter306 = this->column_orders.begin(); _iter306 != this->column_orders.end(); ++_iter306) + std::vector ::const_iterator _iter310; + for (_iter310 = this->column_orders.begin(); _iter310 != this->column_orders.end(); ++_iter310) { - xfer += (*_iter306).write(oprot); + xfer += (*_iter310).write(oprot); } xfer += oprot->writeListEnd(); } @@ -8152,54 +8251,54 @@ void swap(FileMetaData &a, FileMetaData &b) { swap(a.__isset, b.__isset); } -FileMetaData::FileMetaData(const FileMetaData& other307) { - version = other307.version; - schema = other307.schema; - num_rows = other307.num_rows; - row_groups = other307.row_groups; - key_value_metadata = other307.key_value_metadata; - created_by = other307.created_by; - column_orders = other307.column_orders; - encryption_algorithm = other307.encryption_algorithm; - footer_signing_key_metadata = other307.footer_signing_key_metadata; - __isset = other307.__isset; -} -FileMetaData::FileMetaData(FileMetaData&& other308) noexcept { - version = other308.version; - schema = std::move(other308.schema); - num_rows = other308.num_rows; - row_groups = std::move(other308.row_groups); - key_value_metadata = std::move(other308.key_value_metadata); - created_by = std::move(other308.created_by); - column_orders = std::move(other308.column_orders); - encryption_algorithm = std::move(other308.encryption_algorithm); - footer_signing_key_metadata = std::move(other308.footer_signing_key_metadata); - __isset = other308.__isset; -} -FileMetaData& FileMetaData::operator=(const FileMetaData& other309) { - version = other309.version; - schema = other309.schema; - num_rows = other309.num_rows; - row_groups = other309.row_groups; - key_value_metadata = other309.key_value_metadata; - created_by = other309.created_by; - column_orders = other309.column_orders; - encryption_algorithm = other309.encryption_algorithm; - footer_signing_key_metadata = other309.footer_signing_key_metadata; - __isset = other309.__isset; +FileMetaData::FileMetaData(const FileMetaData& other311) { + version = other311.version; + schema = other311.schema; + num_rows = other311.num_rows; + row_groups = other311.row_groups; + key_value_metadata = other311.key_value_metadata; + created_by = other311.created_by; + column_orders = other311.column_orders; + encryption_algorithm = other311.encryption_algorithm; + footer_signing_key_metadata = other311.footer_signing_key_metadata; + __isset = other311.__isset; +} +FileMetaData::FileMetaData(FileMetaData&& other312) noexcept { + version = other312.version; + schema = std::move(other312.schema); + num_rows = other312.num_rows; + row_groups = std::move(other312.row_groups); + key_value_metadata = std::move(other312.key_value_metadata); + created_by = std::move(other312.created_by); + column_orders = std::move(other312.column_orders); + encryption_algorithm = std::move(other312.encryption_algorithm); + footer_signing_key_metadata = std::move(other312.footer_signing_key_metadata); + __isset = other312.__isset; +} +FileMetaData& FileMetaData::operator=(const FileMetaData& other313) { + version = other313.version; + schema = other313.schema; + num_rows = other313.num_rows; + row_groups = other313.row_groups; + key_value_metadata = other313.key_value_metadata; + created_by = other313.created_by; + column_orders = other313.column_orders; + encryption_algorithm = other313.encryption_algorithm; + footer_signing_key_metadata = other313.footer_signing_key_metadata; + __isset = other313.__isset; return *this; } -FileMetaData& FileMetaData::operator=(FileMetaData&& other310) noexcept { - version = other310.version; - schema = std::move(other310.schema); - num_rows = other310.num_rows; - row_groups = std::move(other310.row_groups); - key_value_metadata = std::move(other310.key_value_metadata); - created_by = std::move(other310.created_by); - column_orders = std::move(other310.column_orders); - encryption_algorithm = std::move(other310.encryption_algorithm); - footer_signing_key_metadata = std::move(other310.footer_signing_key_metadata); - __isset = other310.__isset; +FileMetaData& FileMetaData::operator=(FileMetaData&& other314) noexcept { + version = other314.version; + schema = std::move(other314.schema); + num_rows = other314.num_rows; + row_groups = std::move(other314.row_groups); + key_value_metadata = std::move(other314.key_value_metadata); + created_by = std::move(other314.created_by); + column_orders = std::move(other314.column_orders); + encryption_algorithm = std::move(other314.encryption_algorithm); + footer_signing_key_metadata = std::move(other314.footer_signing_key_metadata); + __isset = other314.__isset; return *this; } void FileMetaData::printTo(std::ostream& out) const { @@ -8315,26 +8414,26 @@ void swap(FileCryptoMetaData &a, FileCryptoMetaData &b) { swap(a.__isset, b.__isset); } -FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other311) { - encryption_algorithm = other311.encryption_algorithm; - key_metadata = other311.key_metadata; - __isset = other311.__isset; +FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other315) { + encryption_algorithm = other315.encryption_algorithm; + key_metadata = other315.key_metadata; + __isset = other315.__isset; } -FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other312) noexcept { - encryption_algorithm = std::move(other312.encryption_algorithm); - key_metadata = std::move(other312.key_metadata); - __isset = other312.__isset; +FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other316) noexcept { + encryption_algorithm = std::move(other316.encryption_algorithm); + key_metadata = std::move(other316.key_metadata); + __isset = other316.__isset; } -FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other313) { - encryption_algorithm = other313.encryption_algorithm; - key_metadata = other313.key_metadata; - __isset = other313.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other317) { + encryption_algorithm = other317.encryption_algorithm; + key_metadata = other317.key_metadata; + __isset = other317.__isset; return *this; } -FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other314) noexcept { - encryption_algorithm = std::move(other314.encryption_algorithm); - key_metadata = std::move(other314.key_metadata); - __isset = other314.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other318) noexcept { + encryption_algorithm = std::move(other318.encryption_algorithm); + key_metadata = std::move(other318.key_metadata); + __isset = other318.__isset; return *this; } void FileCryptoMetaData::printTo(std::ostream& out) const { diff --git a/cpp/src/generated/parquet_types.h b/cpp/src/generated/parquet_types.h index 9f468b5051db3..199b4ae747667 100644 --- a/cpp/src/generated/parquet_types.h +++ b/cpp/src/generated/parquet_types.h @@ -359,6 +359,8 @@ class EnumType; class DateType; +class Float16Type; + class NullType; class DecimalType; @@ -770,6 +772,39 @@ void swap(DateType &a, DateType &b); std::ostream& operator<<(std::ostream& out, const DateType& obj); +class Float16Type : public virtual ::apache::thrift::TBase { + public: + + Float16Type(const Float16Type&) noexcept; + Float16Type(Float16Type&&) noexcept; + Float16Type& operator=(const Float16Type&) noexcept; + Float16Type& operator=(Float16Type&&) noexcept; + Float16Type() noexcept { + } + + virtual ~Float16Type() noexcept; + + bool operator == (const Float16Type & /* rhs */) const + { + return true; + } + bool operator != (const Float16Type &rhs) const { + return !(*this == rhs); + } + + bool operator < (const Float16Type & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override; + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override; + + virtual void printTo(std::ostream& out) const; +}; + +void swap(Float16Type &a, Float16Type &b); + +std::ostream& operator<<(std::ostream& out, const Float16Type& obj); + + /** * Logical type to annotate a column that is always null. * @@ -1253,7 +1288,7 @@ void swap(BsonType &a, BsonType &b); std::ostream& operator<<(std::ostream& out, const BsonType& obj); typedef struct _LogicalType__isset { - _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false) {} + _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false), FLOAT16(false) {} bool STRING :1; bool MAP :1; bool LIST :1; @@ -1267,6 +1302,7 @@ typedef struct _LogicalType__isset { bool JSON :1; bool BSON :1; bool UUID :1; + bool FLOAT16 :1; } _LogicalType__isset; /** @@ -1300,6 +1336,7 @@ class LogicalType : public virtual ::apache::thrift::TBase { JsonType JSON; BsonType BSON; UUIDType UUID; + Float16Type FLOAT16; _LogicalType__isset __isset; @@ -1329,6 +1366,8 @@ class LogicalType : public virtual ::apache::thrift::TBase { void __set_UUID(const UUIDType& val); + void __set_FLOAT16(const Float16Type& val); + bool operator == (const LogicalType & rhs) const { if (__isset.STRING != rhs.__isset.STRING) @@ -1383,6 +1422,10 @@ class LogicalType : public virtual ::apache::thrift::TBase { return false; else if (__isset.UUID && !(UUID == rhs.UUID)) return false; + if (__isset.FLOAT16 != rhs.__isset.FLOAT16) + return false; + else if (__isset.FLOAT16 && !(FLOAT16 == rhs.FLOAT16)) + return false; return true; } bool operator != (const LogicalType &rhs) const { diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index 4e23d0fab5c69..fb9e53870583c 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -143,6 +143,8 @@ std::shared_ptr get_logical_type(const DataType& type) { return LogicalType::Date(); case ArrowId::DATE64: return LogicalType::Date(); + case ArrowId::HALF_FLOAT: + return LogicalType::Float16(); case ArrowId::TIMESTAMP: { const auto& ts_type = static_cast(type); const bool adjusted_to_utc = !(ts_type.timezone().empty()); @@ -220,6 +222,7 @@ ParquetType::type get_physical_type(const DataType& type) { case ArrowId::FIXED_SIZE_BINARY: case ArrowId::DECIMAL128: case ArrowId::DECIMAL256: + case ArrowId::HALF_FLOAT: return ParquetType::FIXED_LEN_BYTE_ARRAY; case ArrowId::DATE32: return ParquetType::INT32; @@ -525,6 +528,9 @@ static std::shared_ptr MakeSimpleSchema(const DataType& type, byte_width = static_cast(values_type).byte_width(); break; + case ::arrow::Type::HALF_FLOAT: + byte_width = sizeof(::arrow::HalfFloatType::c_type); + break; case ::arrow::Type::DECIMAL128: case ::arrow::Type::DECIMAL256: { const auto& decimal_type = static_cast(values_type); @@ -537,6 +543,9 @@ static std::shared_ptr MakeSimpleSchema(const DataType& type, case ::arrow::Type::FIXED_SIZE_BINARY: byte_width = static_cast(type).byte_width(); break; + case ::arrow::Type::HALF_FLOAT: + byte_width = sizeof(::arrow::HalfFloatType::c_type); + break; case ::arrow::Type::DECIMAL128: case ::arrow::Type::DECIMAL256: { const auto& decimal_type = static_cast(type); @@ -840,12 +849,12 @@ typedef ::testing::Types< ::arrow::BooleanType, ::arrow::UInt8Type, ::arrow::Int8Type, ::arrow::UInt16Type, ::arrow::Int16Type, ::arrow::Int32Type, ::arrow::UInt64Type, ::arrow::Int64Type, ::arrow::Date32Type, ::arrow::FloatType, ::arrow::DoubleType, ::arrow::StringType, - ::arrow::BinaryType, ::arrow::FixedSizeBinaryType, DecimalWithPrecisionAndScale<1>, - DecimalWithPrecisionAndScale<5>, DecimalWithPrecisionAndScale<10>, - DecimalWithPrecisionAndScale<19>, DecimalWithPrecisionAndScale<23>, - DecimalWithPrecisionAndScale<27>, DecimalWithPrecisionAndScale<38>, - Decimal256WithPrecisionAndScale<39>, Decimal256WithPrecisionAndScale<56>, - Decimal256WithPrecisionAndScale<76>> + ::arrow::BinaryType, ::arrow::FixedSizeBinaryType, ::arrow::HalfFloatType, + DecimalWithPrecisionAndScale<1>, DecimalWithPrecisionAndScale<5>, + DecimalWithPrecisionAndScale<10>, DecimalWithPrecisionAndScale<19>, + DecimalWithPrecisionAndScale<23>, DecimalWithPrecisionAndScale<27>, + DecimalWithPrecisionAndScale<38>, Decimal256WithPrecisionAndScale<39>, + Decimal256WithPrecisionAndScale<56>, Decimal256WithPrecisionAndScale<76>> TestTypes; TYPED_TEST_SUITE(TestParquetIO, TestTypes); @@ -916,9 +925,15 @@ TYPED_TEST(TestParquetIO, SingleColumnOptionalReadWrite) { } TYPED_TEST(TestParquetIO, SingleColumnOptionalDictionaryWrite) { - // Skip tests for BOOL as we don't create dictionaries for it. - if (TypeParam::type_id == ::arrow::Type::BOOL) { - return; + switch (TypeParam::type_id) { + case ::arrow::Type::BOOL: + GTEST_SKIP() << "dictionaries not created for BOOL"; + break; + case ::arrow::Type::HALF_FLOAT: + GTEST_SKIP() << "dictionary_encode not supported for HALF_FLOAT"; + break; + default: + break; } std::shared_ptr values; diff --git a/cpp/src/parquet/arrow/arrow_schema_test.cc b/cpp/src/parquet/arrow/arrow_schema_test.cc index f11101eb24298..5443214f930d7 100644 --- a/cpp/src/parquet/arrow/arrow_schema_test.cc +++ b/cpp/src/parquet/arrow/arrow_schema_test.cc @@ -236,6 +236,8 @@ TEST_F(TestConvertParquetSchema, ParquetAnnotatedFields) { ::arrow::fixed_size_binary(12)}, {"uuid", LogicalType::UUID(), ParquetType::FIXED_LEN_BYTE_ARRAY, 16, ::arrow::fixed_size_binary(16)}, + {"float16", LogicalType::Float16(), ParquetType::FIXED_LEN_BYTE_ARRAY, 2, + ::arrow::float16()}, {"none", LogicalType::None(), ParquetType::BOOLEAN, -1, ::arrow::boolean()}, {"none", LogicalType::None(), ParquetType::INT32, -1, ::arrow::int32()}, {"none", LogicalType::None(), ParquetType::INT64, -1, ::arrow::int64()}, @@ -851,6 +853,8 @@ TEST_F(TestConvertArrowSchema, ArrowFields) { ParquetType::FIXED_LEN_BYTE_ARRAY, 7}, {"decimal(32, 8)", ::arrow::decimal(32, 8), LogicalType::Decimal(32, 8), ParquetType::FIXED_LEN_BYTE_ARRAY, 14}, + {"float16", ::arrow::float16(), LogicalType::Float16(), + ParquetType::FIXED_LEN_BYTE_ARRAY, 2}, {"time32", ::arrow::time32(::arrow::TimeUnit::MILLI), LogicalType::Time(true, LogicalType::TimeUnit::MILLIS), ParquetType::INT32, -1}, {"time64(microsecond)", ::arrow::time64(::arrow::TimeUnit::MICRO), @@ -913,7 +917,8 @@ TEST_F(TestConvertArrowSchema, ArrowNonconvertibleFields) { }; std::vector cases = { - {"float16", ::arrow::float16()}, + {"run_end_encoded", + ::arrow::run_end_encoded(::arrow::int32(), ::arrow::list(::arrow::int8()))}, }; for (const FieldConstructionArguments& c : cases) { diff --git a/cpp/src/parquet/arrow/reader_internal.cc b/cpp/src/parquet/arrow/reader_internal.cc index 5146aa12c2c36..e5aef5a45b5f3 100644 --- a/cpp/src/parquet/arrow/reader_internal.cc +++ b/cpp/src/parquet/arrow/reader_internal.cc @@ -42,6 +42,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/checked_cast.h" #include "arrow/util/endian.h" +#include "arrow/util/float16.h" #include "arrow/util/int_util_overflow.h" #include "arrow/util/logging.h" #include "arrow/util/ubsan.h" @@ -82,6 +83,7 @@ using ::arrow::bit_util::FromBigEndian; using ::arrow::internal::checked_cast; using ::arrow::internal::checked_pointer_cast; using ::arrow::internal::SafeLeftShift; +using ::arrow::util::Float16; using ::arrow::util::SafeLoadAs; using parquet::internal::BinaryRecordReader; @@ -713,6 +715,17 @@ Status TransferDecimal(RecordReader* reader, MemoryPool* pool, return Status::OK(); } +Status TransferHalfFloat(RecordReader* reader, MemoryPool* pool, + const std::shared_ptr& field, Datum* out) { + static const auto binary_type = ::arrow::fixed_size_binary(2); + // Read as a FixedSizeBinaryArray - then, view as a HalfFloatArray + std::shared_ptr chunked_array; + RETURN_NOT_OK( + TransferBinary(reader, pool, field->WithType(binary_type), &chunked_array)); + ARROW_ASSIGN_OR_RAISE(*out, chunked_array->View(field->type())); + return Status::OK(); +} + } // namespace #define TRANSFER_INT32(ENUM, ArrowType) \ @@ -772,6 +785,18 @@ Status TransferColumnData(RecordReader* reader, const std::shared_ptr& va RETURN_NOT_OK(TransferBinary(reader, pool, value_field, &chunked_result)); result = chunked_result; } break; + case ::arrow::Type::HALF_FLOAT: { + const auto& type = *value_field->type(); + if (descr->physical_type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { + return Status::Invalid("Physical type for ", type.ToString(), + " must be fixed length binary"); + } + if (descr->type_length() != type.byte_width()) { + return Status::Invalid("Fixed length binary type for ", type.ToString(), + " must have a byte width of ", type.byte_width()); + } + RETURN_NOT_OK(TransferHalfFloat(reader, pool, value_field, &result)); + } break; case ::arrow::Type::DECIMAL128: { switch (descr->physical_type()) { case ::parquet::Type::INT32: { diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc index 3323b7ff8b608..f5484f131eb07 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc @@ -397,6 +397,11 @@ Status FieldToNode(const std::string& name, const std::shared_ptr& field, case ArrowTypeId::DURATION: type = ParquetType::INT64; break; + case ArrowTypeId::HALF_FLOAT: + type = ParquetType::FIXED_LEN_BYTE_ARRAY; + logical_type = LogicalType::Float16(); + length = sizeof(uint16_t); + break; case ArrowTypeId::STRUCT: { auto struct_type = std::static_pointer_cast<::arrow::StructType>(field->type()); return StructToNode(struct_type, name, field->nullable(), field_id, properties, diff --git a/cpp/src/parquet/arrow/schema_internal.cc b/cpp/src/parquet/arrow/schema_internal.cc index da0427cb31000..bb75cce084097 100644 --- a/cpp/src/parquet/arrow/schema_internal.cc +++ b/cpp/src/parquet/arrow/schema_internal.cc @@ -130,6 +130,8 @@ Result> FromFLBA(const LogicalType& logical_type, switch (logical_type.type()) { case LogicalType::Type::DECIMAL: return MakeArrowDecimal(logical_type); + case LogicalType::Type::FLOAT16: + return ::arrow::float16(); case LogicalType::Type::NONE: case LogicalType::Type::INTERVAL: case LogicalType::Type::UUID: diff --git a/cpp/src/parquet/arrow/test_util.h b/cpp/src/parquet/arrow/test_util.h index 16c03130c9672..b2be1b3c5354d 100644 --- a/cpp/src/parquet/arrow/test_util.h +++ b/cpp/src/parquet/arrow/test_util.h @@ -33,7 +33,9 @@ #include "arrow/type_fwd.h" #include "arrow/type_traits.h" #include "arrow/util/decimal.h" +#include "arrow/util/float16.h" #include "parquet/column_reader.h" +#include "parquet/test_util.h" namespace parquet { @@ -70,7 +72,14 @@ ::arrow::enable_if_floating_point NonNullArray( size_t size, std::shared_ptr* out) { using c_type = typename ArrowType::c_type; std::vector values; - ::arrow::random_real(size, 0, static_cast(0), static_cast(1), &values); + if constexpr (::arrow::is_half_float_type::value) { + values.resize(size); + test::random_float16_numbers(static_cast(size), 0, ::arrow::util::Float16(0.0f), + ::arrow::util::Float16(1.0f), values.data()); + } else { + ::arrow::random_real(size, 0, static_cast(0), static_cast(1), + &values); + } ::arrow::NumericBuilder builder; RETURN_NOT_OK(builder.AppendValues(values.data(), values.size())); return builder.Finish(out); @@ -201,8 +210,14 @@ ::arrow::enable_if_floating_point NullableArray( size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr* out) { using c_type = typename ArrowType::c_type; std::vector values; - ::arrow::random_real(size, seed, static_cast(-1e10), static_cast(1e10), - &values); + if constexpr (::arrow::is_half_float_type::value) { + values.resize(size); + test::random_float16_numbers(static_cast(size), 0, ::arrow::util::Float16(-1e4f), + ::arrow::util::Float16(1e4f), values.data()); + } else { + ::arrow::random_real(size, seed, static_cast(-1e10), + static_cast(1e10), &values); + } std::vector valid_bytes(size, 1); for (size_t i = 0; i < num_nulls; i++) { diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 5dff533c1cce2..a7e7b2f93e174 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -39,6 +39,7 @@ #include "arrow/util/compression.h" #include "arrow/util/crc32.h" #include "arrow/util/endian.h" +#include "arrow/util/float16.h" #include "arrow/util/logging.h" #include "arrow/util/rle_encoding.h" #include "arrow/util/type_traits.h" @@ -65,6 +66,7 @@ using arrow::Status; using arrow::bit_util::BitWriter; using arrow::internal::checked_cast; using arrow::internal::checked_pointer_cast; +using arrow::util::Float16; using arrow::util::RleEncoder; namespace bit_util = arrow::bit_util; @@ -2295,6 +2297,33 @@ struct SerializeFunctor< int64_t* scratch; }; +// ---------------------------------------------------------------------- +// Write Arrow to Float16 + +// Requires a custom serializer because Float16s in Parquet are stored as a 2-byte +// (little-endian) FLBA, whereas in Arrow they're a native `uint16_t`. +template <> +struct SerializeFunctor<::parquet::FLBAType, ::arrow::HalfFloatType> { + Status Serialize(const ::arrow::HalfFloatArray& array, ArrowWriteContext*, FLBA* out) { + const uint16_t* values = array.raw_values(); + if (array.null_count() == 0) { + for (int64_t i = 0; i < array.length(); ++i) { + out[i] = ToFLBA(&values[i]); + } + } else { + for (int64_t i = 0; i < array.length(); ++i) { + out[i] = array.IsValid(i) ? ToFLBA(&values[i]) : FLBA{}; + } + } + return Status::OK(); + } + + private: + FLBA ToFLBA(const uint16_t* value_ptr) const { + return FLBA{reinterpret_cast(value_ptr)}; + } +}; + template <> Status TypedColumnWriterImpl::WriteArrowDense( const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels, @@ -2303,6 +2332,7 @@ Status TypedColumnWriterImpl::WriteArrowDense( WRITE_SERIALIZE_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryType, FLBAType) WRITE_SERIALIZE_CASE(DECIMAL128, Decimal128Type, FLBAType) WRITE_SERIALIZE_CASE(DECIMAL256, Decimal256Type, FLBAType) + WRITE_SERIALIZE_CASE(HALF_FLOAT, HalfFloatType, FLBAType) default: break; } diff --git a/cpp/src/parquet/page_index_test.cc b/cpp/src/parquet/page_index_test.cc index 5bfe38522af7b..4db49b4267415 100644 --- a/cpp/src/parquet/page_index_test.cc +++ b/cpp/src/parquet/page_index_test.cc @@ -21,6 +21,7 @@ #include #include "arrow/io/file.h" +#include "arrow/util/float16.h" #include "parquet/file_reader.h" #include "parquet/metadata.h" #include "parquet/schema.h" @@ -579,6 +580,27 @@ TEST(PageIndex, WriteFLBAColumnIndex) { /*has_null_counts=*/false); } +TEST(PageIndex, WriteFloat16ColumnIndex) { + using ::arrow::util::Float16; + auto encode = [](auto value) { + auto bytes = Float16(value).ToLittleEndian(); + return std::string(reinterpret_cast(bytes.data()), bytes.size()); + }; + + // Float16 (FLBA) values in the ascending order and without null count. + std::vector page_stats(4); + page_stats.at(0).set_min(encode(-1.3)).set_max(encode(+3.6)); + page_stats.at(1).set_min(encode(-0.2)).set_max(encode(+4.5)); + page_stats.at(2).set_min(encode(+1.1)).set_max(encode(+5.4)); + page_stats.at(3).set_min(encode(+2.0)).set_max(encode(+6.3)); + + auto node = schema::PrimitiveNode::Make( + "c1", Repetition::OPTIONAL, LogicalType::Float16(), Type::FIXED_LEN_BYTE_ARRAY, + /*length=*/2); + TestWriteTypedColumnIndex(std::move(node), page_stats, BoundaryOrder::Ascending, + /*has_null_counts=*/false); +} + TEST(PageIndex, WriteColumnIndexWithAllNullPages) { // All values are null. std::vector page_stats(3); diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift index 88e44c96cc24c..d802166be66e8 100644 --- a/cpp/src/parquet/parquet.thrift +++ b/cpp/src/parquet/parquet.thrift @@ -234,6 +234,7 @@ struct MapType {} // see LogicalTypes.md struct ListType {} // see LogicalTypes.md struct EnumType {} // allowed for BINARY, must be encoded with UTF-8 struct DateType {} // allowed for INT32 +struct Float16Type{} // allowed for FIXED[2], must encode raw FLOAT16 bytes /** * Logical type to annotate a column that is always null. @@ -344,6 +345,7 @@ union LogicalType { 12: JsonType JSON // use ConvertedType JSON 13: BsonType BSON // use ConvertedType BSON 14: UUIDType UUID // no compatible ConvertedType + 15: Float16Type FLOAT16 // no compatible ConvertedType } /** diff --git a/cpp/src/parquet/schema_test.cc b/cpp/src/parquet/schema_test.cc index 603d9ed8e2124..a1b5557497d9c 100644 --- a/cpp/src/parquet/schema_test.cc +++ b/cpp/src/parquet/schema_test.cc @@ -1147,6 +1147,9 @@ TEST(TestLogicalTypeConstruction, NewTypeIncompatibility) { auto check_is_UUID = [](const std::shared_ptr& logical_type) { return logical_type->is_UUID(); }; + auto check_is_float16 = [](const std::shared_ptr& logical_type) { + return logical_type->is_float16(); + }; auto check_is_null = [](const std::shared_ptr& logical_type) { return logical_type->is_null(); }; @@ -1159,6 +1162,7 @@ TEST(TestLogicalTypeConstruction, NewTypeIncompatibility) { std::vector cases = { {LogicalType::UUID(), check_is_UUID}, + {LogicalType::Float16(), check_is_float16}, {LogicalType::Null(), check_is_null}, {LogicalType::Time(false, LogicalType::TimeUnit::MILLIS), check_is_time}, {LogicalType::Time(false, LogicalType::TimeUnit::MICROS), check_is_time}, @@ -1242,6 +1246,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeProperties) { {JSONLogicalType::Make(), false, true, true}, {BSONLogicalType::Make(), false, true, true}, {UUIDLogicalType::Make(), false, true, true}, + {Float16LogicalType::Make(), false, true, true}, {NoLogicalType::Make(), false, false, true}, }; @@ -1351,7 +1356,8 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) { int physical_length; }; - std::vector inapplicable_types = {{Type::FIXED_LEN_BYTE_ARRAY, 8}, + std::vector inapplicable_types = {{Type::FIXED_LEN_BYTE_ARRAY, 1}, + {Type::FIXED_LEN_BYTE_ARRAY, 8}, {Type::FIXED_LEN_BYTE_ARRAY, 20}, {Type::BOOLEAN, -1}, {Type::INT32, -1}, @@ -1374,6 +1380,12 @@ TEST(TestLogicalTypeOperation, LogicalTypeApplicability) { for (const InapplicableType& t : inapplicable_types) { ASSERT_FALSE(logical_type->is_applicable(t.physical_type, t.physical_length)); } + + logical_type = LogicalType::Float16(); + ASSERT_TRUE(logical_type->is_applicable(Type::FIXED_LEN_BYTE_ARRAY, 2)); + for (const InapplicableType& t : inapplicable_types) { + ASSERT_FALSE(logical_type->is_applicable(t.physical_type, t.physical_length)); + } } TEST(TestLogicalTypeOperation, DecimalLogicalTypeApplicability) { @@ -1531,6 +1543,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) { {LogicalType::JSON(), "JSON", R"({"Type": "JSON"})"}, {LogicalType::BSON(), "BSON", R"({"Type": "BSON"})"}, {LogicalType::UUID(), "UUID", R"({"Type": "UUID"})"}, + {LogicalType::Float16(), "Float16", R"({"Type": "Float16"})"}, {LogicalType::None(), "None", R"({"Type": "None"})"}, }; @@ -1580,6 +1593,7 @@ TEST(TestLogicalTypeOperation, LogicalTypeSortOrder) { {LogicalType::JSON(), SortOrder::UNSIGNED}, {LogicalType::BSON(), SortOrder::UNSIGNED}, {LogicalType::UUID(), SortOrder::UNSIGNED}, + {LogicalType::Float16(), SortOrder::SIGNED}, {LogicalType::None(), SortOrder::UNKNOWN}}; for (const ExpectedSortOrder& c : cases) { @@ -1712,6 +1726,15 @@ TEST(TestSchemaNodeCreation, FactoryExceptions) { ASSERT_ANY_THROW(PrimitiveNode::Make("uuid", Repetition::REQUIRED, UUIDLogicalType::Make(), Type::FIXED_LEN_BYTE_ARRAY, 64)); + + // Incompatible primitive type ... + ASSERT_ANY_THROW(PrimitiveNode::Make("float16", Repetition::REQUIRED, + Float16LogicalType::Make(), Type::BYTE_ARRAY, 2)); + // Incompatible primitive length ... + ASSERT_ANY_THROW(PrimitiveNode::Make("float16", Repetition::REQUIRED, + Float16LogicalType::Make(), + Type::FIXED_LEN_BYTE_ARRAY, 3)); + // Non-positive length argument for fixed length binary ... ASSERT_ANY_THROW(PrimitiveNode::Make("negative_length", Repetition::REQUIRED, NoLogicalType::Make(), Type::FIXED_LEN_BYTE_ARRAY, @@ -1902,6 +1925,9 @@ TEST_F(TestSchemaElementConstruction, SimpleCases) { [this]() { return element_->logicalType.__isset.BSON; }}, {"uuid", LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16, false, ConvertedType::NA, true, [this]() { return element_->logicalType.__isset.UUID; }}, + {"float16", LogicalType::Float16(), Type::FIXED_LEN_BYTE_ARRAY, 2, false, + ConvertedType::NA, true, + [this]() { return element_->logicalType.__isset.FLOAT16; }}, {"none", LogicalType::None(), Type::INT64, -1, false, ConvertedType::NA, false, check_nothing}}; @@ -2238,6 +2264,7 @@ TEST(TestLogicalTypeSerialization, Roundtrips) { {LogicalType::JSON(), Type::BYTE_ARRAY, -1}, {LogicalType::BSON(), Type::BYTE_ARRAY, -1}, {LogicalType::UUID(), Type::FIXED_LEN_BYTE_ARRAY, 16}, + {LogicalType::Float16(), Type::FIXED_LEN_BYTE_ARRAY, 2}, {LogicalType::None(), Type::BOOLEAN, -1}}; for (const AnnotatedPrimitiveNodeFactoryArguments& c : cases) { diff --git a/cpp/src/parquet/statistics.cc b/cpp/src/parquet/statistics.cc index ccfb69c487d40..37b245e0dd6c2 100644 --- a/cpp/src/parquet/statistics.cc +++ b/cpp/src/parquet/statistics.cc @@ -30,6 +30,7 @@ #include "arrow/type_traits.h" #include "arrow/util/bit_run_reader.h" #include "arrow/util/checked_cast.h" +#include "arrow/util/float16.h" #include "arrow/util/logging.h" #include "arrow/util/ubsan.h" #include "arrow/visit_data_inline.h" @@ -41,6 +42,7 @@ using arrow::default_memory_pool; using arrow::MemoryPool; using arrow::internal::checked_cast; +using arrow::util::Float16; using arrow::util::SafeCopy; using arrow::util::SafeLoad; @@ -53,6 +55,23 @@ namespace { constexpr int value_length(int value_length, const ByteArray& value) { return value.len; } constexpr int value_length(int type_length, const FLBA& value) { return type_length; } +// Static "constants" for normalizing float16 min/max values. These need to be expressed +// as pointers because `Float16LogicalType` represents an FLBA. +struct Float16Constants { + static constexpr const uint8_t* lowest() { return lowest_.data(); } + static constexpr const uint8_t* max() { return max_.data(); } + static constexpr const uint8_t* positive_zero() { return positive_zero_.data(); } + static constexpr const uint8_t* negative_zero() { return negative_zero_.data(); } + + private: + using Bytes = std::array; + static constexpr Bytes lowest_ = + std::numeric_limits::lowest().ToLittleEndian(); + static constexpr Bytes max_ = std::numeric_limits::max().ToLittleEndian(); + static constexpr Bytes positive_zero_ = (+Float16::FromBits(0)).ToLittleEndian(); + static constexpr Bytes negative_zero_ = (-Float16::FromBits(0)).ToLittleEndian(); +}; + template struct CompareHelper { using T = typename DType::c_type; @@ -277,11 +296,43 @@ template struct CompareHelper : public BinaryLikeCompareHelperBase {}; +template <> +struct CompareHelper { + using T = FLBA; + + static T DefaultMin() { return T{Float16Constants::max()}; } + static T DefaultMax() { return T{Float16Constants::lowest()}; } + + static T Coalesce(T val, T fallback) { + return (val.ptr == nullptr || Float16::FromLittleEndian(val.ptr).is_nan()) ? fallback + : val; + } + + static inline bool Compare(int type_length, const T& a, const T& b) { + const auto lhs = Float16::FromLittleEndian(a.ptr); + const auto rhs = Float16::FromLittleEndian(b.ptr); + // NaN is handled here (same behavior as native float compare) + return lhs < rhs; + } + + static T Min(int type_length, const T& a, const T& b) { + if (a.ptr == nullptr) return b; + if (b.ptr == nullptr) return a; + return Compare(type_length, a, b) ? a : b; + } + + static T Max(int type_length, const T& a, const T& b) { + if (a.ptr == nullptr) return b; + if (b.ptr == nullptr) return a; + return Compare(type_length, a, b) ? b : a; + } +}; + using ::std::optional; template ::arrow::enable_if_t::value, optional>> -CleanStatistic(std::pair min_max) { +CleanStatistic(std::pair min_max, LogicalType::Type::type) { return min_max; } @@ -292,7 +343,7 @@ CleanStatistic(std::pair min_max) { // - If max is -0.0f, replace with 0.0f template ::arrow::enable_if_t::value, optional>> -CleanStatistic(std::pair min_max) { +CleanStatistic(std::pair min_max, LogicalType::Type::type) { T min = min_max.first; T max = min_max.second; @@ -318,25 +369,67 @@ CleanStatistic(std::pair min_max) { return {{min, max}}; } -optional> CleanStatistic(std::pair min_max) { +optional> CleanFloat16Statistic(std::pair min_max) { + FLBA min_flba = min_max.first; + FLBA max_flba = min_max.second; + Float16 min = Float16::FromLittleEndian(min_flba.ptr); + Float16 max = Float16::FromLittleEndian(max_flba.ptr); + + if (min.is_nan() || max.is_nan()) { + return ::std::nullopt; + } + + if (min == std::numeric_limits::max() && + max == std::numeric_limits::lowest()) { + return ::std::nullopt; + } + + if (min.is_zero() && !min.signbit()) { + min_flba = FLBA{Float16Constants::negative_zero()}; + } + if (max.is_zero() && max.signbit()) { + max_flba = FLBA{Float16Constants::positive_zero()}; + } + + return {{min_flba, max_flba}}; +} + +optional> CleanStatistic(std::pair min_max, + LogicalType::Type::type logical_type) { if (min_max.first.ptr == nullptr || min_max.second.ptr == nullptr) { return ::std::nullopt; } + if (logical_type == LogicalType::Type::FLOAT16) { + return CleanFloat16Statistic(std::move(min_max)); + } return min_max; } optional> CleanStatistic( - std::pair min_max) { + std::pair min_max, LogicalType::Type::type) { if (min_max.first.ptr == nullptr || min_max.second.ptr == nullptr) { return ::std::nullopt; } return min_max; } +template +struct RebindLogical { + using DType = T; + using c_type = typename DType::c_type; +}; + +template <> +struct RebindLogical { + using DType = FLBAType; + using c_type = DType::c_type; +}; + template -class TypedComparatorImpl : virtual public TypedComparator { +class TypedComparatorImpl + : virtual public TypedComparator::DType> { public: - using T = typename DType::c_type; + using T = typename RebindLogical::c_type; using Helper = CompareHelper; explicit TypedComparatorImpl(int type_length = -1) : type_length_(type_length) {} @@ -384,7 +477,9 @@ class TypedComparatorImpl : virtual public TypedComparator { return {min, max}; } - std::pair GetMinMax(const ::arrow::Array& values) override; + std::pair GetMinMax(const ::arrow::Array& values) override { + ParquetException::NYI(values.type()->ToString()); + } private: int type_length_; @@ -412,12 +507,6 @@ TypedComparatorImpl::GetMinMax(const int32_t* va return {SafeCopy(min), SafeCopy(max)}; } -template -std::pair -TypedComparatorImpl::GetMinMax(const ::arrow::Array& values) { - ParquetException::NYI(values.type()->ToString()); -} - template std::pair GetMinMaxBinaryHelper( const TypedComparatorImpl& comparator, @@ -458,6 +547,16 @@ std::pair TypedComparatorImpl::GetMi return GetMinMaxBinaryHelper(*this, values); } +LogicalType::Type::type LogicalTypeId(const ColumnDescriptor* descr) { + if (const auto& logical_type = descr->logical_type()) { + return logical_type->type(); + } + return LogicalType::Type::NONE; +} +LogicalType::Type::type LogicalTypeId(const Statistics& stats) { + return LogicalTypeId(stats.descr()); +} + template class TypedStatisticsImpl : public TypedStatistics { public: @@ -468,9 +567,9 @@ class TypedStatisticsImpl : public TypedStatistics { : descr_(descr), pool_(pool), min_buffer_(AllocateBuffer(pool_, 0)), - max_buffer_(AllocateBuffer(pool_, 0)) { - auto comp = Comparator::Make(descr); - comparator_ = std::static_pointer_cast>(comp); + max_buffer_(AllocateBuffer(pool_, 0)), + logical_type_(LogicalTypeId(descr_)) { + comparator_ = MakeComparator(descr); TypedStatisticsImpl::Reset(); } @@ -527,9 +626,27 @@ class TypedStatisticsImpl : public TypedStatistics { void IncrementNumValues(int64_t n) override { num_values_ += n; } + static bool IsMeaningfulLogicalType(LogicalType::Type::type type) { + switch (type) { + case LogicalType::Type::FLOAT16: + return true; + default: + return false; + } + } + bool Equals(const Statistics& raw_other) const override { if (physical_type() != raw_other.physical_type()) return false; + const auto other_logical_type = LogicalTypeId(raw_other); + // Only compare against logical types that influence the interpretation of the + // physical type + if (IsMeaningfulLogicalType(logical_type_)) { + if (logical_type_ != other_logical_type) return false; + } else if (IsMeaningfulLogicalType(other_logical_type)) { + return false; + } + const auto& other = checked_cast(raw_other); if (has_min_max_ != other.has_min_max_) return false; @@ -655,6 +772,7 @@ class TypedStatisticsImpl : public TypedStatistics { EncodedStatistics statistics_; std::shared_ptr> comparator_; std::shared_ptr min_buffer_, max_buffer_; + LogicalType::Type::type logical_type_ = LogicalType::Type::NONE; void PlainEncode(const T& src, std::string* dst) const; void PlainDecode(const std::string& src, T* dst) const; @@ -686,7 +804,7 @@ class TypedStatisticsImpl : public TypedStatistics { void SetMinMaxPair(std::pair min_max) { // CleanStatistic can return a nullopt in case of erroneous values, e.g. NaN - auto maybe_min_max = CleanStatistic(min_max); + auto maybe_min_max = CleanStatistic(min_max, logical_type_); if (!maybe_min_max) return; auto min = maybe_min_max.value().first; @@ -795,12 +913,8 @@ void TypedStatisticsImpl::PlainDecode(const std::string& src, dst->ptr = reinterpret_cast(src.c_str()); } -} // namespace - -// ---------------------------------------------------------------------- -// Public factory functions - -std::shared_ptr Comparator::Make(Type::type physical_type, +std::shared_ptr DoMakeComparator(Type::type physical_type, + LogicalType::Type::type logical_type, SortOrder::type sort_order, int type_length) { if (SortOrder::SIGNED == sort_order) { @@ -820,6 +934,10 @@ std::shared_ptr Comparator::Make(Type::type physical_type, case Type::BYTE_ARRAY: return std::make_shared>(); case Type::FIXED_LEN_BYTE_ARRAY: + if (logical_type == LogicalType::Type::FLOAT16) { + return std::make_shared>( + type_length); + } return std::make_shared>(type_length); default: ParquetException::NYI("Signed Compare not implemented"); @@ -845,8 +963,21 @@ std::shared_ptr Comparator::Make(Type::type physical_type, return nullptr; } +} // namespace + +// ---------------------------------------------------------------------- +// Public factory functions + +std::shared_ptr Comparator::Make(Type::type physical_type, + SortOrder::type sort_order, + int type_length) { + return DoMakeComparator(physical_type, LogicalType::Type::NONE, sort_order, + type_length); +} + std::shared_ptr Comparator::Make(const ColumnDescriptor* descr) { - return Make(descr->physical_type(), descr->sort_order(), descr->type_length()); + return DoMakeComparator(descr->physical_type(), LogicalTypeId(descr), + descr->sort_order(), descr->type_length()); } std::shared_ptr Statistics::Make(const ColumnDescriptor* descr, diff --git a/cpp/src/parquet/statistics_test.cc b/cpp/src/parquet/statistics_test.cc index 637832945ec57..cb2e6455abfa9 100644 --- a/cpp/src/parquet/statistics_test.cc +++ b/cpp/src/parquet/statistics_test.cc @@ -34,6 +34,7 @@ #include "arrow/type_traits.h" #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_ops.h" +#include "arrow/util/float16.h" #include "arrow/util/ubsan.h" #include "parquet/column_reader.h" @@ -49,6 +50,7 @@ using arrow::default_memory_pool; using arrow::MemoryPool; +using arrow::util::Float16; using arrow::util::SafeCopy; namespace bit_util = arrow::bit_util; @@ -875,9 +877,22 @@ TEST(CorrectStatistics, Basics) { // Test SortOrder class static const int NUM_VALUES = 10; -template +template +struct RebindLogical { + using ParquetType = T; + using CType = typename T::c_type; +}; + +template <> +struct RebindLogical { + using ParquetType = FLBAType; + using CType = ParquetType::c_type; +}; + +template class TestStatisticsSortOrder : public ::testing::Test { public: + using TestType = typename RebindLogical::ParquetType; using c_type = typename TestType::c_type; void SetUp() override { @@ -955,7 +970,7 @@ class TestStatisticsSortOrder : public ::testing::Test { }; using CompareTestTypes = ::testing::Types; + ByteArrayType, FLBAType, Float16LogicalType>; // TYPE::INT32 template <> @@ -1102,6 +1117,39 @@ void TestStatisticsSortOrder::SetValues() { .set_max(std::string(reinterpret_cast(&vals[8][0]), FLBA_LENGTH)); } +template <> +void TestStatisticsSortOrder::AddNodes(std::string name) { + auto node = + schema::PrimitiveNode::Make(name, Repetition::REQUIRED, LogicalType::Float16(), + Type::FIXED_LEN_BYTE_ARRAY, sizeof(uint16_t)); + fields_.push_back(std::move(node)); +} + +template <> +void TestStatisticsSortOrder::SetValues() { + constexpr int kValueLen = 2; + constexpr int kNumBytes = NUM_VALUES * kValueLen; + + const Float16 f16_vals[NUM_VALUES] = { + Float16::FromFloat(+2.0f), Float16::FromFloat(-4.0f), Float16::FromFloat(+4.0f), + Float16::FromFloat(-2.0f), Float16::FromFloat(-1.0f), Float16::FromFloat(+3.0f), + Float16::FromFloat(+1.0f), Float16::FromFloat(-5.0f), Float16::FromFloat(+0.0f), + Float16::FromFloat(-3.0f), + }; + + values_buf_.resize(kNumBytes); + uint8_t* ptr = values_buf_.data(); + for (int i = 0; i < NUM_VALUES; ++i) { + f16_vals[i].ToLittleEndian(ptr); + values_[i].ptr = ptr; + ptr += kValueLen; + } + + stats_[0] + .set_min(std::string(reinterpret_cast(values_[7].ptr), kValueLen)) + .set_max(std::string(reinterpret_cast(values_[2].ptr), kValueLen)); +} + TYPED_TEST_SUITE(TestStatisticsSortOrder, CompareTestTypes); TYPED_TEST(TestStatisticsSortOrder, MinMax) { @@ -1167,12 +1215,20 @@ TEST_F(TestStatisticsSortOrderFLBA, UnknownSortOrder) { ASSERT_FALSE(cc_metadata->is_stats_set()); } +template +static std::string EncodeValue(const T& val) { + return std::string(reinterpret_cast(&val), sizeof(val)); +} +static std::string EncodeValue(const FLBA& val, int length = sizeof(uint16_t)) { + return std::string(reinterpret_cast(val.ptr), length); +} + template void AssertMinMaxAre(Stats stats, const Array& values, T expected_min, T expected_max) { stats->Update(values.data(), values.size(), 0); ASSERT_TRUE(stats->HasMinMax()); - EXPECT_EQ(stats->min(), expected_min); - EXPECT_EQ(stats->max(), expected_max); + EXPECT_EQ(stats->EncodeMin(), EncodeValue(expected_min)); + EXPECT_EQ(stats->EncodeMax(), EncodeValue(expected_max)); } template @@ -1184,8 +1240,8 @@ void AssertMinMaxAre(Stats stats, const Array& values, const uint8_t* valid_bitm stats->UpdateSpaced(values.data(), valid_bitmap, 0, non_null_count + null_count, non_null_count, null_count); ASSERT_TRUE(stats->HasMinMax()); - EXPECT_EQ(stats->min(), expected_min); - EXPECT_EQ(stats->max(), expected_max); + EXPECT_EQ(stats->EncodeMin(), EncodeValue(expected_min)); + EXPECT_EQ(stats->EncodeMax(), EncodeValue(expected_max)); } template @@ -1268,50 +1324,225 @@ void CheckExtrema() { TEST(TestStatistic, Int32Extrema) { CheckExtrema(); } TEST(TestStatistic, Int64Extrema) { CheckExtrema(); } -// PARQUET-1225: Float NaN values may lead to incorrect min-max -template -void CheckNaNs() { - using T = typename ParquetType::c_type; +template +class TestFloatStatistics : public ::testing::Test { + public: + using ParquetType = typename RebindLogical::ParquetType; + using c_type = typename ParquetType::c_type; + + void Init(); + void SetUp() override { + this->Init(); + ASSERT_NE(EncodeValue(negative_zero_), EncodeValue(positive_zero_)); + } + + bool signbit(c_type val); + void CheckEq(const c_type& l, const c_type& r); + NodePtr MakeNode(const std::string& name, Repetition::type rep); + + template + void CheckMinMaxZeroesSign(Stats stats, const Values& values) { + stats->Update(values.data(), values.size(), /*null_count=*/0); + ASSERT_TRUE(stats->HasMinMax()); + + this->CheckEq(stats->min(), positive_zero_); + ASSERT_TRUE(this->signbit(stats->min())); + ASSERT_EQ(stats->EncodeMin(), EncodeValue(negative_zero_)); + + this->CheckEq(stats->max(), positive_zero_); + ASSERT_FALSE(this->signbit(stats->max())); + ASSERT_EQ(stats->EncodeMax(), EncodeValue(positive_zero_)); + } + + // ARROW-5562: Ensure that -0.0f and 0.0f values are properly handled like in + // parquet-mr + void TestNegativeZeroes() { + NodePtr node = this->MakeNode("f", Repetition::OPTIONAL); + ColumnDescriptor descr(node, 1, 1); + + { + std::array values{negative_zero_, positive_zero_}; + auto stats = MakeStatistics(&descr); + CheckMinMaxZeroesSign(stats, values); + } + + { + std::array values{positive_zero_, negative_zero_}; + auto stats = MakeStatistics(&descr); + CheckMinMaxZeroesSign(stats, values); + } + + { + std::array values{negative_zero_, negative_zero_}; + auto stats = MakeStatistics(&descr); + CheckMinMaxZeroesSign(stats, values); + } + + { + std::array values{positive_zero_, positive_zero_}; + auto stats = MakeStatistics(&descr); + CheckMinMaxZeroesSign(stats, values); + } + } + + // PARQUET-1225: Float NaN values may lead to incorrect min-max + template + void CheckNaNs(ColumnDescriptor* descr, const Values& all_nans, const Values& some_nans, + const Values& other_nans, c_type min, c_type max, uint8_t valid_bitmap, + uint8_t valid_bitmap_no_nans) { + auto some_nan_stats = MakeStatistics(descr); + // Ingesting only nans should not yield valid min max + AssertUnsetMinMax(some_nan_stats, all_nans); + // Ingesting a mix of NaNs and non-NaNs should yield a valid min max. + AssertMinMaxAre(some_nan_stats, some_nans, min, max); + // Ingesting only nans after a valid min/max, should have no effect + AssertMinMaxAre(some_nan_stats, all_nans, min, max); + + some_nan_stats = MakeStatistics(descr); + AssertUnsetMinMax(some_nan_stats, all_nans, &valid_bitmap); + // NaNs should not pollute min max when excluded via null bitmap. + AssertMinMaxAre(some_nan_stats, some_nans, &valid_bitmap_no_nans, min, max); + // Ingesting NaNs with a null bitmap should not change the result. + AssertMinMaxAre(some_nan_stats, some_nans, &valid_bitmap, min, max); + + // An array that doesn't start with NaN + auto other_stats = MakeStatistics(descr); + AssertMinMaxAre(other_stats, other_nans, min, max); + } + + void TestNaNs(); + + protected: + std::vector data_buf_; + c_type positive_zero_; + c_type negative_zero_; +}; + +template +void TestFloatStatistics::Init() { + positive_zero_ = c_type{}; + negative_zero_ = -positive_zero_; +} +template <> +void TestFloatStatistics::Init() { + data_buf_.resize(4); + (+Float16(0)).ToLittleEndian(&data_buf_[0]); + positive_zero_ = FLBA{&data_buf_[0]}; + (-Float16(0)).ToLittleEndian(&data_buf_[2]); + negative_zero_ = FLBA{&data_buf_[2]}; +} + +template +NodePtr TestFloatStatistics::MakeNode(const std::string& name, Repetition::type rep) { + return PrimitiveNode::Make(name, rep, ParquetType::type_num); +} +template <> +NodePtr TestFloatStatistics::MakeNode(const std::string& name, + Repetition::type rep) { + return PrimitiveNode::Make(name, rep, LogicalType::Float16(), + Type::FIXED_LEN_BYTE_ARRAY, 2); +} + +template +void TestFloatStatistics::CheckEq(const c_type& l, const c_type& r) { + ASSERT_EQ(l, r); +} +template <> +void TestFloatStatistics::CheckEq(const c_type& a, const c_type& b) { + auto l = Float16::FromLittleEndian(a.ptr); + auto r = Float16::FromLittleEndian(b.ptr); + ASSERT_EQ(l, r); +} +template +bool TestFloatStatistics::signbit(c_type val) { + return std::signbit(val); +} +template <> +bool TestFloatStatistics::signbit(c_type val) { + return Float16::FromLittleEndian(val.ptr).signbit(); +} + +template +void TestFloatStatistics::TestNaNs() { constexpr int kNumValues = 8; - NodePtr node = PrimitiveNode::Make("f", Repetition::OPTIONAL, ParquetType::type_num); + NodePtr node = this->MakeNode("f", Repetition::OPTIONAL); ColumnDescriptor descr(node, 1, 1); - constexpr T nan = std::numeric_limits::quiet_NaN(); - constexpr T min = -4.0f; - constexpr T max = 3.0f; + constexpr c_type nan = std::numeric_limits::quiet_NaN(); + constexpr c_type min = -4.0f; + constexpr c_type max = 3.0f; + + std::array all_nans{nan, nan, nan, nan, nan, nan, nan, nan}; + std::array some_nans{nan, max, -3.0f, -1.0f, nan, 2.0f, min, nan}; + std::array other_nans{1.5f, max, -3.0f, -1.0f, nan, 2.0f, min, nan}; - std::array all_nans{nan, nan, nan, nan, nan, nan, nan, nan}; - std::array some_nans{nan, max, -3.0f, -1.0f, nan, 2.0f, min, nan}; uint8_t valid_bitmap = 0x7F; // 0b01111111 // NaNs excluded uint8_t valid_bitmap_no_nans = 0x6E; // 0b01101110 - // Test values - auto some_nan_stats = MakeStatistics(&descr); - // Ingesting only nans should not yield valid min max - AssertUnsetMinMax(some_nan_stats, all_nans); - // Ingesting a mix of NaNs and non-NaNs should not yield valid min max. - AssertMinMaxAre(some_nan_stats, some_nans, min, max); - // Ingesting only nans after a valid min/max, should have not effect - AssertMinMaxAre(some_nan_stats, all_nans, min, max); + this->CheckNaNs(&descr, all_nans, some_nans, other_nans, min, max, valid_bitmap, + valid_bitmap_no_nans); +} - some_nan_stats = MakeStatistics(&descr); - AssertUnsetMinMax(some_nan_stats, all_nans, &valid_bitmap); - // NaNs should not pollute min max when excluded via null bitmap. - AssertMinMaxAre(some_nan_stats, some_nans, &valid_bitmap_no_nans, min, max); - // Ingesting NaNs with a null bitmap should not change the result. - AssertMinMaxAre(some_nan_stats, some_nans, &valid_bitmap, min, max); +struct BufferedFloat16 { + explicit BufferedFloat16(Float16 f16) : f16(f16) { + this->f16.ToLittleEndian(bytes_.data()); + } + explicit BufferedFloat16(float f) : BufferedFloat16(Float16::FromFloat(f)) {} + const uint8_t* bytes() const { return bytes_.data(); } + + Float16 f16; + std::array bytes_; +}; + +template <> +void TestFloatStatistics::TestNaNs() { + constexpr int kNumValues = 8; + + NodePtr node = this->MakeNode("f", Repetition::OPTIONAL); + ColumnDescriptor descr(node, 1, 1); + + using F16 = BufferedFloat16; + const auto nan_f16 = F16(std::numeric_limits::quiet_NaN()); + const auto min_f16 = F16(-4.0f); + const auto max_f16 = F16(+3.0f); + + const auto min = FLBA{min_f16.bytes()}; + const auto max = FLBA{max_f16.bytes()}; + + std::array all_nans_f16 = {nan_f16, nan_f16, nan_f16, nan_f16, + nan_f16, nan_f16, nan_f16, nan_f16}; + std::array some_nans_f16 = { + nan_f16, max_f16, F16(-3.0f), F16(-1.0f), nan_f16, F16(+2.0f), min_f16, nan_f16}; + std::array other_nans_f16 = some_nans_f16; + other_nans_f16[0] = F16(+1.5f); // +1.5 + + auto prepare_values = [](const auto& values) -> std::vector { + std::vector out(values.size()); + std::transform(values.begin(), values.end(), out.begin(), + [](const F16& f16) { return FLBA{f16.bytes()}; }); + return out; + }; + + auto all_nans = prepare_values(all_nans_f16); + auto some_nans = prepare_values(some_nans_f16); + auto other_nans = prepare_values(other_nans_f16); + + uint8_t valid_bitmap = 0x7F; // 0b01111111 + // NaNs excluded + uint8_t valid_bitmap_no_nans = 0x6E; // 0b01101110 - // An array that doesn't start with NaN - std::array other_nans{1.5f, max, -3.0f, -1.0f, nan, 2.0f, min, nan}; - auto other_stats = MakeStatistics(&descr); - AssertMinMaxAre(other_stats, other_nans, min, max); + this->CheckNaNs(&descr, all_nans, some_nans, other_nans, min, max, valid_bitmap, + valid_bitmap_no_nans); } -TEST(TestStatistic, NaNFloatValues) { CheckNaNs(); } +using FloatingPointTypes = ::testing::Types; + +TYPED_TEST_SUITE(TestFloatStatistics, FloatingPointTypes); -TEST(TestStatistic, NaNDoubleValues) { CheckNaNs(); } +TYPED_TEST(TestFloatStatistics, NegativeZeros) { this->TestNegativeZeroes(); } +TYPED_TEST(TestFloatStatistics, NaNs) { this->TestNaNs(); } // ARROW-7376 TEST(TestStatisticsSortOrderFloatNaN, NaNAndNullsInfiniteLoop) { @@ -1327,58 +1558,6 @@ TEST(TestStatisticsSortOrderFloatNaN, NaNAndNullsInfiniteLoop) { AssertUnsetMinMax(stats, nans_but_last, &all_but_last_valid); } -template -void AssertMinMaxZeroesSign(Stats stats, const Array& values) { - stats->Update(values.data(), values.size(), 0); - ASSERT_TRUE(stats->HasMinMax()); - - T zero{}; - ASSERT_EQ(stats->min(), zero); - ASSERT_TRUE(std::signbit(stats->min())); - - ASSERT_EQ(stats->max(), zero); - ASSERT_FALSE(std::signbit(stats->max())); -} - -// ARROW-5562: Ensure that -0.0f and 0.0f values are properly handled like in -// parquet-mr -template -void CheckNegativeZeroStats() { - using T = typename ParquetType::c_type; - - NodePtr node = PrimitiveNode::Make("f", Repetition::OPTIONAL, ParquetType::type_num); - ColumnDescriptor descr(node, 1, 1); - T zero{}; - - { - std::array values{-zero, zero}; - auto stats = MakeStatistics(&descr); - AssertMinMaxZeroesSign(stats, values); - } - - { - std::array values{zero, -zero}; - auto stats = MakeStatistics(&descr); - AssertMinMaxZeroesSign(stats, values); - } - - { - std::array values{-zero, -zero}; - auto stats = MakeStatistics(&descr); - AssertMinMaxZeroesSign(stats, values); - } - - { - std::array values{zero, zero}; - auto stats = MakeStatistics(&descr); - AssertMinMaxZeroesSign(stats, values); - } -} - -TEST(TestStatistics, FloatNegativeZero) { CheckNegativeZeroStats(); } - -TEST(TestStatistics, DoubleNegativeZero) { CheckNegativeZeroStats(); } - // Test statistics for binary column with UNSIGNED sort order TEST(TestStatisticsSortOrderMinMax, Unsigned) { std::string dir_string(test::get_data_dir()); diff --git a/cpp/src/parquet/test_util.cc b/cpp/src/parquet/test_util.cc index b65945cc7329f..a6fa8afc0f5b3 100644 --- a/cpp/src/parquet/test_util.cc +++ b/cpp/src/parquet/test_util.cc @@ -101,6 +101,16 @@ void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_v } } +void random_float16_numbers(int n, uint32_t seed, ::arrow::util::Float16 min_value, + ::arrow::util::Float16 max_value, uint16_t* out) { + std::vector values(n); + random_numbers(n, seed, static_cast(min_value), static_cast(max_value), + values.data()); + for (int i = 0; i < n; ++i) { + out[i] = ::arrow::util::Float16(values[i]).bits(); + } +} + void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out) { std::default_random_engine gen(seed); std::uniform_int_distribution d(0, 255); diff --git a/cpp/src/parquet/test_util.h b/cpp/src/parquet/test_util.h index c8578609e9b1d..59728cf53f699 100644 --- a/cpp/src/parquet/test_util.h +++ b/cpp/src/parquet/test_util.h @@ -33,6 +33,7 @@ #include "arrow/io/memory.h" #include "arrow/testing/util.h" +#include "arrow/util/float16.h" #include "parquet/column_page.h" #include "parquet/column_reader.h" @@ -148,6 +149,9 @@ inline void random_numbers(int n, uint32_t seed, double min_value, double max_va void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_value, Int96* out); +void random_float16_numbers(int n, uint32_t seed, ::arrow::util::Float16 min_value, + ::arrow::util::Float16 max_value, uint16_t* out); + void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out); void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int min_size, diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc index 3127b60e5d1ae..04a0fc2e0117b 100644 --- a/cpp/src/parquet/types.cc +++ b/cpp/src/parquet/types.cc @@ -441,6 +441,8 @@ std::shared_ptr LogicalType::FromThrift( return BSONLogicalType::Make(); } else if (type.__isset.UUID) { return UUIDLogicalType::Make(); + } else if (type.__isset.FLOAT16) { + return Float16LogicalType::Make(); } else { throw ParquetException("Metadata contains Thrift LogicalType that is not recognized"); } @@ -494,6 +496,10 @@ std::shared_ptr LogicalType::BSON() { return BSONLogicalType: std::shared_ptr LogicalType::UUID() { return UUIDLogicalType::Make(); } +std::shared_ptr LogicalType::Float16() { + return Float16LogicalType::Make(); +} + std::shared_ptr LogicalType::None() { return NoLogicalType::Make(); } /* @@ -575,6 +581,7 @@ class LogicalType::Impl { class JSON; class BSON; class UUID; + class Float16; class No; class Undefined; @@ -644,6 +651,9 @@ bool LogicalType::is_null() const { return impl_->type() == LogicalType::Type::N bool LogicalType::is_JSON() const { return impl_->type() == LogicalType::Type::JSON; } bool LogicalType::is_BSON() const { return impl_->type() == LogicalType::Type::BSON; } bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; } +bool LogicalType::is_float16() const { + return impl_->type() == LogicalType::Type::FLOAT16; +} bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; } bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNDEFINED; @@ -1557,6 +1567,22 @@ class LogicalType::Impl::UUID final : public LogicalType::Impl::Incompatible, GENERATE_MAKE(UUID) +class LogicalType::Impl::Float16 final : public LogicalType::Impl::Incompatible, + public LogicalType::Impl::TypeLengthApplicable { + public: + friend class Float16LogicalType; + + OVERRIDE_TOSTRING(Float16) + OVERRIDE_TOTHRIFT(Float16Type, FLOAT16) + + private: + Float16() + : LogicalType::Impl(LogicalType::Type::FLOAT16, SortOrder::SIGNED), + LogicalType::Impl::TypeLengthApplicable(parquet::Type::FIXED_LEN_BYTE_ARRAY, 2) {} +}; + +GENERATE_MAKE(Float16) + class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible, public LogicalType::Impl::UniversalApplicable { public: diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h index 0315376a883e9..76dd0efc7cb4a 100644 --- a/cpp/src/parquet/types.h +++ b/cpp/src/parquet/types.h @@ -157,6 +157,7 @@ class PARQUET_EXPORT LogicalType { JSON, BSON, UUID, + FLOAT16, NONE // Not a real logical type; should always be last element }; }; @@ -210,6 +211,7 @@ class PARQUET_EXPORT LogicalType { static std::shared_ptr JSON(); static std::shared_ptr BSON(); static std::shared_ptr UUID(); + static std::shared_ptr Float16(); /// \brief Create a placeholder for when no logical type is specified static std::shared_ptr None(); @@ -263,6 +265,7 @@ class PARQUET_EXPORT LogicalType { bool is_JSON() const; bool is_BSON() const; bool is_UUID() const; + bool is_float16() const; bool is_none() const; /// \brief Return true if this logical type is of a known type. bool is_valid() const; @@ -433,6 +436,16 @@ class PARQUET_EXPORT UUIDLogicalType : public LogicalType { UUIDLogicalType() = default; }; +/// \brief Allowed for physical type FIXED_LEN_BYTE_ARRAY with length 2, +/// must encode raw FLOAT16 bytes. +class PARQUET_EXPORT Float16LogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + Float16LogicalType() = default; +}; + /// \brief Allowed for any physical type. class PARQUET_EXPORT NoLogicalType : public LogicalType { public: diff --git a/docs/source/cpp/parquet.rst b/docs/source/cpp/parquet.rst index 23fca8fd73010..3e06352f5dde3 100644 --- a/docs/source/cpp/parquet.rst +++ b/docs/source/cpp/parquet.rst @@ -481,6 +481,8 @@ physical type. +-------------------+-----------------------------+----------------------------+---------+ | MAP | Any | Map | \(6) | +-------------------+-----------------------------+----------------------------+---------+ +| FLOAT16 | FIXED_LENGTH_BYTE_ARRAY | HalfFloat | | ++-------------------+-----------------------------+----------------------------+---------+ * \(1) On the write side, the Parquet physical type INT32 is generated. From 3c0b315621366c80c8eab03118dca20f0cfe7ffc Mon Sep 17 00:00:00 2001 From: Junming Chen Date: Thu, 16 Nov 2023 08:38:45 +0800 Subject: [PATCH 57/57] Update cpp/src/arrow/array/array_test.cc Co-authored-by: Benjamin Kietzman --- cpp/src/arrow/array/array_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/array/array_test.cc b/cpp/src/arrow/array/array_test.cc index 63fa555455896..3a160bd81f40b 100644 --- a/cpp/src/arrow/array/array_test.cc +++ b/cpp/src/arrow/array/array_test.cc @@ -90,10 +90,10 @@ void CheckDictionaryNullCount(const std::shared_ptr& dict_type, std::shared_ptr arr = DictArrayFromJSON(dict_type, input_index_json, input_dictionary_json); - ASSERT_EQ(expected_null_count, arr->null_count()); - ASSERT_EQ(expected_logical_null_count, arr->ComputeLogicalNullCount()); - ASSERT_EQ(expected_may_have_nulls, arr->data()->MayHaveNulls()); - ASSERT_EQ(expected_may_have_logical_nulls, arr->data()->MayHaveLogicalNulls()); + ASSERT_EQ(arr->null_count(), expected_null_count); + ASSERT_EQ(arr->ComputeLogicalNullCount(), expected_logical_null_count); + ASSERT_EQ(arr->data()->MayHaveNulls(), expected_may_have_nulls); + ASSERT_EQ(arr->data()->MayHaveLogicalNulls(), expected_may_have_logical_nulls); } TEST_F(TestArray, TestNullCount) {