Merge branch 'GH-36760-Go]-Adding-avro-ocf-reader---reader' of https:…

…//github.com/loicalleyne/arrow into GH-36760-Go]-Adding-avro-ocf-reader---reader
apache · Aug 10, 2023 · 5865fef · 5865fef
2 parents b730534 + 4505b43
commit 5865fef
Show file tree

Hide file tree

Showing 266 changed files with 7,717 additions and 1,833 deletions.
diff --git a/.env b/.env
@@ -98,7 +98,7 @@ VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"    # 2023.04.15 Release
 # ci/docker/python-wheel-windows-vs2017.dockerfile.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2023-08-02
 
 # Use conanio/${CONAN} for "docker-compose run --rm conan". See
 # https://github.com/conan-io/conan-docker-tools#readme for available

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -23,6 +23,12 @@ updates:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [CI] "
+  - package-ecosystem: "npm"
+    directory: "/js/"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "MINOR: [JS] "
   - package-ecosystem: "nuget"
     directory: "/csharp/"
     schedule:

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -23,6 +23,7 @@ on:
       - '.github/workflows/cpp.yml'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
+      - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
       - 'ci/scripts/install_minio.sh'
       - 'ci/scripts/msys2_*'
@@ -34,6 +35,7 @@ on:
       - '.github/workflows/cpp.yml'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
+      - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
       - 'ci/scripts/install_minio.sh'
       - 'ci/scripts/msys2_*'
@@ -201,6 +203,8 @@ jobs:
             ci/scripts/install_minio.sh latest /usr/local
       - name: Install Google Cloud Storage Testbench
         run: ci/scripts/install_gcs_testbench.sh default
+      - name: Install Azurite Storage Emulator
+        run: ci/scripts/install_azurite.sh
       - name: Setup ccache
         run: |
           ci/scripts/ccache_setup.sh

diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile
@@ -84,7 +84,8 @@ RUN vcpkg install \
         --x-feature=flight \
         --x-feature=gcs \
         --x-feature=json \
-        --x-feature=parquet
+        --x-feature=parquet \
+        --x-feature=s3
 
 # Configure Python for applications running in the bash shell of this Dockerfile
 ARG python=3.8

diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -66,7 +66,8 @@ RUN vcpkg install \
         --x-feature=flight \
         --x-feature=gcs \
         --x-feature=json \
-        --x-feature=parquet
+        --x-feature=parquet \
+        --x-feature=s3
 
 # Remove previous installations of python from the base image
 # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7

diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile
@@ -103,6 +103,7 @@ RUN apt-get update -y -q && \
         make \
         ninja-build \
         nlohmann-json3-dev \
+        npm \
         pkg-config \
         protobuf-compiler \
         python3-dev \
@@ -123,6 +124,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 COPY ci/scripts/install_ceph.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_ceph.sh
 

diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile
@@ -102,6 +102,7 @@ RUN apt-get update -y -q && \
         make \
         ninja-build \
         nlohmann-json3-dev \
+        npm \
         pkg-config \
         protobuf-compiler \
         protobuf-compiler-grpc \
@@ -153,6 +154,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 

diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh
@@ -106,6 +106,7 @@ cmake \
   -DARROW_C_FLAGS_RELWITHDEBINFO="${ARROW_C_FLAGS_RELWITHDEBINFO:-}" \
   -DARROW_DATASET=${ARROW_DATASET:-ON} \
   -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \
+  -DARROW_ENABLE_THREADING=${ARROW_ENABLE_THREADING:-ON} \
   -DARROW_ENABLE_TIMING_TESTS=${ARROW_ENABLE_TIMING_TESTS:-ON} \
   -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \
   -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \

diff --git a/ci/scripts/install_azurite.sh b/ci/scripts/install_azurite.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+case "$(uname)" in
+  Darwin)
+    npm install -g azurite
+    which azurite
+    ;;
+  MINGW*)
+    choco install nodejs.install
+    npm install -g azurite
+    ;;
+  Linux)
+    npm install -g azurite
+    which azurite
+    ;;
+esac
+echo "node version = $(node --version)"
+echo "azurite version = $(azurite --version)"
diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh
@@ -81,7 +81,6 @@ cmake \
   -DARROW_PARQUET=${ARROW_PARQUET} \
   -DARROW_S3=${ARROW_S3} \
   -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
-  -DAWSSDK_SOURCE=BUNDLED \
   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
   -DCMAKE_INSTALL_LIBDIR=lib \
   -DCMAKE_INSTALL_PREFIX=${install_dir} \

diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh
@@ -85,9 +85,6 @@ fi
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
 
-# ARROW-17501: We can remove -DAWSSDK_SOURCE=BUNDLED once
-# https://github.com/aws/aws-sdk-cpp/issues/1809 is fixed and vcpkg
-# ships the fix.
 cmake \
     -DARROW_ACERO=${ARROW_ACERO} \
     -DARROW_BUILD_SHARED=ON \
@@ -120,7 +117,6 @@ cmake \
     -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
     -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
     -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
-    -DAWSSDK_SOURCE=BUNDLED \
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
     -DCMAKE_INSTALL_LIBDIR=lib \
     -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \

diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat
@@ -49,9 +49,6 @@ set ARROW_WITH_LZ4=ON
 set ARROW_WITH_SNAPPY=ON
 set ARROW_WITH_ZLIB=ON
 set ARROW_WITH_ZSTD=ON
-@rem Workaround for https://github.com/aws/aws-sdk-cpp/issues/1809 .
-@rem Use (old) bundled AWS SDK C++ instead of (newer) AWS SDK C++.
-set AWSSDK_SOURCE=BUNDLED
 set CMAKE_UNITY_BUILD=ON
 set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
 set VCPKG_ROOT=C:\vcpkg
@@ -90,7 +87,6 @@ cmake ^
     -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
     -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
     -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
-    -DAWSSDK_SOURCE=%AWSSDK_SOURCE% ^
     -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
     -DCMAKE_CXX_COMPILER=clcache ^
     -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^

diff --git a/cpp/Brewfile b/cpp/Brewfile
@@ -30,6 +30,7 @@ brew "grpc"
 brew "llvm@14"
 brew "lz4"
 brew "ninja"
+brew "node"
 brew "openssl@3"
 brew "protobuf"
 brew "python"

diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake
@@ -196,6 +196,8 @@ takes precedence over ccache if a storage backend is configured" ON)
 
   define_option(ARROW_WITH_MUSL "Whether the system libc is musl or not" OFF)
 
+  define_option(ARROW_ENABLE_THREADING "Enable threading in Arrow core" ON)
+
   #----------------------------------------------------------------------
   set_option_category("Test and benchmark")
 

diff --git a/cpp/src/arrow/acero/CMakeLists.txt b/cpp/src/arrow/acero/CMakeLists.txt
@@ -173,7 +173,14 @@ add_arrow_acero_test(hash_join_node_test SOURCES hash_join_node_test.cc
                      bloom_filter_test.cc)
 add_arrow_acero_test(pivot_longer_node_test SOURCES pivot_longer_node_test.cc
                      test_nodes.cc)
-add_arrow_acero_test(asof_join_node_test SOURCES asof_join_node_test.cc test_nodes.cc)
+
+# asof_join_node uses std::thread internally
+# and doesn't use ThreadPool so it will
+# be broken if threading is turned off
+if(ARROW_ENABLE_THREADING)
+  add_arrow_acero_test(asof_join_node_test SOURCES asof_join_node_test.cc test_nodes.cc)
+endif()
+
 add_arrow_acero_test(tpch_node_test SOURCES tpch_node_test.cc)
 add_arrow_acero_test(union_node_test SOURCES union_node_test.cc)
 add_arrow_acero_test(aggregate_node_test SOURCES aggregate_node_test.cc)
@@ -221,7 +228,9 @@ if(ARROW_BUILD_BENCHMARKS)
   add_arrow_acero_benchmark(project_benchmark SOURCES benchmark_util.cc
                             project_benchmark.cc)
 
-  add_arrow_acero_benchmark(asof_join_benchmark SOURCES asof_join_benchmark.cc)
+  if(ARROW_ENABLE_THREADING)
+    add_arrow_acero_benchmark(asof_join_benchmark SOURCES asof_join_benchmark.cc)
+  endif()
 
   add_arrow_acero_benchmark(tpch_benchmark SOURCES tpch_benchmark.cc)
 
@@ -244,7 +253,9 @@ if(ARROW_BUILD_BENCHMARKS)
     target_link_libraries(arrow-acero-expression-benchmark PUBLIC arrow_acero_static)
     target_link_libraries(arrow-acero-filter-benchmark PUBLIC arrow_acero_static)
     target_link_libraries(arrow-acero-project-benchmark PUBLIC arrow_acero_static)
-    target_link_libraries(arrow-acero-asof-join-benchmark PUBLIC arrow_acero_static)
+    if(ARROW_ENABLE_THREADING)
+      target_link_libraries(arrow-acero-asof-join-benchmark PUBLIC arrow_acero_static)
+    endif()
     target_link_libraries(arrow-acero-tpch-benchmark PUBLIC arrow_acero_static)
     if(ARROW_BUILD_OPENMP_BENCHMARKS)
       target_link_libraries(arrow-acero-hash-join-benchmark PUBLIC arrow_acero_static)
@@ -253,7 +264,9 @@ if(ARROW_BUILD_BENCHMARKS)
     target_link_libraries(arrow-acero-expression-benchmark PUBLIC arrow_acero_shared)
     target_link_libraries(arrow-acero-filter-benchmark PUBLIC arrow_acero_shared)
     target_link_libraries(arrow-acero-project-benchmark PUBLIC arrow_acero_shared)
-    target_link_libraries(arrow-acero-asof-join-benchmark PUBLIC arrow_acero_shared)
+    if(ARROW_ENABLE_THREADING)
+      target_link_libraries(arrow-acero-asof-join-benchmark PUBLIC arrow_acero_shared)
+    endif()
     target_link_libraries(arrow-acero-tpch-benchmark PUBLIC arrow_acero_shared)
     if(ARROW_BUILD_OPENMP_BENCHMARKS)
       target_link_libraries(arrow-acero-hash-join-benchmark PUBLIC arrow_acero_shared)

diff --git a/cpp/src/arrow/acero/aggregate_internal.h b/cpp/src/arrow/acero/aggregate_internal.h
@@ -52,8 +52,8 @@
 // segment-keys is used to refine the partitioning. However, segment-keys are different in
 // that they partition only consecutive rows into a single group. Such a partition of
 // consecutive rows is called a segment group. For example, consider a column X with
-// values [A, A, B, A] at row-indices [0, 1, 2]. A regular group-by aggregation with keys
-// [X] yields a row-index partitioning [[0, 1, 3], [2]] whereas a segmented-group-by
+// values [A, A, B, A] at row-indices [0, 1, 2, 3]. A regular group-by aggregation with
+// keys [X] yields a row-index partitioning [[0, 1, 3], [2]] whereas a segmented-group-by
 // aggregation with segment-keys [X] yields [[0, 1], [2], [3]].
 //
 // The implementation first segments the input using the segment-keys, then groups by the

diff --git a/cpp/src/arrow/acero/asof_join_node.cc b/cpp/src/arrow/acero/asof_join_node.cc
@@ -49,6 +49,7 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/bit_util.h"
 #include "arrow/util/checked_cast.h"
+#include "arrow/util/config.h"
 #include "arrow/util/future.h"
 #include "arrow/util/string.h"
 
@@ -1707,6 +1708,10 @@ class AsofJoinNode : public ExecNode {
   }
 
   Status StartProducing() override {
+#ifndef ARROW_ENABLE_THREADING
+    return Status::NotImplemented("ASOF join requires threading enabled");
+#endif
+
     ARROW_ASSIGN_OR_RAISE(process_task_, plan_->query_context()->BeginExternalTask(
                                              "AsofJoinNode::ProcessThread"));
     if (!process_task_.is_valid()) {

diff --git a/cpp/src/arrow/acero/bloom_filter.cc b/cpp/src/arrow/acero/bloom_filter.cc
@@ -20,6 +20,7 @@
 #include "arrow/acero/util.h"       // PREFETCH
 #include "arrow/util/bit_util.h"    // Log2
 #include "arrow/util/bitmap_ops.h"  // CountSetBits
+#include "arrow/util/config.h"
 
 namespace arrow {
 namespace acero {
@@ -426,6 +427,9 @@ void BloomFilterBuilder_Parallel::CleanUp() {
 
 std::unique_ptr<BloomFilterBuilder> BloomFilterBuilder::Make(
     BloomFilterBuildStrategy strategy) {
+#ifndef ARROW_ENABLE_THREADING
+  strategy = BloomFilterBuildStrategy::SINGLE_THREADED;
+#endif
   switch (strategy) {
     case BloomFilterBuildStrategy::SINGLE_THREADED: {
       std::unique_ptr<BloomFilterBuilder> impl{new BloomFilterBuilder_SingleThreaded()};

diff --git a/cpp/src/arrow/acero/bloom_filter_test.cc b/cpp/src/arrow/acero/bloom_filter_test.cc
@@ -29,6 +29,8 @@
 #include "arrow/acero/util.h"
 #include "arrow/compute/key_hash.h"
 #include "arrow/util/bitmap_ops.h"
+#include "arrow/util/config.h"
+#include "arrow/util/cpu_info.h"
 
 namespace arrow {
 
@@ -468,7 +470,7 @@ TEST(BloomFilter, Basic) {
 
   std::vector<BloomFilterBuildStrategy> strategies;
   strategies.push_back(BloomFilterBuildStrategy::SINGLE_THREADED);
-#ifndef ARROW_VALGRIND
+#if defined(ARROW_ENABLE_THREADING) && !defined(ARROW_VALGRIND)
   strategies.push_back(BloomFilterBuildStrategy::PARALLEL);
 #endif
 
@@ -501,7 +503,10 @@ TEST(BloomFilter, Scaling) {
   num_build.push_back(4000000);
 
   std::vector<BloomFilterBuildStrategy> strategies;
+#ifdef ARROW_ENABLE_THREADING
   strategies.push_back(BloomFilterBuildStrategy::PARALLEL);
+#endif
+  strategies.push_back(BloomFilterBuildStrategy::SINGLE_THREADED);
 
   for (const auto hardware_flags : HardwareFlagsForTesting()) {
     for (const auto& strategy : strategies) {

diff --git a/cpp/src/arrow/acero/exec_plan.h b/cpp/src/arrow/acero/exec_plan.h
@@ -739,7 +739,7 @@ DeclarationToBatchesAsync(Declaration declaration, ExecContext exec_context);
 /// \brief Utility method to run a declaration and return results as a RecordBatchReader
 ///
 /// If an exec context is not provided then a default exec context will be used based
-/// on the value of `use_threads`.  If `use_threads` is false then the CPU exeuctor will
+/// on the value of `use_threads`.  If `use_threads` is false then the CPU executor will
 /// be a serial executor and all CPU work will be done on the calling thread.  I/O tasks
 /// will still happen on the I/O executor and may be multi-threaded.
 ///

diff --git a/cpp/src/arrow/acero/options.h b/cpp/src/arrow/acero/options.h
@@ -80,7 +80,7 @@ class ARROW_ACERO_EXPORT ExecNodeOptions {
 ///
 /// For each batch received a new task will be created to push that batch downstream.
 /// This task will slice smaller units of size `ExecPlan::kMaxBatchSize` from the
-/// parent batch and call InputRecieved.  Thus, if the `generator` yields a large
+/// parent batch and call InputReceived.  Thus, if the `generator` yields a large
 /// batch it may result in several calls to InputReceived.
 ///
 /// The SourceNode will, by default, assign an implicit ordering to outgoing batches.
@@ -115,7 +115,7 @@ class ARROW_ACERO_EXPORT TableSourceNodeOptions : public ExecNodeOptions {
   /// Create an instance from values
   TableSourceNodeOptions(std::shared_ptr<Table> table,
                          int64_t max_batch_size = kDefaultMaxBatchSize)
-      : table(table), max_batch_size(max_batch_size) {}
+      : table(std::move(table)), max_batch_size(max_batch_size) {}
 
   /// \brief a table which acts as the data source
   std::shared_ptr<Table> table;
@@ -135,7 +135,7 @@ class ARROW_ACERO_EXPORT NamedTableNodeOptions : public ExecNodeOptions {
  public:
   /// Create an instance from values
   NamedTableNodeOptions(std::vector<std::string> names, std::shared_ptr<Schema> schema)
-      : names(std::move(names)), schema(schema) {}
+      : names(std::move(names)), schema(std::move(schema)) {}
 
   /// \brief the names to put in the serialized plan
   std::vector<std::string> names;
@@ -156,7 +156,7 @@ class ARROW_ACERO_EXPORT SchemaSourceNodeOptions : public ExecNodeOptions {
   /// Create an instance that will create a new task on io_executor for each iteration
   SchemaSourceNodeOptions(std::shared_ptr<Schema> schema, ItMaker it_maker,
                           arrow::internal::Executor* io_executor)
-      : schema(schema),
+      : schema(std::move(schema)),
         it_maker(std::move(it_maker)),
         io_executor(io_executor),
         requires_io(true) {}
@@ -165,7 +165,7 @@ class ARROW_ACERO_EXPORT SchemaSourceNodeOptions : public ExecNodeOptions {
   /// executor
   SchemaSourceNodeOptions(std::shared_ptr<Schema> schema, ItMaker it_maker,
                           bool requires_io = false)
-      : schema(schema),
+      : schema(std::move(schema)),
         it_maker(std::move(it_maker)),
         io_executor(NULLPTR),
         requires_io(requires_io) {}