Support linking to Arrow::python instead of Arrow::arrow_shared

This is mostly useful for hickpy, where linking to Arrow::arrow_shared
causes linking to a specific version of Arrow (which depends on
the specific version of Pyarrow that was installed when building
hictkpy wheels).
This is not desirable, because it forces us to pin the version of pyarrow
required by hictkpy (which could easily result in unresolvable environments).
Here we assume that libarrow_python.so links to some version of
libarrow.so, which when loaded brings in all symbols required by
hictk and hictkpy.
It should be noted that Arrow has very few  guarantees when it comes to ABI
stability (see apache/arrow#41707).
This means that hictkpy needs to test all possible combinations of pyarrow/arrow
versions on all supported platforms and architectures to make sure there
are no ABI changes (at least to the parts that hictk and hictkpy depend on).

CMakeLists.txt

@@ -155,6 +155,7 @@ option(HICTK_ENABLE_FUZZY_TESTING "Build fuzzy tests" OFF)
 option(HICTK_BUILD_EXAMPLES "Build examples" OFF)
 option(HICTK_BUILD_BENCHMARKS "Build benchmarks" OFF)
 option(HICTK_WITH_ARROW "Build with arrow support" ON)
+option(HICTK_WITH_PYARROW_UNSAFE "Build with arrow support but link to libarrow_python instead of libarrow" OFF)
 option(HICTK_WITH_ARROW_SHARED "Force dynamic linking to Arrow libs" OFF)
 option(HICTK_WITH_EIGEN "Build with Eigen3 support" ON)
 option(HICTK_BUILD_TOOLS "Build cli tools" ON)
@@ -163,14 +164,19 @@ if(HICTK_WITH_EIGEN)
   target_compile_definitions(hictk_project_options INTERFACE HICTK_WITH_EIGEN)
 endif()
 
-if(HICTK_WITH_ARROW)
-  target_compile_definitions(hictk_project_options INTERFACE HICTK_WITH_ARROW)
+if(BUILD_SHARED_LIBS)
+  set(HICTK_WITH_ARROW_SHARED ON CACHE BOOL "Force dynamic linking to Arrow libs" FORCE)
 endif()
 
-if(BUILD_SHARED_LIBS)
+if(HICTK_WITH_PYARROW_UNSAFE)
+  set(HICTK_WITH_ARROW ON CACHE BOOL "Build with arrow support" FORCE)
   set(HICTK_WITH_ARROW_SHARED ON CACHE BOOL "Force dynamic linking to Arrow libs" FORCE)
 endif()
 
+if(HICTK_WITH_ARROW)
+  target_compile_definitions(hictk_project_options INTERFACE HICTK_WITH_ARROW)
+endif()
+
 add_subdirectory(src)
 
 if(HICTK_ENABLE_TESTING)

src/libhictk/transformers/CMakeLists.txt

@@ -3,7 +3,11 @@
 # SPDX-License-Identifier: MIT
 
 if(HICTK_WITH_ARROW)
-  find_package(Arrow REQUIRED QUIET)
+  if(HICTK_WITH_PYARROW_UNSAFE)
+    find_package(Pyarrow REQUIRED QUIET)
+  else()
+    find_package(Arrow REQUIRED QUIET)
+  endif()
 endif()
 
 find_package(spdlog REQUIRED QUIET)
@@ -33,8 +37,12 @@ target_link_libraries(
 
 target_link_system_libraries(transformers INTERFACE spdlog::spdlog_header_only)
 
-if(HICTK_WITH_ARROW_SHARED)
-  target_link_system_libraries(transformers INTERFACE "$<$<BOOL:${HICTK_WITH_ARROW}>:Arrow::arrow_shared>")
+if(HICTK_WITH_PYARROW_UNSAFE)
+  target_link_system_libraries(transformers INTERFACE Arrow::python)
 else()
-  target_link_system_libraries(transformers INTERFACE "$<$<BOOL:${HICTK_WITH_ARROW}>:Arrow::arrow_static>")
+  if(HICTK_WITH_ARROW_SHARED)
+    target_link_system_libraries(transformers INTERFACE "$<$<BOOL:${HICTK_WITH_ARROW}>:Arrow::arrow_shared>")
+  else()
+    target_link_system_libraries(transformers INTERFACE "$<$<BOOL:${HICTK_WITH_ARROW}>:Arrow::arrow_static>")
+  endif()
 endif()