fix(ci, profiling): build native macos arm64 wheel (#12029)

.github/workflows/build_python_3.yml

@@ -35,7 +35,8 @@ jobs:
               cibuildwheel --print-build-identifiers --platform linux --arch x86_64,i686 | jq -cR '{only: ., os: "ubuntu-latest"}' \
               && cibuildwheel --print-build-identifiers --platform linux --arch aarch64  | jq -cR '{only: ., os: "arm-4core-linux"}' \
               && cibuildwheel --print-build-identifiers --platform windows --arch AMD64,x86 | grep -v 313 | jq -cR '{only: ., os: "windows-latest"}' \
-              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,universal2 | jq -cR '{only: ., os: "macos-13"}'
+              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | jq -cR '{only: ., os: "macos-13"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch arm64 | jq -cR '{only: ., os: "macos-latest"}'
             } | jq -sc
           )
           echo $MATRIX_INCLUDE
@@ -112,6 +113,7 @@ jobs:
             choco install -y 7zip &&
             7z d -r "{wheel}" *.c *.cpp *.cc *.h *.hpp *.pyx &&
             move "{wheel}" "{dest_dir}"
+          CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
           # DEV: Uncomment to debug MacOS
           # CIBW_BUILD_VERBOSITY_MACOS: 3
 
@@ -152,6 +154,7 @@ jobs:
             choco install -y 7zip &&
             7z d -r "{wheel}" *.c *.cpp *.cc *.h *.hpp *.pyx &&
             move "{wheel}" "{dest_dir}"
+          CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
           # DEV: Uncomment to debug MacOS
           # CIBW_BUILD_VERBOSITY_MACOS: 3
 

ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake

@@ -32,6 +32,7 @@ function(add_ddup_config target)
     if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
         # macOS-specific linker options
         target_link_options(${target} PRIVATE "$<$<CONFIG:Release>:-Wl,-dead_strip>")
+        target_link_options(${target} PRIVATE -ldl -undefined dynamic_lookup)
     else()
         # Linux/ELF-based linker options
         target_link_options(

ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt

@@ -58,16 +58,15 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "")
 
 # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is
 # typical.
-set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
+if(APPLE)
+    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..")
+elseif(UNIX)
+    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
+endif()
 target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include ${Datadog_INCLUDE_DIRS}
                                                      ${Python3_INCLUDE_DIRS})
 
-if(Python3_LIBRARIES)
-    target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper ${Python3_LIBRARIES})
-else()
-    # for manylinux builds
-    target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper)
-endif()
+target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper)
 
 # Set the output directory for the built library
 if(LIB_INSTALL_DIR)
@@ -87,19 +86,19 @@ if(NOT CRASHTRACKER_EXE_TARGET_NAME)
     message(FATAL_ERROR "CRASHTRACKER_EXE_TARGET_NAME not set")
 endif()
 
-set_target_properties(crashtracker_exe PROPERTIES INSTALL_RPATH "$ORIGIN/.." OUTPUT_NAME
-                                                                             ${CRASHTRACKER_EXE_TARGET_NAME})
+if(APPLE)
+    set_target_properties(crashtracker_exe PROPERTIES INSTALL_RPATH "@loader_path/.." OUTPUT_NAME
+                                                                                      ${CRASHTRACKER_EXE_TARGET_NAME})
+elseif(UNIX)
+    set_target_properties(crashtracker_exe PROPERTIES INSTALL_RPATH "$ORIGIN/.." OUTPUT_NAME
+                                                                                 ${CRASHTRACKER_EXE_TARGET_NAME})
 
-# To let crashtracker find Python library at runtime
-set_target_properties(crashtracker_exe PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE)
-
-if(Python3_LIBRARIES)
-    target_link_libraries(crashtracker_exe PRIVATE dd_wrapper ${Python3_LIBRARIES})
-else()
-    # for manylinux builds
-    target_link_libraries(crashtracker_exe PRIVATE dd_wrapper)
+    # To let crashtracker find Python library at runtime
+    set_target_properties(crashtracker_exe PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE)
 endif()
 
+target_link_libraries(crashtracker_exe PRIVATE dd_wrapper)
+
 # See the dd_wrapper CMakeLists.txt for a more detailed explanation of why we do what we do.
 if(INPLACE_LIB_INSTALL_DIR)
     set(LIB_INSTALL_DIR "${INPLACE_LIB_INSTALL_DIR}")

ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt

@@ -61,15 +61,15 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "")
 
 # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is
 # typical.
-set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
+if(APPLE)
+    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..")
+elseif(UNIX)
+    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
+endif()
 target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include ${Datadog_INCLUDE_DIRS}
                                                      ${Python3_INCLUDE_DIRS})
 
-if(Python3_LIBRARIES)
-    target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper ${Python3_LIBRARIES})
-else()
-    target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper)
-endif()
+target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper)
 
 # Set the output directory for the built library
 if(LIB_INSTALL_DIR)

ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt

@@ -95,14 +95,14 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "")
 
 # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is
 # typical.
-set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
+if(APPLE)
+    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..")
+elseif(UNIX)
+    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
+endif()
 
 target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper Threads::Threads)
 
-if(Python3_LIBRARIES)
-    target_link_libraries(${EXTENSION_NAME} PRIVATE ${Python3_LIBRARIES})
-endif()
-
 # Set the output directory for the built library
 if(LIB_INSTALL_DIR)
     install(

ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt

@@ -35,6 +35,11 @@ function(dd_wrapper_add_test name)
     target_include_directories(${name} PRIVATE ../include)
     # this has to refer to the stack_v2 extension name to properly link against
     target_link_libraries(${name} PRIVATE gmock gtest_main ${EXTENSION_NAME})
+
+    if(Python3_LIBRARIES)
+        target_link_libraries(${name} PRIVATE ${Python3_LIBRARIES})
+    endif()
+
     set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/../stack_v2")
 
     add_ddup_config(${name})

pyproject.toml

@@ -200,12 +200,6 @@ exclude_dirs = [
   "ddtrace/sourcecode/_utils.py",
 ]
 
-[tool.cibuildwheel]
-build = ["cp27-*", "cp35-*", "cp36-*", "cp37-*", "cp38-*", "cp39-*", "cp310-*", "cp311-*"]
-test-command = ["python {project}/tests/smoke_test.py"]
-# Skip trying to test arm64 builds on Intel Macs
-test-skip = "*-macosx_universal2:arm64"
-
 [tool.ruff]
 exclude = [
     ".riot",

releasenotes/notes/ci-mac-build-37f7e55c8289de61.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    profiling: fixes an issue on arm64 macOS where the profiler was not able
+    to find native extension modules to enable the following features:
+    ``DD_PROFILING_TIMELINE_ENABLED`` and ``DD_PROFILING_STACK_V2_ENABLED``.