Skip to content

Commit

Permalink
Merge branch 'main' into munir/get-rid-of-require-modules-integrations
Browse files Browse the repository at this point in the history
  • Loading branch information
mabdinur authored Dec 19, 2024
2 parents 1ad8ae1 + 5cee25e commit 8b0def1
Show file tree
Hide file tree
Showing 482 changed files with 36,615 additions and 4,128 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.templ.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ mongo_image: &mongo_image mongo:3.6@sha256:19c11a8f1064fd2bb713ef1270f79a742a184
httpbin_image: &httpbin_image kennethreitz/httpbin@sha256:2c7abc4803080c22928265744410173b6fea3b898872c01c5fd0f0f9df4a59fb
vertica_image: &vertica_image vertica/vertica-ce:latest
rabbitmq_image: &rabbitmq_image rabbitmq:3.7-alpine
testagent_image: &testagent_image ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.17.0
testagent_image: &testagent_image ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.20.0

parameters:
coverage:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/build_deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
build_wheels:
uses: ./.github/workflows/build_python_3.yml
with:
cibw_build: 'cp37* cp38* cp39* cp310* cp311* cp312*'
cibw_build: 'cp37* cp38* cp39* cp310* cp311* cp312* cp313*'

build_sdist:
name: Build source distribution
Expand All @@ -40,7 +40,7 @@ jobs:
- uses: actions/setup-python@v5
name: Install Python
with:
python-version: '3.7'
python-version: '3.12'
- name: Build sdist
run: |
pip install "setuptools_scm[toml]>=4" "cython" "cmake>=3.24.2,<3.28" "setuptools-rust"
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/build_python_3.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: '3.8'
- run: pip install cibuildwheel==2.16.5
- run: pip install cibuildwheel==2.22.0
- id: set-matrix
env:
CIBW_BUILD: ${{ inputs.cibw_build }}
Expand All @@ -34,7 +34,7 @@ jobs:
{
cibuildwheel --print-build-identifiers --platform linux --arch x86_64,i686 | jq -cR '{only: ., os: "ubuntu-latest"}' \
&& cibuildwheel --print-build-identifiers --platform linux --arch aarch64 | jq -cR '{only: ., os: "arm-4core-linux"}' \
&& cibuildwheel --print-build-identifiers --platform windows --arch AMD64,x86 | jq -cR '{only: ., os: "windows-latest"}' \
&& cibuildwheel --print-build-identifiers --platform windows --arch AMD64,x86 | grep -v 313 | jq -cR '{only: ., os: "windows-latest"}' \
&& cibuildwheel --print-build-identifiers --platform macos --arch x86_64,universal2 | jq -cR '{only: ., os: "macos-13"}'
} | jq -sc
)
Expand Down Expand Up @@ -83,7 +83,7 @@ jobs:

- name: Build wheels arm64
if: always() && matrix.os == 'arm-4core-linux'
run: /home/runner/.local/bin/pipx run cibuildwheel==2.16.5 --only ${{ matrix.only }}
run: /home/runner/.local/bin/pipx run cibuildwheel==2.22.0 --only ${{ matrix.only }}
env:
CIBW_SKIP: ${{ inputs.cibw_skip }}
CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }}
Expand All @@ -107,7 +107,7 @@ jobs:
rm -rf ./tempwheelhouse
CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
zip -d {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx &&
delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS:
choco install -y 7zip &&
7z d -r "{wheel}" *.c *.cpp *.cc *.h *.hpp *.pyx &&
Expand All @@ -117,7 +117,7 @@ jobs:

- name: Build wheels
if: always() && matrix.os != 'arm-4core-linux'
uses: pypa/cibuildwheel@v2.16.5
uses: pypa/cibuildwheel@v2.22.0
with:
only: ${{ matrix.only }}
env:
Expand All @@ -143,7 +143,7 @@ jobs:
rm -rf ./tempwheelhouse
CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
zip -d {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx &&
delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS:
choco install -y 7zip &&
7z d -r "{wheel}" *.c *.cpp *.cc *.h *.hpp *.pyx &&
Expand Down
7 changes: 6 additions & 1 deletion .github/workflows/generate-package-versions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
jobs:
generate-package-versions:
name: Generate package versions
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
permissions:
actions: read
contents: write
Expand Down Expand Up @@ -49,6 +49,11 @@ jobs:
with:
python-version: "3.12"

- name: Setup Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

Expand Down
121 changes: 121 additions & 0 deletions .github/workflows/generate-supported-versions.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
name: Generate Supported Integration Versions

on:
workflow_dispatch: # can be triggered manually

jobs:
generate-supported-versions:
name: Generate supported integration versions
runs-on: ubuntu-22.04
permissions:
actions: read
contents: write
pull-requests: write

steps:
- uses: actions/checkout@v4
with:
persist-credentials: false

- name: Setup Python 3.7
uses: actions/setup-python@v5
with:
python-version: "3.7"

- name: Setup Python 3.8
uses: actions/setup-python@v5
with:
python-version: "3.8"

- name: Setup Python 3.9
uses: actions/setup-python@v5
with:
python-version: "3.9"

- name: Setup Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Setup Python 3.11
uses: actions/setup-python@v5
with:
python-version: "3.11"

- name: Setup Python 3.12
uses: actions/setup-python@v5
with:
python-version: "3.12"

- name: Setup Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y libmariadb-dev
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install packaging
pip install requests
pip install riot==0.20.1
pip install wrapt==1.16.0
- name: Install ddtrace
run: |
pip install -e .
- run: python scripts/freshvenvs.py generate

- name: Generate table
run: python scripts/generate_table.py

- run: git diff

- name: Create Pull Request
id: pr
uses: peter-evans/create-pull-request@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
branch: "update-supported-versions"
commit-message: "Update supported versions table"
delete-branch: true
base: main
title: "chore: update supported versions"
labels: changelog/no-changelog
body: |
Generates / updates the supported versions table for integrations.
This should be tied to releases, or triggered manually.
Workflow runs: [Generate Supported Integration Versions](https://github.com/DataDog/dd-trace-py/actions/workflows/generate-supported-versions.yml)
## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))
## Reviewer Checklist
- [ ] Reviewer has checked that all the criteria below are met
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
6 changes: 5 additions & 1 deletion .github/workflows/profiling-native.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
matrix:
os: [ubuntu-24.04]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
sanitizer: ["safety", "thread"]
sanitizer: ["safety", "thread", "valgrind"]

steps:
- uses: actions/checkout@v4
Expand All @@ -40,6 +40,10 @@ jobs:
chmod +x llvm.sh
sudo ./llvm.sh 19
- name: Install Valgrind
run: |
sudo apt-get install -y valgrind
- name: Run tests with sanitizers
run: |
# DEV: We currently have tests in dd_wrapper and stack_v2, setting
Expand Down
43 changes: 43 additions & 0 deletions .github/workflows/pytorch_gpu_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: Pytorch Unit Tests (with GPU)

on:
push:
branches:
- 'main'
- 'mq-working-branch**'
paths:
- 'ddtrace/profiling/collector/pytorch.py'
pull_request:
paths:
- 'ddtrace/profiling/collector/pytorch.py'
workflow_dispatch:

jobs:
unit-tests:
runs-on: APM-4-CORE-GPU-LINUX
steps:
- uses: actions/checkout@v4
# Include all history and tags
with:
persist-credentials: false
fetch-depth: 0

- uses: actions/setup-python@v5
name: Install Python
with:
python-version: '3.12'

- uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Install latest stable toolchain and rustfmt
run: rustup update stable && rustup default stable && rustup component add rustfmt clippy

- name: Install hatch
uses: pypa/hatch@install
with:
version: "1.12.0"

- name: Install PyTorch
run: pip install torch

- name: Run tests
run: hatch run profiling_pytorch:test
4 changes: 2 additions & 2 deletions .github/workflows/requirements-locks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
validate:
name: Check requirements lockfiles
runs-on: ubuntu-latest
container: ghcr.io/datadog/dd-trace-py/testrunner:47c7b5287da25643e46652e6d222a40a52f2382a@sha256:3a02dafeff9cd72966978816d1b39b54f5517af4049396923b95c8452f604269
container: ghcr.io/datadog/dd-trace-py/testrunner:0a50e839f4b1600f02157518b8d016451b346578@sha256:5dae9bc7872f69b31b612690f0748c7ad71ab90ef28a754b2ae93d0ba505837b
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -23,7 +23,7 @@ jobs:
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

- name: Set python interpreters
run: pyenv global 3.10 3.7 3.8 3.9 3.11 3.12
run: pyenv global 3.10 3.7 3.8 3.9 3.11 3.12 3.13

- name: Install Dependencies
run: pip install --upgrade pip && pip install riot==0.20.1
Expand Down
16 changes: 14 additions & 2 deletions .github/workflows/system-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
# system-tests requires an API_KEY, but it does not have to be a valid key, as long as we don't run a scenario
# that make assertion on backend data. Using a fake key allow to run system tests on PR originating from forks.
# If ever it's needed, a valid key exists in the repo, using ${{ secrets.DD_API_KEY }}
DD_API_KEY: 1234567890abcdef1234567890abcdef
DD_API_KEY: ${{ secrets.FAKE_DD_API_KEY }}
CMAKE_BUILD_PARALLEL_LEVEL: 12
SYSTEM_TESTS_AWS_ACCESS_KEY_ID: ${{ secrets.IDM_AWS_ACCESS_KEY_ID }}
SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: ${{ secrets.IDM_AWS_SECRET_ACCESS_KEY }}
Expand Down Expand Up @@ -106,7 +106,7 @@ jobs:
# system-tests requires an API_KEY, but it does not have to be a valid key, as long as we don't run a scenario
# that make assertion on backend data. Using a fake key allow to run system tests on PR originating from forks.
# If ever it's needed, a valid key exists in the repo, using ${{ secrets.DD_API_KEY }}
DD_API_KEY: 1234567890abcdef1234567890abcdef
DD_API_KEY: ${{ secrets.FAKE_DD_API_KEY }}
CMAKE_BUILD_PARALLEL_LEVEL: 12
SYSTEM_TESTS_AWS_ACCESS_KEY_ID: ${{ secrets.IDM_AWS_ACCESS_KEY_ID }}
SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: ${{ secrets.IDM_AWS_SECRET_ACCESS_KEY }}
Expand Down Expand Up @@ -153,6 +153,14 @@ jobs:
if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'other'
run: ./run.sh CROSSED_TRACING_LIBRARIES

- name: Run PROFILING
if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'other'
run: |
cat /proc/sys/kernel/perf_event_paranoid
sudo sysctl kernel.perf_event_paranoid=1
sudo sysctl -p
./run.sh PROFILING
- name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'remote-config'
run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
Expand Down Expand Up @@ -205,6 +213,10 @@ jobs:
if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1'
run: ./run.sh IAST_STANDALONE

- name: Run SCA_STANDALONE
if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1'
run: ./run.sh SCA_STANDALONE

- name: Run APPSEC_RUNTIME_ACTIVATION
if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1'
run: ./run.sh APPSEC_RUNTIME_ACTIVATION
Expand Down
14 changes: 14 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ stages:
- package
- tests-gen
- tests-trigger
- quality-gate
- shared-pipeline
- benchmarks
- macrobenchmarks
Expand Down Expand Up @@ -87,3 +88,16 @@ deploy_to_di_backend:manual:
UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
UPSTREAM_TAG: $CI_COMMIT_TAG
UPSTREAM_PACKAGE_JOB: build

check_new_flaky_tests:
stage: quality-gate
extends: .testrunner
script:
- export DD_SITE=datadoghq.com
- export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.${CI_PROJECT_NAME}.dd-api-key-qualitygate --with-decryption --query "Parameter.Value" --out text)
- export DD_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.${CI_PROJECT_NAME}.dd-app-key-qualitygate --with-decryption --query "Parameter.Value" --out text)
- datadog-ci gate evaluate
except:
- main
- '[0-9].[0-9]*'
- 'mq-working-branch**'
2 changes: 1 addition & 1 deletion .gitlab/download-dependency-wheels.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export PYTHONUNBUFFERED=TRUE
--local-ddtrace \
--arch x86_64 \
--arch aarch64 \
--platform musllinux_1_1 \
--platform musllinux_1_2 \
--platform manylinux2014 \
--output-dir ../pywheels-dep \
--verbose
2 changes: 2 additions & 0 deletions .gitlab/package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ download_dependency_wheels:
PYTHON_VERSION: "3.11"
- PYTHON_IMAGE_TAG: "3.12.0"
PYTHON_VERSION: "3.12"
- PYTHON_IMAGE_TAG: "3.13.0"
PYTHON_VERSION: "3.13"
script:
- .gitlab/download-dependency-wheels.sh
artifacts:
Expand Down
Loading

0 comments on commit 8b0def1

Please sign in to comment.