ci: merge suitespec and jobspec (#11401)

We merge the suitespec and jobspec into a unique, modular solution that contains all the information to dynamically generate CI jobs based on the changes contained in a PR. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
DataDog · Nov 26, 2024 · d386312 · d386312
1 parent 9c9fe4d
commit d386312
Show file tree

Hide file tree

Showing 25 changed files with 1,996 additions and 2,293 deletions.
diff --git a/.circleci/config.templ.yml b/.circleci/config.templ.yml
@@ -281,109 +281,6 @@ commands:
                 name: Get APM Test Agent Trace Check Results
                 command: bash ./scripts/get-test-agent-results.sh
 
-  run_hatch_env_test:
-    description: "Run hatch env test"
-    parameters:
-      env:
-        type: string
-        default: ""
-      snapshot:
-        type: boolean
-        default: false
-      docker_services:
-        type: string
-        default: ""
-      store_coverage:
-        type: boolean
-        default: true
-      trace_agent_url:
-        type: string
-        default: "http://localhost:9126"
-      run_agent_checks:
-        type: boolean
-        default: true
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - restore_cache:
-          keys:
-            - lastsuccess-{{ .Environment.CIRCLE_BRANCH }}-<<parameters.env>>-{{ .Environment.CIRCLE_NODE_INDEX }}
-      - setup_hatch
-      - when:
-          condition:
-              << parameters.snapshot >>
-          steps:
-            - start_docker_services:
-                env: SNAPSHOT_CI=1
-                services: testagent << parameters.docker_services >>
-            - run:
-                name: Run tests
-                environment:
-                  DD_TRACE_AGENT_URL: << parameters.trace_agent_url >>
-                command: |
-                  ulimit -c unlimited
-                  ./scripts/run-test-suite-hatch '<<parameters.env>>' 1
-            - run:
-                command: |
-                  mkdir -p /tmp/core_dumps
-                  cp core.* /tmp/core_dumps || true
-                  ./scripts/bt
-                when: on_fail
-            - store_artifacts:
-                name: "Store core dumps"
-                path: /tmp/core_dumps
-      - unless:
-          condition:
-              << parameters.snapshot >>
-          steps:
-            - run:
-                name: Run tests
-                command: |
-                  hatch env show --json | jq -r 'keys[] | select(. | contains("<< parameters.env >>"))' | sort | circleci tests split | xargs -n 1 -I {} hatch run {}:test
-      - when:
-          condition:
-            << parameters.store_coverage >>
-          steps:
-            - save_coverage
-      - store_test_results:
-          path: test-results
-      - store_artifacts:
-          path: test-results
-      - save_cache:
-          key: lastsuccess-{{ .Environment.CIRCLE_BRANCH }}-<<parameters.env>>-{{ .Environment.CIRCLE_NODE_INDEX }}-{{ epoch }}
-          paths:
-            - ./latest-success-commit
-      - run:
-          name: "Store Test Agent Supported Integrations Data"
-          command: |
-            if [[ -z "$(curl -s << parameters.trace_agent_url >>/test/integrations/tested_versions)" ]]; then
-              # No integrations were tested. Not saving any artifacts
-              echo "Response body is empty. Skipping saving integration artifacts."
-            else
-              # make temporary files to save response data to
-              response=$(mktemp) && headers=$(mktemp)
-              # create artifacts directory if it doesn't exist
-              [ -d "./artifacts" ] || mkdir -p "./artifacts"
-              # get tested integrations
-              curl -o "$response" -D "$headers" << parameters.trace_agent_url >>/test/integrations/tested_versions
-              # get filename representing the name of the tested integration from headers
-              filename=$(awk -F': ' '/file-name/{print $2}' "$headers" | tr -d '\r\n')
-              # copy data to final file and remove temp files
-              mv "$response" "artifacts/${filename}_supported_versions.csv"
-              rm "$headers"
-            fi
-      - store_artifacts:
-          path: artifacts
-          destination: supported-integrations
-      - when:
-          condition:
-              << parameters.run_agent_checks >>
-          steps:
-            - run:
-                name: Get APM Test Agent Trace Check Results
-                command: bash ./scripts/get-test-agent-results.sh
-
 executors:
   cimg_base:
     docker:

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -39,7 +39,7 @@ ddtrace/internal/compat.py          @DataDog/python-guild  @DataDog/apm-core-pyt
 ddtrace/settings/config.py          @DataDog/python-guild  @DataDog/apm-sdk-api-python
 docs/                               @DataDog/python-guild
 tests/utils.py                      @DataDog/python-guild
-tests/.suitespec.json               @DataDog/python-guild  @DataDog/apm-core-python
+tests/suitespec.yml                 @DataDog/python-guild  @DataDog/apm-core-python
 tests/suitespec.py                  @DataDog/python-guild  @DataDog/apm-core-python
 
 # Core / Language Platform

diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml
@@ -119,18 +119,5 @@ build_base_venvs:
     - export DD_TRACE_AGENT_URL="http://testagent:9126"
     - ln -s "${CI_PROJECT_DIR}" "/root/project"
 
-slotscheck:
-  extends: .testrunner
-  stage: tests
-  needs: []
-  script:
-    - hatch run slotscheck:_
-
-conftests:
-  extends: .testrunner
-  stage: tests
-  needs: []
-  script:
-    - hatch run meta-testing:meta-testing
 
 # Required jobs will appear here
diff --git a/docs/contributing-integrations.rst b/docs/contributing-integrations.rst
@@ -216,7 +216,5 @@ The following is the check list for ensuring you have all of the components to h
 - The virtual environment configurations for your tests in ``riotfile.py``.
 - The Circle CI configurations for your tests in ``.circleci/config.templ.yml``.
 - Your integration added to ``PATCH_MODULES`` in ``ddtrace/_monkey.py`` to enable auto instrumentation for it.
-- The relevant file paths for your integration added to ``tests/.suitespec.json`` in two locations:
-    - Add non-test file paths under ``components``.
-    - Add test file paths under ``suites``.
+- The relevant file paths for your integration added to a suitespec file (see ``tests/README.md`` for details).
 - A release note for your addition generated with ``riot run reno new YOUR_TITLE_SLUG``, which will add ``releasenotes/notes/YOUR_TITLE_SLUG.yml``.
diff --git a/docs/contributing-testing.rst b/docs/contributing-testing.rst
@@ -153,24 +153,8 @@ Next, we will need to add a new CircleCI job to run the newly added test suite a
           pattern: 'asyncio'
 
 
-After this, a new component must be added to ``tests/.suitespec.json`` under ``"components":`` like:
-
-.. code-block:: JSON
-
-    "asyncio": [
-        "ddtrace/contrib/asyncio/*"
-    ],
-
-Lastly, we will register it as a suite in the same file under ``"suites":``:
-
-.. code-block:: JSON
-
-    "asyncio": [
-        "@asyncio",
-        "tests/contrib/asyncio/*"
-    ],
-
-Once you've completed these steps, CircleCI will run the new test suite.
+After this, check out ``tests/README.md`` for instructions on how to add new
+jobs to run the new tests in CI.
 
 How do I update a Riot environment to use the latest version of a package?
 --------------------------------------------------------------------------

diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
@@ -298,4 +298,5 @@ assertIn
 # tests/contrib/openai/test_openai_v1.py
 Nam
 # docs/configuration.rst
-uest
+uest
+suitespec
diff --git a/hatch.toml b/hatch.toml
@@ -21,6 +21,7 @@ dependencies = [
     "ruff==0.1.3",
     "clang-format==18.1.5",
     "cmake-format==0.6.13",
+    "ruamel.yaml==0.18.6",
 ]
 
 [envs.lint.scripts]
@@ -133,6 +134,7 @@ detached = true
 python = "3.10"
 extra-dependencies = [
     "packaging==23.1",
+    "ruamel.yaml==0.18.6",
 ]
 
 [envs.scripts.scripts]

diff --git a/scripts/gen_circleci_config.py b/scripts/gen_circleci_config.py
@@ -11,15 +11,15 @@
 def gen_required_suites(template: dict) -> None:
     """Generate the list of test suites that need to be run."""
     from needs_testrun import extract_git_commit_selections
-    from needs_testrun import for_each_testrun_needed as fetn
+    from needs_testrun import for_each_testrun_needed
     from suitespec import get_suites
 
-    suites = get_suites()
-    jobs = set(template["jobs"].keys())
-
     required_suites = template["requires_tests"]["requires"] = []
-    fetn(
-        suites=sorted(suites & jobs),
+    for_each_testrun_needed(
+        suites=sorted(
+            set(n.rpartition("::")[-1] for n, s in get_suites().items() if not s.get("skip", False))
+            & set(template["jobs"].keys())
+        ),
         action=lambda suite: required_suites.append(suite),
         git_selections=extract_git_commit_selections(os.getenv("GIT_COMMIT_DESC", "")),
     )
@@ -77,12 +77,7 @@ def check(name: str, command: str, paths: t.Set[str]) -> None:
     check(
         name="Run scripts/*.py tests",
         command="hatch run scripts:test",
-        paths={"docker*", "scripts/*.py", "scripts/mkwheelhouse", "scripts/run-test-suite", "tests/.suitespec.json"},
-    )
-    check(
-        name="Validate suitespec JSON file",
-        command="python -m tests.suitespec",
-        paths={"docker*", "tests/.suitespec.json", "tests/suitespec.py"},
+        paths={"docker*", "scripts/*.py", "scripts/mkwheelhouse", "scripts/run-test-suite", "**suitespec.yml"},
     )
     check(
         name="Check suitespec coverage",