chore(iast): smoke tests II (#9364)

Enable more packages and filter by python version or add docstring to
explain why is not working

This PR continues #9348
## Checklist

- [x] Change(s) are motivated and described in the PR description
- [x] Testing strategy is described if automated tests are not included
in the PR
- [x] Risks are described (performance impact, potential for breakage,
maintainability)
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed or label `changelog/no-changelog` is set
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/))
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))
- [x] If this PR changes the public interface, I've notified
`@DataDog/apm-tees`.
- [x] If change touches code that signs or publishes builds or packages,
or handles credentials of any kind, I've requested a review from
`@DataDog/security-design-and-guidance`.

## Reviewer Checklist

- [x] Title is accurate
- [x] All changes are related to the pull request's stated goal
- [x] Description motivates each change
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- [x] Testing strategy adequately addresses listed risks
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] Release note makes sense to a user of the library
- [x] Author has acknowledged and discussed the performance implications
of this PR as reported in the benchmarks PR comment
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

tests/appsec/iast_packages/test_packages.py

@@ -12,8 +12,11 @@
 from tests.utils import override_env
 
 
+PYTHON_VERSION = sys.version_info[:2]
+
+
 class PackageForTesting:
-    package_name = ""
+    name = ""
     import_name = ""
     package_version = ""
     url_to_test = ""
@@ -22,6 +25,7 @@ class PackageForTesting:
     expected_result2 = ""
     extra_packages = []
     test_import = True
+    test_import_python_versions_to_skip = []
     test_e2e = True
 
     def __init__(
@@ -33,12 +37,14 @@ def __init__(
         expected_result2,
         extras=[],
         test_import=True,
+        skip_python_version=[],
         test_e2e=True,
         import_name=None,
     ):
-        self.package_name = name
+        self.name = name
         self.package_version = version
         self.test_import = test_import
+        self.test_import_python_versions_to_skip = skip_python_version
         self.test_e2e = test_e2e
         if expected_param:
             self.expected_param = expected_param
@@ -51,29 +57,46 @@ def __init__(
         if import_name:
             self.import_name = import_name
         else:
-            self.import_name = self.package_name
+            self.import_name = self.name
 
     @property
     def url(self):
-        return f"/{self.package_name}?package_param={self.expected_param}"
+        return f"/{self.name}?package_param={self.expected_param}"
+
+    def __str__(self):
+        return f"{self.name}=={self.package_version}: {self.url_to_test}"
 
     def __repr__(self):
-        return f"{self.package_name}: {self.url_to_test}"
+        return f"{self.name}=={self.package_version}: {self.url_to_test}"
+
+    @property
+    def skip(self):
+        for version in self.test_import_python_versions_to_skip:
+            if version == PYTHON_VERSION:
+                return True, f"{self.name} not yet compatible with Python {version}"
+        return False, ""
+
+    def _install(self, package_name, package_version=""):
+        if package_version:
+            package_fullversion = package_name + "==" + package_version
+        else:
+            package_fullversion = package_name
 
-    def _install(self, package_name, package_version):
-        package_fullversion = package_name + "==" + package_version
         cmd = ["python", "-m", "pip", "install", package_fullversion]
         env = {}
         env.update(os.environ)
         # CAVEAT: we use subprocess instead of `pip.main(["install", package_fullversion])` due to pip package
         # doesn't work correctly with riot environment and python packages path
         proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr, close_fds=True, env=env)
         proc.wait()
-        print(proc.stdout)
-        print(proc.stderr)
 
     def install(self):
-        self._install(self.package_name, self.package_version)
+        self._install(self.name, self.package_version)
+        for package_name, package_version in self.extra_packages:
+            self._install(package_name, package_version)
+
+    def install_latest(self):
+        self._install(self.name)
         for package_name, package_version in self.extra_packages:
             self._install(package_name, package_version)
 
@@ -84,6 +107,7 @@ def install(self):
 
 # pypular package is discarded because it is not a real top package
 # wheel, importlib-metadata and pip is discarded because they are package to build projects
+# colorama and awscli are terminal commands
 PACKAGES = [
     PackageForTesting(
         "charset-normalizer", "3.3.2", "my-bytes-string", "my-bytes-string", "", import_name="charset_normalizer"
@@ -98,7 +122,8 @@ def install(self):
         import_name="googleapiclient",
     ),
     PackageForTesting("idna", "3.6", "xn--eckwd4c7c.xn--zckzah", "ドメイン.テスト", "xn--eckwd4c7c.xn--zckzah"),
-    # PackageForTesting("numpy", "1.24.4", "9 8 7 6 5 4 3", [3, 4, 5, 6, 7, 8, 9], 5),
+    # Python 3.12 fails in all steps with "import error" when import numpy
+    PackageForTesting("numpy", "1.24.4", "9 8 7 6 5 4 3", [3, 4, 5, 6, 7, 8, 9], 5, skip_python_version=[(3, 12)]),
     PackageForTesting(
         "python-dateutil",
         "2.8.2",
@@ -131,7 +156,19 @@ def install(self):
     PackageForTesting("cryptography", "42.0.7", "", "", "", test_e2e=False),
     PackageForTesting("fsspec", "2024.5.0", "", "", "", test_e2e=False, test_import=False),
     PackageForTesting("boto3", "1.34.110", "", "", "", test_e2e=False, test_import=False),
-    # PackageForTesting("typing-extensions", "4.11.0", "", "", "", import_name="typing_extensions", test_e2e=False),
+    # Python 3.8 fails in test_packages_patched_import with
+    # TypeError: '>' not supported between instances of 'int' and 'object'
+    # TODO: try to fix it
+    PackageForTesting(
+        "typing-extensions",
+        "4.11.0",
+        "",
+        "",
+        "",
+        import_name="typing_extensions",
+        test_e2e=False,
+        skip_python_version=[(3, 8)],
+    ),
     PackageForTesting("botocore", "1.34.110", "", "", "", test_e2e=False),
     PackageForTesting("packaging", "24.0", "", "", "", test_e2e=False),
     PackageForTesting("cffi", "1.16.0", "", "", "", test_e2e=False),
@@ -142,85 +179,108 @@ def install(self):
     PackageForTesting("google-api-core", "2.19.0", "", "", "", test_e2e=False, import_name="google"),
     PackageForTesting("cffi", "1.16.0", "", "", "", test_e2e=False),
     PackageForTesting("pycparser", "2.22", "", "", "", test_e2e=False),
-    # PackageForTesting("grpcio-status", "1.64.0", "", "", "", test_e2e=False),
-    # PackageForTesting("pandas", "2.2.2", "", "", "", test_e2e=False),
+    # Pandas dropped Python 3.8 support in pandas>2.0.3
+    PackageForTesting("pandas", "2.2.2", "", "", "", test_e2e=False, skip_python_version=[(3, 8)]),
     PackageForTesting("zipp", "3.18.2", "", "", "", test_e2e=False),
     PackageForTesting("attrs", "23.2.0", "", "", "", test_e2e=False),
     PackageForTesting("pyasn1", "0.6.0", "", "", "", test_e2e=False),
     PackageForTesting("rsa", "4.9", "", "", "", test_e2e=False),
+    # protobuf fails for all python versions with No module named 'protobuf
     # PackageForTesting("protobuf", "5.26.1", "", "", "", test_e2e=False),
     PackageForTesting("jmespath", "1.0.1", "", "", "", test_e2e=False),
     PackageForTesting("click", "8.1.7", "", "", "", test_e2e=False),
     PackageForTesting("pydantic", "2.7.1", "", "", "", test_e2e=False),
     PackageForTesting("pytz", "2024.1", "", "", "", test_e2e=False),
-    # PackageForTesting("colorama", "0.4.6", "", "", "", test_e2e=False),
-    # PackageForTesting("awscli", "1.32.110", "", "", "", test_e2e=False),
     PackageForTesting("markupsafe", "2.1.5", "", "", "", test_e2e=False),
     PackageForTesting("jinja2", "3.1.4", "", "", "", test_e2e=False),
     PackageForTesting("platformdirs", "4.2.2", "", "", "", test_e2e=False),
-    # PackageForTesting("pyjwt", "2.8.0", "", "", "", test_e2e=False, import_name="jwt"),
+    PackageForTesting("pyjwt", "2.8.0", "", "", "", test_e2e=False, import_name="jwt"),
     PackageForTesting("tomli", "2.0.1", "", "", "", test_e2e=False),
-    # PackageForTesting("googleapis-common-protos", "1.63.0", "", "", "", test_e2e=False),
     PackageForTesting("filelock", "3.14.0", "", "", "", test_e2e=False),
-    # PackageForTesting("google-auth", "2.29.0", "", "", "", test_e2e=False),
     PackageForTesting("wrapt", "1.16.0", "", "", "", test_e2e=False),
     PackageForTesting("cachetools", "5.3.3", "", "", "", test_e2e=False),
     PackageForTesting("pluggy", "1.5.0", "", "", "", test_e2e=False),
     PackageForTesting("virtualenv", "20.26.2", "", "", "", test_e2e=False),
-    # PackageForTesting("docutils", "0.21.2", "", "", "", test_e2e=False),
-    # PackageForTesting("pyarrow", "16.1.0", "", "", "", test_e2e=False),
+    # docutils dropped Python 3.8 support in pandas> 1.10.10.21.2
+    PackageForTesting("docutils", "0.21.2", "", "", "", test_e2e=False, skip_python_version=[(3, 8)]),
+    PackageForTesting("pyarrow", "16.1.0", "", "", "", test_e2e=False),
     PackageForTesting("exceptiongroup", "1.2.1", "", "", "", test_e2e=False),
-    # PackageForTesting("jsonschema", "4.22.0", "", "", "", test_e2e=False),
+    # jsonschema fails for Python 3.8
+    #        except KeyError:
+    # >           raise exceptions.NoSuchResource(ref=uri) from None
+    # E           referencing.exceptions.NoSuchResource: 'http://json-schema.org/draft-03/schema#'
+    PackageForTesting("jsonschema", "4.22.0", "", "", "", test_e2e=False, skip_python_version=[(3, 8)]),
     PackageForTesting("requests-oauthlib", "2.0.0", "", "", "", test_e2e=False, import_name="requests_oauthlib"),
     PackageForTesting("pyparsing", "3.1.2", "", "", "", test_e2e=False),
     PackageForTesting("pytest", "8.2.1", "", "", "", test_e2e=False),
     PackageForTesting("oauthlib", "3.2.2", "", "", "", test_e2e=False),
     PackageForTesting("sqlalchemy", "2.0.30", "", "", "", test_e2e=False),
-    # PackageForTesting("pyasn1-modules", "0.4.0", "", "", "", test_e2e=False),
     PackageForTesting("aiohttp", "3.9.5", "", "", "", test_e2e=False),
-    # PackageForTesting("scipy", "1.13.0", "", "", "", test_e2e=False, import_name="scipy.special"),
-    # PackageForTesting("isodate", "0.6.1", "", "", "", test_e2e=False),
+    # scipy dropped Python 3.8 support in pandas> 1.10.1
+    PackageForTesting(
+        "scipy", "1.13.0", "", "", "", test_e2e=False, import_name="scipy.special", skip_python_version=[(3, 8)]
+    ),
+    PackageForTesting("isodate", "0.6.1", "", "", "", test_e2e=False),
     PackageForTesting("multidict", "6.0.5", "", "", "", test_e2e=False),
     PackageForTesting("iniconfig", "2.0.0", "", "", "", test_e2e=False),
     PackageForTesting("psutil", "5.9.8", "", "", "", test_e2e=False),
     PackageForTesting("soupsieve", "2.5", "", "", "", test_e2e=False),
     PackageForTesting("yarl", "1.9.4", "", "", "", test_e2e=False),
-    # PackageForTesting("async-timeout", "4.0.3", "", "", "", test_e2e=False),
     PackageForTesting("frozenlist", "1.4.1", "", "", "", test_e2e=False),
     PackageForTesting("aiosignal", "1.3.1", "", "", "", test_e2e=False),
     PackageForTesting("werkzeug", "3.0.3", "", "", "", test_e2e=False),
-    # PackageForTesting("pillow", "10.3.0", "", "", "", test_e2e=False, import_name="PIL.Image"),
+    PackageForTesting("pillow", "10.3.0", "", "", "", test_e2e=False, import_name="PIL.Image"),
     PackageForTesting("tqdm", "4.66.4", "", "", "", test_e2e=False),
     PackageForTesting("pygments", "2.18.0", "", "", "", test_e2e=False),
-    # PackageForTesting("grpcio", "1.64.0", "", "", "", test_e2e=False),
+    PackageForTesting("grpcio", "1.64.0", "", "", "", test_e2e=False, import_name="grpc"),
     PackageForTesting("greenlet", "3.0.3", "", "", "", test_e2e=False),
     PackageForTesting("pyopenssl", "24.1.0", "", "", "", test_e2e=False, import_name="OpenSSL.SSL"),
     PackageForTesting("flask", "3.0.3", "", "", "", test_e2e=False),
     PackageForTesting("decorator", "5.1.1", "", "", "", test_e2e=False),
     PackageForTesting("pydantic-core", "2.18.2", "", "", "", test_e2e=False, import_name="pydantic_core"),
-    # PackageForTesting("lxml", "5.2.2", "", "", "", test_e2e=False, import_name="lxml.etree"),
+    PackageForTesting("lxml", "5.2.2", "", "", "", test_e2e=False, import_name="lxml.etree"),
     PackageForTesting("requests-toolbelt", "1.0.0", "", "", "", test_e2e=False, import_name="requests_toolbelt"),
-    # PackageForTesting("openpyxl", "3.1.2", "", "", "", test_e2e=False, import_name="openpyxl.Workbook"),
+    PackageForTesting("openpyxl", "3.1.2", "", "", "", test_e2e=False),
     PackageForTesting("tzdata", "2024.1", "", "", "", test_e2e=False),
-    # PackageForTesting("et-xmlfile", "1.1.0", "", "", "", test_e2e=False),
-    # PackageForTesting("importlib-resources", "6.4.0", "", "", "", test_e2e=False, import_name="importlib_resources"),
-    # PackageForTesting("proto-plus", "1.23.0", "", "", "", test_e2e=False),
-    # PackageForTesting("asn1crypto", "1.5.1", "", "", "", test_e2e=False),
+    PackageForTesting(
+        "importlib-resources",
+        "6.4.0",
+        "",
+        "",
+        "",
+        test_e2e=False,
+        import_name="importlib_resources",
+        skip_python_version=[(3, 8)],
+    ),
+    PackageForTesting("asn1crypto", "1.5.1", "", "", "", test_e2e=False),
     PackageForTesting("coverage", "7.5.1", "", "", "", test_e2e=False),
-    # PackageForTesting("azure-core", "1.30.1", "", "", "", test_e2e=False, import_name="azure"),
+    PackageForTesting("azure-core", "1.30.1", "", "", "", test_e2e=False, import_name="azure"),
     PackageForTesting("distlib", "0.3.8", "", "", "", test_e2e=False),
     PackageForTesting("tomlkit", "0.12.5", "", "", "", test_e2e=False),
-    # PackageForTesting("pynacl", "1.5.0", "", "", "", test_e2e=False),
+    PackageForTesting("pynacl", "1.5.0", "", "", "", test_e2e=False, import_name="nacl.utils"),
     PackageForTesting("itsdangerous", "2.2.0", "", "", "", test_e2e=False),
-    # PackageForTesting("annotated-types", "0.7.0", "", "", "", test_e2e=False),
+    PackageForTesting("annotated-types", "0.7.0", "", "", "", test_e2e=False, import_name="annotated_types"),
     PackageForTesting("sniffio", "1.3.1", "", "", "", test_e2e=False),
     PackageForTesting("more-itertools", "10.2.0", "", "", "", test_e2e=False, import_name="more_itertools"),
-    # PackageForTesting("google-cloud-storage", "2.16.0", "", "", "", test_e2e=False),
 ]
 
 
-@pytest.mark.parametrize("package", [package for package in PACKAGES if package.test_e2e])
+# Use this function if you want to test one or a filter number of package for debug proposes
+# SKIP_FUNCTION = lambda package: package.name == "pynacl"  # noqa: E731
+SKIP_FUNCTION = lambda package: True  # noqa: E731
+
+
+@pytest.mark.parametrize(
+    "package",
+    [package for package in PACKAGES if package.test_e2e and SKIP_FUNCTION(package)],
+    ids=lambda package: package.name,
+)
 def test_packages_not_patched(package):
+    should_skip, reason = package.skip
+    if should_skip:
+        pytest.skip(reason)
+        return
+
     package.install()
     with flask_server(
         iast_enabled="false", tracer_enabled="true", remote_configuration_enabled="false", token=None
@@ -237,8 +297,17 @@ def test_packages_not_patched(package):
         assert content["params_are_tainted"] is False
 
 
-@pytest.mark.parametrize("package", [package for package in PACKAGES if package.test_e2e])
+@pytest.mark.parametrize(
+    "package",
+    [package for package in PACKAGES if package.test_e2e and SKIP_FUNCTION(package)],
+    ids=lambda package: package.name,
+)
 def test_packages_patched(package):
+    should_skip, reason = package.skip
+    if should_skip:
+        pytest.skip(reason)
+        return
+
     package.install()
     with flask_server(iast_enabled="true", remote_configuration_enabled="false", token=None) as context:
         _, client, pid = context
@@ -253,14 +322,63 @@ def test_packages_patched(package):
         assert content["params_are_tainted"] is True
 
 
-@pytest.mark.parametrize("package", [package for package in PACKAGES if package.test_import])
+@pytest.mark.parametrize(
+    "package",
+    [package for package in PACKAGES if package.test_import and SKIP_FUNCTION(package)],
+    ids=lambda package: package.name,
+)
 def test_packages_not_patched_import(package):
+    should_skip, reason = package.skip
+    if should_skip:
+        pytest.skip(reason)
+        return
+
     package.install()
     importlib.import_module(package.import_name)
 
 
-@pytest.mark.parametrize("package", [package for package in PACKAGES if package.test_import])
+@pytest.mark.parametrize(
+    "package",
+    [package for package in PACKAGES if package.test_import and SKIP_FUNCTION(package)],
+    ids=lambda package: package.name,
+)
 def test_packages_patched_import(package):
+    should_skip, reason = package.skip
+    if should_skip:
+        pytest.skip(reason)
+        return
+
     with override_env({IAST_ENV: "true"}):
         package.install()
         assert _iast_patched_module(package.import_name, fromlist=[])
+
+
+@pytest.mark.parametrize(
+    "package",
+    [package for package in PACKAGES if package.test_import and SKIP_FUNCTION(package)],
+    ids=lambda package: package.name,
+)
+def test_packages_latest_not_patched_import(package):
+    should_skip, reason = package.skip
+    if should_skip:
+        pytest.skip(reason)
+        return
+
+    package.install_latest()
+    importlib.import_module(package.import_name)
+
+
+@pytest.mark.parametrize(
+    "package",
+    [package for package in PACKAGES if package.test_import and SKIP_FUNCTION(package)],
+    ids=lambda package: package.name,
+)
+def test_packages_latest_patched_import(package):
+    should_skip, reason = package.skip
+    if should_skip:
+        pytest.skip(reason)
+        return
+
+    with override_env({IAST_ENV: "true"}):
+        package.install_latest()
+        assert _iast_patched_module(package.import_name, fromlist=[])