From e125f22d4ee397748a07889ce7c451ab4f064bf7 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 24 May 2024 12:39:11 +0200 Subject: [PATCH] chore(iast): smoke tests II (#9364) Enable more packages and filter by python version or add docstring to explain why is not working This PR continues https://github.com/DataDog/dd-trace-py/pull/9348 ## Checklist - [x] Change(s) are motivated and described in the PR description - [x] Testing strategy is described if automated tests are not included in the PR - [x] Risks are described (performance impact, potential for breakage, maintainability) - [x] Change is maintainable (easy to change, telemetry, documentation) - [x] [Library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) are followed or label `changelog/no-changelog` is set - [x] Documentation is included (in-code, generated user docs, [public corp docs](https://github.com/DataDog/documentation/)) - [x] Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) - [x] If this PR changes the public interface, I've notified `@DataDog/apm-tees`. - [x] If change touches code that signs or publishes builds or packages, or handles credentials of any kind, I've requested a review from `@DataDog/security-design-and-guidance`. ## Reviewer Checklist - [x] Title is accurate - [x] All changes are related to the pull request's stated goal - [x] Description motivates each change - [x] Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - [x] Testing strategy adequately addresses listed risks - [x] Change is maintainable (easy to change, telemetry, documentation) - [x] Release note makes sense to a user of the library - [x] Author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - [x] Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- tests/appsec/iast_packages/test_packages.py | 202 ++++++++++++++++---- 1 file changed, 160 insertions(+), 42 deletions(-) diff --git a/tests/appsec/iast_packages/test_packages.py b/tests/appsec/iast_packages/test_packages.py index 73c169d9be8..be0860b5377 100644 --- a/tests/appsec/iast_packages/test_packages.py +++ b/tests/appsec/iast_packages/test_packages.py @@ -12,8 +12,11 @@ from tests.utils import override_env +PYTHON_VERSION = sys.version_info[:2] + + class PackageForTesting: - package_name = "" + name = "" import_name = "" package_version = "" url_to_test = "" @@ -22,6 +25,7 @@ class PackageForTesting: expected_result2 = "" extra_packages = [] test_import = True + test_import_python_versions_to_skip = [] test_e2e = True def __init__( @@ -33,12 +37,14 @@ def __init__( expected_result2, extras=[], test_import=True, + skip_python_version=[], test_e2e=True, import_name=None, ): - self.package_name = name + self.name = name self.package_version = version self.test_import = test_import + self.test_import_python_versions_to_skip = skip_python_version self.test_e2e = test_e2e if expected_param: self.expected_param = expected_param @@ -51,17 +57,31 @@ def __init__( if import_name: self.import_name = import_name else: - self.import_name = self.package_name + self.import_name = self.name @property def url(self): - return f"/{self.package_name}?package_param={self.expected_param}" + return f"/{self.name}?package_param={self.expected_param}" + + def __str__(self): + return f"{self.name}=={self.package_version}: {self.url_to_test}" def __repr__(self): - return f"{self.package_name}: {self.url_to_test}" + return f"{self.name}=={self.package_version}: {self.url_to_test}" + + @property + def skip(self): + for version in self.test_import_python_versions_to_skip: + if version == PYTHON_VERSION: + return True, f"{self.name} not yet compatible with Python {version}" + return False, "" + + def _install(self, package_name, package_version=""): + if package_version: + package_fullversion = package_name + "==" + package_version + else: + package_fullversion = package_name - def _install(self, package_name, package_version): - package_fullversion = package_name + "==" + package_version cmd = ["python", "-m", "pip", "install", package_fullversion] env = {} env.update(os.environ) @@ -69,11 +89,14 @@ def _install(self, package_name, package_version): # doesn't work correctly with riot environment and python packages path proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr, close_fds=True, env=env) proc.wait() - print(proc.stdout) - print(proc.stderr) def install(self): - self._install(self.package_name, self.package_version) + self._install(self.name, self.package_version) + for package_name, package_version in self.extra_packages: + self._install(package_name, package_version) + + def install_latest(self): + self._install(self.name) for package_name, package_version in self.extra_packages: self._install(package_name, package_version) @@ -84,6 +107,7 @@ def install(self): # pypular package is discarded because it is not a real top package # wheel, importlib-metadata and pip is discarded because they are package to build projects +# colorama and awscli are terminal commands PACKAGES = [ PackageForTesting( "charset-normalizer", "3.3.2", "my-bytes-string", "my-bytes-string", "", import_name="charset_normalizer" @@ -98,7 +122,8 @@ def install(self): import_name="googleapiclient", ), PackageForTesting("idna", "3.6", "xn--eckwd4c7c.xn--zckzah", "ドメイン.テスト", "xn--eckwd4c7c.xn--zckzah"), - # PackageForTesting("numpy", "1.24.4", "9 8 7 6 5 4 3", [3, 4, 5, 6, 7, 8, 9], 5), + # Python 3.12 fails in all steps with "import error" when import numpy + PackageForTesting("numpy", "1.24.4", "9 8 7 6 5 4 3", [3, 4, 5, 6, 7, 8, 9], 5, skip_python_version=[(3, 12)]), PackageForTesting( "python-dateutil", "2.8.2", @@ -131,7 +156,19 @@ def install(self): PackageForTesting("cryptography", "42.0.7", "", "", "", test_e2e=False), PackageForTesting("fsspec", "2024.5.0", "", "", "", test_e2e=False, test_import=False), PackageForTesting("boto3", "1.34.110", "", "", "", test_e2e=False, test_import=False), - # PackageForTesting("typing-extensions", "4.11.0", "", "", "", import_name="typing_extensions", test_e2e=False), + # Python 3.8 fails in test_packages_patched_import with + # TypeError: '>' not supported between instances of 'int' and 'object' + # TODO: try to fix it + PackageForTesting( + "typing-extensions", + "4.11.0", + "", + "", + "", + import_name="typing_extensions", + test_e2e=False, + skip_python_version=[(3, 8)], + ), PackageForTesting("botocore", "1.34.110", "", "", "", test_e2e=False), PackageForTesting("packaging", "24.0", "", "", "", test_e2e=False), PackageForTesting("cffi", "1.16.0", "", "", "", test_e2e=False), @@ -142,85 +179,108 @@ def install(self): PackageForTesting("google-api-core", "2.19.0", "", "", "", test_e2e=False, import_name="google"), PackageForTesting("cffi", "1.16.0", "", "", "", test_e2e=False), PackageForTesting("pycparser", "2.22", "", "", "", test_e2e=False), - # PackageForTesting("grpcio-status", "1.64.0", "", "", "", test_e2e=False), - # PackageForTesting("pandas", "2.2.2", "", "", "", test_e2e=False), + # Pandas dropped Python 3.8 support in pandas>2.0.3 + PackageForTesting("pandas", "2.2.2", "", "", "", test_e2e=False, skip_python_version=[(3, 8)]), PackageForTesting("zipp", "3.18.2", "", "", "", test_e2e=False), PackageForTesting("attrs", "23.2.0", "", "", "", test_e2e=False), PackageForTesting("pyasn1", "0.6.0", "", "", "", test_e2e=False), PackageForTesting("rsa", "4.9", "", "", "", test_e2e=False), + # protobuf fails for all python versions with No module named 'protobuf # PackageForTesting("protobuf", "5.26.1", "", "", "", test_e2e=False), PackageForTesting("jmespath", "1.0.1", "", "", "", test_e2e=False), PackageForTesting("click", "8.1.7", "", "", "", test_e2e=False), PackageForTesting("pydantic", "2.7.1", "", "", "", test_e2e=False), PackageForTesting("pytz", "2024.1", "", "", "", test_e2e=False), - # PackageForTesting("colorama", "0.4.6", "", "", "", test_e2e=False), - # PackageForTesting("awscli", "1.32.110", "", "", "", test_e2e=False), PackageForTesting("markupsafe", "2.1.5", "", "", "", test_e2e=False), PackageForTesting("jinja2", "3.1.4", "", "", "", test_e2e=False), PackageForTesting("platformdirs", "4.2.2", "", "", "", test_e2e=False), - # PackageForTesting("pyjwt", "2.8.0", "", "", "", test_e2e=False, import_name="jwt"), + PackageForTesting("pyjwt", "2.8.0", "", "", "", test_e2e=False, import_name="jwt"), PackageForTesting("tomli", "2.0.1", "", "", "", test_e2e=False), - # PackageForTesting("googleapis-common-protos", "1.63.0", "", "", "", test_e2e=False), PackageForTesting("filelock", "3.14.0", "", "", "", test_e2e=False), - # PackageForTesting("google-auth", "2.29.0", "", "", "", test_e2e=False), PackageForTesting("wrapt", "1.16.0", "", "", "", test_e2e=False), PackageForTesting("cachetools", "5.3.3", "", "", "", test_e2e=False), PackageForTesting("pluggy", "1.5.0", "", "", "", test_e2e=False), PackageForTesting("virtualenv", "20.26.2", "", "", "", test_e2e=False), - # PackageForTesting("docutils", "0.21.2", "", "", "", test_e2e=False), - # PackageForTesting("pyarrow", "16.1.0", "", "", "", test_e2e=False), + # docutils dropped Python 3.8 support in pandas> 1.10.10.21.2 + PackageForTesting("docutils", "0.21.2", "", "", "", test_e2e=False, skip_python_version=[(3, 8)]), + PackageForTesting("pyarrow", "16.1.0", "", "", "", test_e2e=False), PackageForTesting("exceptiongroup", "1.2.1", "", "", "", test_e2e=False), - # PackageForTesting("jsonschema", "4.22.0", "", "", "", test_e2e=False), + # jsonschema fails for Python 3.8 + # except KeyError: + # > raise exceptions.NoSuchResource(ref=uri) from None + # E referencing.exceptions.NoSuchResource: 'http://json-schema.org/draft-03/schema#' + PackageForTesting("jsonschema", "4.22.0", "", "", "", test_e2e=False, skip_python_version=[(3, 8)]), PackageForTesting("requests-oauthlib", "2.0.0", "", "", "", test_e2e=False, import_name="requests_oauthlib"), PackageForTesting("pyparsing", "3.1.2", "", "", "", test_e2e=False), PackageForTesting("pytest", "8.2.1", "", "", "", test_e2e=False), PackageForTesting("oauthlib", "3.2.2", "", "", "", test_e2e=False), PackageForTesting("sqlalchemy", "2.0.30", "", "", "", test_e2e=False), - # PackageForTesting("pyasn1-modules", "0.4.0", "", "", "", test_e2e=False), PackageForTesting("aiohttp", "3.9.5", "", "", "", test_e2e=False), - # PackageForTesting("scipy", "1.13.0", "", "", "", test_e2e=False, import_name="scipy.special"), - # PackageForTesting("isodate", "0.6.1", "", "", "", test_e2e=False), + # scipy dropped Python 3.8 support in pandas> 1.10.1 + PackageForTesting( + "scipy", "1.13.0", "", "", "", test_e2e=False, import_name="scipy.special", skip_python_version=[(3, 8)] + ), + PackageForTesting("isodate", "0.6.1", "", "", "", test_e2e=False), PackageForTesting("multidict", "6.0.5", "", "", "", test_e2e=False), PackageForTesting("iniconfig", "2.0.0", "", "", "", test_e2e=False), PackageForTesting("psutil", "5.9.8", "", "", "", test_e2e=False), PackageForTesting("soupsieve", "2.5", "", "", "", test_e2e=False), PackageForTesting("yarl", "1.9.4", "", "", "", test_e2e=False), - # PackageForTesting("async-timeout", "4.0.3", "", "", "", test_e2e=False), PackageForTesting("frozenlist", "1.4.1", "", "", "", test_e2e=False), PackageForTesting("aiosignal", "1.3.1", "", "", "", test_e2e=False), PackageForTesting("werkzeug", "3.0.3", "", "", "", test_e2e=False), - # PackageForTesting("pillow", "10.3.0", "", "", "", test_e2e=False, import_name="PIL.Image"), + PackageForTesting("pillow", "10.3.0", "", "", "", test_e2e=False, import_name="PIL.Image"), PackageForTesting("tqdm", "4.66.4", "", "", "", test_e2e=False), PackageForTesting("pygments", "2.18.0", "", "", "", test_e2e=False), - # PackageForTesting("grpcio", "1.64.0", "", "", "", test_e2e=False), + PackageForTesting("grpcio", "1.64.0", "", "", "", test_e2e=False, import_name="grpc"), PackageForTesting("greenlet", "3.0.3", "", "", "", test_e2e=False), PackageForTesting("pyopenssl", "24.1.0", "", "", "", test_e2e=False, import_name="OpenSSL.SSL"), PackageForTesting("flask", "3.0.3", "", "", "", test_e2e=False), PackageForTesting("decorator", "5.1.1", "", "", "", test_e2e=False), PackageForTesting("pydantic-core", "2.18.2", "", "", "", test_e2e=False, import_name="pydantic_core"), - # PackageForTesting("lxml", "5.2.2", "", "", "", test_e2e=False, import_name="lxml.etree"), + PackageForTesting("lxml", "5.2.2", "", "", "", test_e2e=False, import_name="lxml.etree"), PackageForTesting("requests-toolbelt", "1.0.0", "", "", "", test_e2e=False, import_name="requests_toolbelt"), - # PackageForTesting("openpyxl", "3.1.2", "", "", "", test_e2e=False, import_name="openpyxl.Workbook"), + PackageForTesting("openpyxl", "3.1.2", "", "", "", test_e2e=False), PackageForTesting("tzdata", "2024.1", "", "", "", test_e2e=False), - # PackageForTesting("et-xmlfile", "1.1.0", "", "", "", test_e2e=False), - # PackageForTesting("importlib-resources", "6.4.0", "", "", "", test_e2e=False, import_name="importlib_resources"), - # PackageForTesting("proto-plus", "1.23.0", "", "", "", test_e2e=False), - # PackageForTesting("asn1crypto", "1.5.1", "", "", "", test_e2e=False), + PackageForTesting( + "importlib-resources", + "6.4.0", + "", + "", + "", + test_e2e=False, + import_name="importlib_resources", + skip_python_version=[(3, 8)], + ), + PackageForTesting("asn1crypto", "1.5.1", "", "", "", test_e2e=False), PackageForTesting("coverage", "7.5.1", "", "", "", test_e2e=False), - # PackageForTesting("azure-core", "1.30.1", "", "", "", test_e2e=False, import_name="azure"), + PackageForTesting("azure-core", "1.30.1", "", "", "", test_e2e=False, import_name="azure"), PackageForTesting("distlib", "0.3.8", "", "", "", test_e2e=False), PackageForTesting("tomlkit", "0.12.5", "", "", "", test_e2e=False), - # PackageForTesting("pynacl", "1.5.0", "", "", "", test_e2e=False), + PackageForTesting("pynacl", "1.5.0", "", "", "", test_e2e=False, import_name="nacl.utils"), PackageForTesting("itsdangerous", "2.2.0", "", "", "", test_e2e=False), - # PackageForTesting("annotated-types", "0.7.0", "", "", "", test_e2e=False), + PackageForTesting("annotated-types", "0.7.0", "", "", "", test_e2e=False, import_name="annotated_types"), PackageForTesting("sniffio", "1.3.1", "", "", "", test_e2e=False), PackageForTesting("more-itertools", "10.2.0", "", "", "", test_e2e=False, import_name="more_itertools"), - # PackageForTesting("google-cloud-storage", "2.16.0", "", "", "", test_e2e=False), ] -@pytest.mark.parametrize("package", [package for package in PACKAGES if package.test_e2e]) +# Use this function if you want to test one or a filter number of package for debug proposes +# SKIP_FUNCTION = lambda package: package.name == "pynacl" # noqa: E731 +SKIP_FUNCTION = lambda package: True # noqa: E731 + + +@pytest.mark.parametrize( + "package", + [package for package in PACKAGES if package.test_e2e and SKIP_FUNCTION(package)], + ids=lambda package: package.name, +) def test_packages_not_patched(package): + should_skip, reason = package.skip + if should_skip: + pytest.skip(reason) + return + package.install() with flask_server( iast_enabled="false", tracer_enabled="true", remote_configuration_enabled="false", token=None @@ -237,8 +297,17 @@ def test_packages_not_patched(package): assert content["params_are_tainted"] is False -@pytest.mark.parametrize("package", [package for package in PACKAGES if package.test_e2e]) +@pytest.mark.parametrize( + "package", + [package for package in PACKAGES if package.test_e2e and SKIP_FUNCTION(package)], + ids=lambda package: package.name, +) def test_packages_patched(package): + should_skip, reason = package.skip + if should_skip: + pytest.skip(reason) + return + package.install() with flask_server(iast_enabled="true", remote_configuration_enabled="false", token=None) as context: _, client, pid = context @@ -253,14 +322,63 @@ def test_packages_patched(package): assert content["params_are_tainted"] is True -@pytest.mark.parametrize("package", [package for package in PACKAGES if package.test_import]) +@pytest.mark.parametrize( + "package", + [package for package in PACKAGES if package.test_import and SKIP_FUNCTION(package)], + ids=lambda package: package.name, +) def test_packages_not_patched_import(package): + should_skip, reason = package.skip + if should_skip: + pytest.skip(reason) + return + package.install() importlib.import_module(package.import_name) -@pytest.mark.parametrize("package", [package for package in PACKAGES if package.test_import]) +@pytest.mark.parametrize( + "package", + [package for package in PACKAGES if package.test_import and SKIP_FUNCTION(package)], + ids=lambda package: package.name, +) def test_packages_patched_import(package): + should_skip, reason = package.skip + if should_skip: + pytest.skip(reason) + return + with override_env({IAST_ENV: "true"}): package.install() assert _iast_patched_module(package.import_name, fromlist=[]) + + +@pytest.mark.parametrize( + "package", + [package for package in PACKAGES if package.test_import and SKIP_FUNCTION(package)], + ids=lambda package: package.name, +) +def test_packages_latest_not_patched_import(package): + should_skip, reason = package.skip + if should_skip: + pytest.skip(reason) + return + + package.install_latest() + importlib.import_module(package.import_name) + + +@pytest.mark.parametrize( + "package", + [package for package in PACKAGES if package.test_import and SKIP_FUNCTION(package)], + ids=lambda package: package.name, +) +def test_packages_latest_patched_import(package): + should_skip, reason = package.skip + if should_skip: + pytest.skip(reason) + return + + with override_env({IAST_ENV: "true"}): + package.install_latest() + assert _iast_patched_module(package.import_name, fromlist=[])