diff --git a/MANIFEST.in b/MANIFEST.in index 468f5b7e..89c5f63c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,6 +8,7 @@ include .pre-commit-config.yaml recursive-include docs * recursive-include tests *.py recursive-include tests hello-world-* +recursive-include packaging *.py exclude noxfile.py exclude .travis.yml diff --git a/noxfile.py b/noxfile.py index 925215cf..5554853a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -26,7 +26,8 @@ def coverage(*args): session.run("python", "-m", "coverage", *args) session.install("coverage<5.0.0", "pretend", "pytest", "pip>=9.0.2") - + session.install("typing_extensions") + if "pypy" not in session.python: coverage( "run", diff --git a/packaging/metadata/__init__.py b/packaging/metadata/__init__.py new file mode 100644 index 00000000..28bad092 --- /dev/null +++ b/packaging/metadata/__init__.py @@ -0,0 +1,136 @@ +from email.parser import HeaderParser +from email.message import Message +from typing import Dict, Iterator, Union, List, Any +from typing_extensions import TypedDict +import inspect +import json +from .constants import VERSIONED_METADATA_FIELDS +import sys + + +def _json_form(val: str) -> str: + return val.lower().replace("-", "_") + + +def _canonicalize( + metadata: Dict[str, Union[List[str], str]] +) -> Dict[str, Union[List[str], str]]: + """ + Transforms a metadata object to the canonical representation + as specified in + https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata + All transformed keys should be reduced to lower case. Hyphens + should be replaced with underscores, but otherwise should retain all + other characters. + """ + return {_json_form(key): value for key, value in metadata.items()} + + +def check_python_compatability() -> None: + if sys.version_info[0] < 3: + raise ModuleNotFoundError() + + +check_python_compatability() + + +class Metadata: + def __init__(self, **kwargs: Union[List[str], str]) -> None: + self._meta_dict = kwargs + + def __eq__(self, other: object) -> bool: + if isinstance(other, Metadata): + return self._meta_dict == other._meta_dict + return NotImplemented + + @classmethod + def from_json(cls, data: str) -> "Metadata": + return cls(**_canonicalize(json.loads(data))) + + @classmethod + def from_dict(cls, data: Dict[str, Union[List[str], str]]) -> "Metadata": + return cls(**_canonicalize(data)) + + @classmethod + def from_rfc822(cls, rfc822_string: str) -> "Metadata": + return cls(**Metadata._rfc822_string_to_dict(rfc822_string)) + + def to_json(self) -> str: + return json.dumps(self._meta_dict, sort_keys=True) + + def to_dict(self) -> Dict: + return self._meta_dict + + def to_rfc822(self) -> str: + msg = Message() + metadata_version = self._meta_dict["metadata_version"] + metadata_fields = VERSIONED_METADATA_FIELDS[metadata_version] + for field in ( + metadata_fields["SINGLE"] + | metadata_fields["MULTI"] + | metadata_fields["TREAT_AS_MULTI"] + ): + value = self._meta_dict.get(_json_form(field)) + if value: + if field == "Description": + # Special case - put in payload + msg.set_payload(value) + continue + if field == "Keywords": + value = ",".join(value) + if isinstance(value, str): + value = [value] + for item in value: + msg.add_header(field, item) + + return msg.as_string() + + def __iter__(self) -> Iterator[Any]: + return iter(self._meta_dict.items()) + + @classmethod + def _rfc822_string_to_dict( + cls, rfc822_string: str + ) -> Dict[str, Union[List[str], str]]: + """Extracts metadata information from a metadata-version 2.1 object. + + https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata + + - The original key-value format should be read with email.parser.HeaderParser; + - All transformed keys should be reduced to lower case. Hyphens should + be replaced with underscores, but otherwise should retain all other + characters; + - The transformed value for any field marked with "(Multiple-use") + should be a single list containing all the original values for the + given key; + - The Keywords field should be converted to a list by splitting the + original value on whitespace characters; + - The message body, if present, should be set to the value of the + description key. + - The result should be stored as a string-keyed dictionary. + """ + metadata: Dict[str, Union[List[str], str]] = {} + parsed = HeaderParser().parsestr(rfc822_string) + metadata_fields = VERSIONED_METADATA_FIELDS[parsed.get("Metadata-Version")] + + for key, value in parsed.items(): + if key in metadata_fields["MULTI"]: + metadata.setdefault(key, []).append(value) + elif key in metadata_fields["TREAT_AS_MULTI"]: + metadata[key] = [val.strip() for val in value.split(",")] + elif key == "Description": + metadata[key] = inspect.cleandoc(value) + else: + metadata[key] = value + + # Handle the message payload + payload = parsed.get_payload() + if payload: + if "Description" in metadata: + print("Both Description and payload given - ignoring Description") + metadata["Description"] = payload + + return _canonicalize(metadata) + + def validate(self) -> bool: + raise NotImplementedError diff --git a/packaging/metadata/constants.py b/packaging/metadata/constants.py new file mode 100644 index 00000000..6355a765 --- /dev/null +++ b/packaging/metadata/constants.py @@ -0,0 +1,121 @@ +MULTI_1_0 = {"Platform"} # type : typing.Set[str] + +TREAT_AS_MULTI_1_0 = {"Keywords"} # type : typing.Set[str] + +SINGLE_1_0 = { + "Metadata-Version", + "Name", + "Version", + "Summary", + "Description", + "Home-page", + "Author", + "Author-email", + "License", +} # type : typing.Set[str] + + +MULTI_1_1 = {"Platform", "Supported-Platform", "Classifier"} # type : typing.Set[str] + +TREAT_AS_MULTI_1_1 = {"Keywords"} # type : typing.Set[str] + +SINGLE_1_1 = { + "Metadata-Version", + "Name", + "Version", + "Summary", + "Description", + "Home-page", + "Download-URL", + "Author", + "Author-email", + "License", +} # type : typing.Set[str] + + +MULTI_1_2 = { + "Platform", + "Supported-Platform", + "Classifier", + "Requires-Dist", + "Provides-Dist", + "Obsoletes-Dist", + "Requires-External", + "Project-URL", +} # type : typing.Set[str] + +TREAT_AS_MULTI_1_2 = {"Keywords"} # type : typing.Set[str] + +SINGLE_1_2 = { + "Metadata-Version", + "Name", + "Version", + "Summary", + "Description", + "Home-page", + "Download-URL", + "Author", + "Author-email", + "Maintainer", + "Maintainer-email", + "License", + "Requires-Python", +} # type : typing.Set[str] + + +MULTI_2_1 = { + "Platform", + "Supported-Platform", + "Classifier", + "Requires-Dist", + "Provides-Dist", + "Obsoletes-Dist", + "Requires-External", + "Project-URL", + "Provides-Extra", +} # type : typing.Set[str] + +TREAT_AS_MULTI_2_1 = {"Keywords"} # type : typing.Set[str] + +SINGLE_2_1 = { + "Metadata-Version", + "Name", + "Version", + "Summary", + "Description", + "Description-Content-Type", + "Home-page", + "Download-URL", + "Author", + "Author-email", + "Maintainer", + "Maintainer-email", + "License", + "Requires-Python", +} # type : typing.Set[str] + + +VERSIONED_METADATA_FIELDS = { + "1.0": { + "MULTI": MULTI_1_0, + "TREAT_AS_MULTI": TREAT_AS_MULTI_1_0, + "SINGLE": SINGLE_1_0, + }, + "1.1": { + "MULTI": MULTI_1_1, + "TREAT_AS_MULTI": TREAT_AS_MULTI_1_1, + "SINGLE": SINGLE_1_1, + }, + "1.2": { + "MULTI": MULTI_1_2, + "TREAT_AS_MULTI": TREAT_AS_MULTI_1_2, + "SINGLE": SINGLE_1_2, + }, + "2.1": { + "MULTI": MULTI_2_1, + "TREAT_AS_MULTI": TREAT_AS_MULTI_2_1, + "SINGLE": SINGLE_2_1, + }, +} # type : typing.Any + +# typing.Dict[typing.Union[typing.List[str],str], typing.Dict[str, typing.Set[str]]] diff --git a/tests/metadata/2_1_pkginfo_string.txt b/tests/metadata/2_1_pkginfo_string.txt new file mode 100644 index 00000000..55050b55 --- /dev/null +++ b/tests/metadata/2_1_pkginfo_string.txt @@ -0,0 +1,71 @@ +Metadata-Version: 2.1 +Name: sampleproject +Version: 2.0.0 +Summary: A sample Python project +Home-page: https://github.com/pypa/sampleproject +Author: A. Random Developer +Author-email: author@example.com +License: UNKNOWN +Project-URL: Bug Reports, https://github.com/pypa/sampleproject/issues +Project-URL: Funding, https://donate.pypi.org +Project-URL: Say Thanks!, http://saythanks.io/to/example +Project-URL: Source, https://github.com/pypa/sampleproject/ +Description: # A sample Python project + + ![Python Logo](https://www.python.org/static/community_logos/python-logo.png "Sample inline image") + + A sample project that exists as an aid to the [Python Packaging User + Guide][packaging guide]'s [Tutorial on Packaging and Distributing + Projects][distribution tutorial]. + + This project does not aim to cover best practices for Python project + development as a whole. For example, it does not provide guidance or tool + recommendations for version control, documentation, or testing. + + [The source for this project is available here][src]. + + Most of the configuration for a Python project is done in the `setup.py` file, + an example of which is included in this project. You should edit this file + accordingly to adapt this sample project to your needs. + + ---- + + This is the README file for the project. + + The file should use UTF-8 encoding and can be written using + [reStructuredText][rst] or [markdown][md use] with the appropriate [key set][md + use]. It will be used to generate the project webpage on PyPI and will be + displayed as the project homepage on common code-hosting services, and should be + written for that purpose. + + Typical contents for this file would include an overview of the project, basic + usage examples, etc. Generally, including the project changelog in here is not a + good idea, although a simple “What's New” section for the most recent version + may be appropriate. + + [packaging guide]: https://packaging.python.org + [distribution tutorial]: https://packaging.python.org/tutorials/packaging-projects/ + [src]: https://github.com/pypa/sampleproject + [rst]: http://docutils.sourceforge.net/rst.html + [md]: https://tools.ietf.org/html/rfc7764#section-3.5 "CommonMark variant" + [md use]: https://packaging.python.org/specifications/core-metadata/#description-content-type-optional + +Keywords: sample,setuptools,development +Platform: UNKNOWN +Classifier: Development Status :: 3 - Alpha +Classifier: Intended Audience :: Developers +Classifier: Topic :: Software Development :: Build Tools +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3 :: Only +Requires-Python: >=3.5, <4 +Description-Content-Type: text/markdown +Provides-Extra: dev +Provides-Extra: test +Requires-Dist: peppercorn +Requires-Dist: check-manifest ; extra == 'dev' +Requires-Dist: coverage ; extra == 'test' \ No newline at end of file diff --git a/tests/metadata/__init__.py b/tests/metadata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/metadata/test_metadata.py b/tests/metadata/test_metadata.py new file mode 100644 index 00000000..2cb739ad --- /dev/null +++ b/tests/metadata/test_metadata.py @@ -0,0 +1,223 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function +from packaging.metadata import Metadata, check_python_compatability +from .test_metadata_constants import ( + VALID_PACKAGE_2_1_RFC822, + VALID_PACKAGE_2_1_JSON, + VALID_PACKAGE_2_1_DICT, + VALID_PACKAGE_1_0_RFC822, + VALID_PACKAGE_1_0_DICT, + VALID_PACKAGE_1_0_JSON, + VALID_PACKAGE_1_1_RFC822, + VALID_PACKAGE_1_1_DICT, + VALID_PACKAGE_1_1_JSON, + VALID_PACKAGE_1_2_RFC822, + VALID_PACKAGE_1_2_DICT, + VALID_PACKAGE_1_2_JSON, + VALID_PACKAGE_1_0_REPEATED_DESC, + VALID_PACKAGE_1_0_SINGLE_LINE_DESC, +) + +import pytest +import sys + + +class TestMetaData: + def test_kwargs_init(self): + metadata = Metadata( + name="foo", + version="1.0", + keywords=["a", "b", "c"], + description="Hello\nworld", + ) + assert metadata._meta_dict == { + "name": "foo", + "version": "1.0", + "keywords": ["a", "b", "c"], + "description": "Hello\nworld", + } + + @pytest.mark.parametrize( + ("metadata_dict", "metadata_json"), + [ + (VALID_PACKAGE_2_1_DICT, VALID_PACKAGE_2_1_JSON), + (VALID_PACKAGE_1_0_DICT, VALID_PACKAGE_1_0_JSON), + (VALID_PACKAGE_1_1_DICT, VALID_PACKAGE_1_1_JSON), + (VALID_PACKAGE_1_2_DICT, VALID_PACKAGE_1_2_JSON), + ], + ) + def test_from_json(self, metadata_dict, metadata_json): + metadata_1 = Metadata(**metadata_dict) + metadata_2 = Metadata.from_json(metadata_json) + + assert metadata_1 == metadata_2 + + @pytest.mark.parametrize( + ("metadata_dict", "metadata_rfc822"), + [ + (VALID_PACKAGE_2_1_DICT, VALID_PACKAGE_2_1_RFC822), + (VALID_PACKAGE_1_0_DICT, VALID_PACKAGE_1_0_RFC822), + (VALID_PACKAGE_1_1_DICT, VALID_PACKAGE_1_1_RFC822), + (VALID_PACKAGE_1_2_DICT, VALID_PACKAGE_1_2_RFC822), + ], + ) + def test_from_rfc822(self, metadata_dict, metadata_rfc822): + metadata_1 = Metadata(**metadata_dict) + metadata_2 = Metadata.from_rfc822(metadata_rfc822) + + assert metadata_1 == metadata_2 + + @pytest.mark.parametrize( + ("metadata_dict", "metadata_json"), + [ + (VALID_PACKAGE_2_1_DICT, VALID_PACKAGE_2_1_JSON), + (VALID_PACKAGE_1_0_DICT, VALID_PACKAGE_1_0_JSON), + (VALID_PACKAGE_1_1_DICT, VALID_PACKAGE_1_1_JSON), + (VALID_PACKAGE_1_2_DICT, VALID_PACKAGE_1_2_JSON), + ], + ) + def test_from_dict(self, metadata_dict, metadata_json): + metadata_1 = Metadata.from_dict(metadata_dict) + metadata_2 = Metadata.from_json(metadata_json) + + assert metadata_1 == metadata_2 + + @pytest.mark.parametrize( + ("expected_json_string", "input_dict"), + [ + (VALID_PACKAGE_1_2_JSON, VALID_PACKAGE_1_2_DICT), + (VALID_PACKAGE_1_0_JSON, VALID_PACKAGE_1_0_DICT), + (VALID_PACKAGE_1_1_JSON, VALID_PACKAGE_1_1_DICT), + (VALID_PACKAGE_2_1_JSON, VALID_PACKAGE_2_1_DICT), + ], + ) + def test_to_json(self, expected_json_string, input_dict): + metadata_1 = Metadata(**input_dict) + generated_json_string = metadata_1.to_json() + + assert expected_json_string == generated_json_string + + @pytest.mark.parametrize( + ("expected_rfc822_string", "input_dict"), + [ + (VALID_PACKAGE_2_1_RFC822, VALID_PACKAGE_2_1_DICT), + (VALID_PACKAGE_1_0_RFC822, VALID_PACKAGE_1_0_DICT), + (VALID_PACKAGE_1_1_RFC822, VALID_PACKAGE_1_1_DICT), + (VALID_PACKAGE_1_2_RFC822, VALID_PACKAGE_1_2_DICT), + ], + ) + def test_to_rfc822(self, expected_rfc822_string, input_dict): + metadata_1 = Metadata(**input_dict) + generated_rfc822_string = metadata_1.to_rfc822() + + assert ( + Metadata.from_rfc822(generated_rfc822_string).to_dict() + == Metadata.from_rfc822(expected_rfc822_string).to_dict() + ) + assert TestMetaData._compare_rfc822_strings( + expected_rfc822_string, generated_rfc822_string + ) + + @pytest.mark.parametrize( + "expected_dict", + [ + VALID_PACKAGE_1_2_DICT, + VALID_PACKAGE_1_0_DICT, + VALID_PACKAGE_1_1_DICT, + VALID_PACKAGE_2_1_DICT, + ], + ) + def test_to_dict(self, expected_dict): + metadata_1 = Metadata(**expected_dict) + generated_dict = metadata_1.to_dict() + + assert expected_dict == generated_dict + + def test_metadata_iter(self): + metadata_1 = Metadata( + name="foo", + version="1.0", + keywords=["a", "b", "c"], + description="Hello\nworld", + ) + + for key, value in metadata_1.__iter__(): + assert key in metadata_1._meta_dict + assert metadata_1._meta_dict[key] == value + + def test_repeated_description_in_rfc822(self): + metadata_1 = Metadata.from_rfc822(VALID_PACKAGE_1_0_REPEATED_DESC) + expected_description = ( + "# This is the long description\n\n" + + "This will overwrite the Description field\n" + ) + + assert metadata_1._meta_dict["description"] == expected_description + + def test_single_line_description_in_rfc822(self): + metdata_1 = Metadata.from_rfc822(VALID_PACKAGE_1_0_SINGLE_LINE_DESC) + + description = metdata_1._meta_dict["description"] + + assert len(description.splitlines()) == 1 + + def test_metadata_validation(self): + # Validation not currently implemented + with pytest.raises(NotImplementedError): + metadata = Metadata( + name="foo", + version="1.0", + keywords=["a", "b", "c"], + description="Hello\nworld", + ) + metadata.validate() + + def test_metadata_equals_different_order(self): + metadata_1 = Metadata( + name="foo", + version="1.0", + keywords=["a", "b", "c"], + description="Hello\nworld", + ) + metadata_2 = Metadata( + version="1.0", + keywords=["a", "b", "c"], + description="Hello\nworld", + name="foo", + ) + + assert metadata_1 == metadata_2 + + def test_metadata_equals_non_metadata(self): + metadata_1 = Metadata( + name="foo", + version="1.0", + keywords=["a", "b", "c"], + description="Hello\nworld", + ) + assert ( + metadata_1.__eq__( + { + "name": "foo", + "version": "1.0", + "keywords": ["a", "b", "c"], + "description": "Hello\nworld", + } + ) + == NotImplemented + ) + + def test_raise_when_python2(self, monkeypatch): + with pytest.raises(ModuleNotFoundError): + monkeypatch.setattr(sys, "version_info", (2, 0)) + check_python_compatability() + + @classmethod + def _compare_rfc822_strings(cls, rfc822_1, rfc822_2): + + rfc822_1_dict = Metadata.from_rfc822(rfc822_1).to_dict() + rfc822_2_dict = Metadata.from_rfc822(rfc822_2).to_dict() + + return rfc822_1_dict == rfc822_2_dict diff --git a/tests/metadata/test_metadata_constants.py b/tests/metadata/test_metadata_constants.py new file mode 100644 index 00000000..08a04048 --- /dev/null +++ b/tests/metadata/test_metadata_constants.py @@ -0,0 +1,101 @@ +# -*- coding: UTF-8 -*- + +from packaging.metadata import Metadata +import json +import os + + +VALID_PACKAGE_2_1_RFC822 = open( + os.path.join(os.path.dirname(__file__), "2_1_pkginfo_string.txt") +).read() + +VALID_PACKAGE_2_1_DICT = Metadata._rfc822_string_to_dict(VALID_PACKAGE_2_1_RFC822) + +VALID_PACKAGE_2_1_JSON = json.dumps(VALID_PACKAGE_2_1_DICT, sort_keys=True) + + +VALID_PACKAGE_1_0_RFC822 = """Metadata-Version: 1.0 +Name: sampleproject +Version: 2.0.0 +Summary: A sample Python project +Home-page: https://github.com/pypa/sampleproject +Author: A. Random Developer +Author-email: author@example.com +License: UNKNOWN +Description: # A sample Python project + A longer description +Keywords: sample,setuptools,development +Platform: UNKNOWN +""" + +VALID_PACKAGE_1_0_REPEATED_DESC = """Metadata-Version: 1.0 +Name: sampleproject +Version: 2.0.0 +Summary: A sample Python project +Home-page: https://github.com/pypa/sampleproject +Author: A. Random Developer +Author-email: author@example.com +License: UNKNOWN +Description: # A sample Python project + A longer description +Keywords: sample,setuptools,development +Platform: UNKNOWN + +# This is the long description + +This will overwrite the Description field +""" +VALID_PACKAGE_1_0_SINGLE_LINE_DESC = """Metadata-Version: 1.0 +Name: sampleproject +Version: 2.0.0 +Summary: A sample Python project +Home-page: https://github.com/pypa/sampleproject +Author: A. Random Developer +Author-email: author@example.com +License: UNKNOWN +Description: # A sample Python project +Keywords: sample,setuptools,development +Platform: UNKNOWN +""" + +VALID_PACKAGE_1_0_DICT = Metadata._rfc822_string_to_dict(VALID_PACKAGE_1_0_RFC822) +VALID_PACKAGE_1_0_JSON = json.dumps(VALID_PACKAGE_1_0_DICT, sort_keys=True) + + +VALID_PACKAGE_1_2_RFC822 = """Metadata-Version: 1.2 +Name: sampleproject +Version: 2.0.0 +Summary: A sample Python project +Home-page: https://github.com/pypa/sampleproject +Author: A. Random Developer +Author-email: author@example.com +License: UNKNOWN +Description: # A sample Python project + A longer description +Keywords: sample,setuptools,development +Platform: UNKNOWN +Requires-Python: >=3.5, <4 +""" + +VALID_PACKAGE_1_2_DICT = Metadata._rfc822_string_to_dict(VALID_PACKAGE_1_2_RFC822) +VALID_PACKAGE_1_2_JSON = json.dumps(VALID_PACKAGE_1_2_DICT, sort_keys=True) + +VALID_PACKAGE_1_1_RFC822 = """Metadata-Version: 1.1 +Name: sampleproject +Version: 2.0.0 +Summary: A sample Python project +Home-page: https://github.com/pypa/sampleproject +Author: A. Random Developer +Author-email: author@example.com +License: UNKNOWN +Description: # A sample Python project + A longer description +Keywords: sample,setuptools,development +Platform: UNKNOWN +Classifier: Development Status :: 3 - Alpha +Classifier: Intended Audience :: Developers +Classifier: Topic :: Software Development :: Build Tools +""" + +VALID_PACKAGE_1_1_DICT = Metadata._rfc822_string_to_dict(VALID_PACKAGE_1_1_RFC822) +VALID_PACKAGE_1_1_JSON = json.dumps(VALID_PACKAGE_1_1_DICT, sort_keys=True)