Merge pull request #134 from gmathiou4/Relative-links-confuse-PyPi

relative links_fix
aboutcode-org · Aug 3, 2023 · e5043a8 · e5043a8
2 parents 1831e01 + b74a7c3
commit e5043a8
Show file tree

Hide file tree

Showing 18 changed files with 1,413 additions and 1,269 deletions.
diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py
@@ -20,6 +20,8 @@
 from typing import List
 from typing import NamedTuple
 from urllib.parse import quote_plus
+from urllib.parse import urlparse
+from urllib.parse import urlunparse
 
 import attr
 import packageurl
@@ -949,7 +951,6 @@ def get_sdist_name_ver_ext(filename):
 
 @attr.attributes
 class Sdist(Distribution):
-
     extension = attr.ib(
         repr=False,
         type=str,
@@ -1595,11 +1596,37 @@ def fetch_links(
             url, _, _sha256 = anchor_tag["href"].partition("#sha256=")
             if "data-requires-python" in anchor_tag.attrs:
                 python_requires = anchor_tag.attrs["data-requires-python"]
+            # Resolve relative URL
+            url = resolve_relative_url(package_url, url)
             links.append(Link(url=url, python_requires=python_requires))
         # TODO: keep sha256
         return links
 
 
+def resolve_relative_url(package_url, url):
+    """
+    Return the resolved `url` URLstring given a `package_url` base URL string
+    of a package.
+
+    For example:
+    >>> resolve_relative_url("https://example.com/package", "../path/file.txt")
+    'https://example.com/path/file.txt'
+    """
+    if not url.startswith(("http://", "https://")):
+        base_url_parts = urlparse(package_url)
+        url_parts = urlparse(url)
+        # If the relative URL starts with '..', remove the last directory from the base URL
+        if url_parts.path.startswith(".."):
+            path = base_url_parts.path.rstrip("/").rsplit("/", 1)[0] + url_parts.path[2:]
+        else:
+            path = urlunparse(
+                ("", "", url_parts.path, url_parts.params, url_parts.query, url_parts.fragment)
+            )
+        resolved_url_parts = base_url_parts._replace(path=path)
+        url = urlunparse(resolved_url_parts)
+    return url
+
+
 PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL)
 DEFAULT_PYPI_REPOS = (PYPI_PUBLIC_REPO,)
 DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS}

diff --git a/tests/data/azure-devops.req-310-expected.json b/tests/data/azure-devops.req-310-expected.json
diff --git a/tests/data/azure-devops.req-38-expected.json b/tests/data/azure-devops.req-38-expected.json
diff --git a/tests/data/fetch_links_test.html b/tests/data/fetch_links_test.html
@@ -0,0 +1,4 @@
+<!DOCTYPE html>
+    <html><head><title>Simple Index</title><meta name="api-version" value="2" /></head><body>
+    <a href="../sources.whl#sha256=a4e66406acabbb0a6606c89c5e61bacd2847f5393818b90490021a3929b47807" rel="internal">sources.whl</a><br />
+    </body></html>
diff --git a/tests/data/frozen-requirements.txt-expected.json b/tests/data/frozen-requirements.txt-expected.json
diff --git a/tests/data/insecure-setup-2/setup.py-expected.json b/tests/data/insecure-setup-2/setup.py-expected.json
diff --git a/tests/data/pinned-pdt-requirements.txt-expected.json b/tests/data/pinned-pdt-requirements.txt-expected.json
diff --git a/tests/data/pinned-requirements.txt-expected.json b/tests/data/pinned-requirements.txt-expected.json
diff --git a/tests/data/relative-links-expected.json b/tests/data/relative-links-expected.json
@@ -0,0 +1,6 @@
+[
+  [
+    "https://pypi.org/simple/sources.whl",
+    null
+  ]
+]
diff --git a/tests/data/single-url-except-simple-expected.json b/tests/data/single-url-except-simple-expected.json
diff --git a/tests/data/test-api-expected.json b/tests/data/test-api-expected.json
diff --git a/tests/data/test-api-pdt-expected.json b/tests/data/test-api-pdt-expected.json
diff --git a/tests/data/test-api-with-partial-setup-py.json b/tests/data/test-api-with-partial-setup-py.json
@@ -59,12 +59,12 @@
       "type": "pypi",
       "namespace": null,
       "name": "semver",
-      "version": "3.0.0",
+      "version": "3.0.1",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "Python helper for Semantic Versioning (https://semver.org)\nQuickstart\n==========\n\n.. teaser-begin\n\nA Python module for `semantic versioning`_. Simplifies comparing versions.\n\n|GHAction| |python-support| |downloads| |license| |docs| |black|\n|openissues| |GHDiscussion|\n\n.. teaser-end\n\n.. note::\n\n   This project works for Python 3.7 and greater only. If you are\n   looking for a compatible version for Python 2, use the\n   maintenance branch |MAINT|_.\n\n   The last version of semver which supports Python 2.7 to 3.5 will be\n   2.x.y However, keep in mind, the major 2 release is frozen: no new\n   features nor backports will be integrated.\n\n   We recommend to upgrade your workflow to Python 3 to gain support,\n   bugfixes, and new features.\n\n.. |MAINT| replace:: ``maint/v2``\n.. _MAINT: https://github.com/python-semver/python-semver/tree/maint/v2\n\n\nThe module follows the ``MAJOR.MINOR.PATCH`` style:\n\n* ``MAJOR`` version when you make incompatible API changes,\n* ``MINOR`` version when you add functionality in a backwards compatible manner, and\n* ``PATCH`` version when you make backwards compatible bug fixes.\n\nAdditional labels for pre-release and build metadata are supported.\n\nTo import this library, use:\n\n.. code-block:: python\n\n    >>> import semver\n\nWorking with the library is quite straightforward. To turn a version string into the\ndifferent parts, use the ``semver.Version.parse`` function:\n\n.. code-block:: python\n\n    >>> ver = semver.Version.parse('1.2.3-pre.2+build.4')\n    >>> ver.major\n    1\n    >>> ver.minor\n    2\n    >>> ver.patch\n    3\n    >>> ver.prerelease\n    'pre.2'\n    >>> ver.build\n    'build.4'\n\nTo raise parts of a version, there are a couple of functions available for\nyou. The function ``semver.Version.bump_major`` leaves the original object untouched, but\nreturns a new ``semver.Version`` instance with the raised major part:\n\n.. code-block:: python\n\n    >>> ver = semver.Version.parse(\"3.4.5\")\n    >>> ver.bump_major()\n    Version(major=4, minor=0, patch=0, prerelease=None, build=None)\n\nIt is allowed to concatenate different \"bump functions\":\n\n.. code-block:: python\n\n    >>> ver.bump_major().bump_minor()\n    Version(major=4, minor=1, patch=0, prerelease=None, build=None)\n\nTo compare two versions, semver provides the ``semver.compare`` function.\nThe return value indicates the relationship between the first and second\nversion:\n\n.. code-block:: python\n\n    >>> semver.compare(\"1.0.0\", \"2.0.0\")\n    -1\n    >>> semver.compare(\"2.0.0\", \"1.0.0\")\n    1\n    >>> semver.compare(\"2.0.0\", \"2.0.0\")\n    0\n\n\nThere are other functions to discover. Read on!\n\n\n.. |latest-version| image:: https://img.shields.io/pypi/v/semver.svg\n   :alt: Latest version on PyPI\n   :target: https://pypi.org/project/semver\n.. |python-support| image:: https://img.shields.io/pypi/pyversions/semver.svg\n   :target: https://pypi.org/project/semver\n   :alt: Python versions\n.. |downloads| image:: https://img.shields.io/pypi/dm/semver.svg\n   :alt: Monthly downloads from PyPI\n   :target: https://pypi.org/project/semver\n.. |license| image:: https://img.shields.io/pypi/l/semver.svg\n   :alt: Software license\n   :target: https://github.com/python-semver/python-semver/blob/master/LICENSE.txt\n.. |docs| image:: https://readthedocs.org/projects/python-semver/badge/?version=latest\n   :target: http://python-semver.readthedocs.io/en/latest/?badge=latest\n   :alt: Documentation Status\n.. _semantic versioning: https://semver.org/\n.. |black| image:: https://img.shields.io/badge/code%20style-black-000000.svg\n    :target: https://github.com/psf/black\n    :alt: Black Formatter\n.. |Gitter| image:: https://badges.gitter.im/python-semver/community.svg\n    :target: https://gitter.im/python-semver/community\n    :alt: Gitter\n.. |openissues| image:: http://isitmaintained.com/badge/open/python-semver/python-semver.svg\n    :target: http://isitmaintained.com/project/python-semver/python-semver\n    :alt: Percentage of open issues\n.. |GHAction| image:: https://github.com/python-semver/python-semver/workflows/Python/badge.svg\n    :alt: Python\n.. |GHDiscussion| image:: https://shields.io/badge/GitHub-%20Discussions-green?logo=github\n    :target: https://github.com/python-semver/python-semver/discussions\n    :alt: GitHub Discussion",
-      "release_date": "2023-04-02T13:20:12",
+      "release_date": "2023-06-14T11:43:22",
       "parties": [
         {
           "type": "person",
@@ -96,11 +96,11 @@
         "Topic :: Software Development :: Libraries :: Python Modules"
       ],
       "homepage_url": "https://github.com/python-semver/python-semver",
-      "download_url": "https://files.pythonhosted.org/packages/9f/93/b7389cdd7e573e70cfbeb4b0bbe101af1050a6681342f5d2bc6f1bf2d150/semver-3.0.0.tar.gz",
-      "size": 204359,
+      "download_url": "https://files.pythonhosted.org/packages/46/30/a14b56e500e8eabf8c349edd0583d736b231e652b7dce776e85df11e9e0b/semver-3.0.1.tar.gz",
+      "size": 205762,
       "sha1": null,
-      "md5": "0d36e4e2b2c4366f2b4b5c53b2abe3c0",
-      "sha256": "94df43924c4521ec7d307fc86da1531db6c2c33d9d5cdc3e64cca0eb68569269",
+      "md5": "b7502c12ce325ffffeab694fed52f6f5",
+      "sha256": "9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1",
       "sha512": null,
       "bug_tracking_url": "https://github.com/python-semver/python-semver/issues",
       "code_view_url": null,
@@ -120,14 +120,14 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/semver/3.0.0/json",
+      "api_data_url": "https://pypi.org/pypi/semver/3.0.1/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/[email protected].0"
+      "purl": "pkg:pypi/[email protected].1"
     }
   ],
   "resolution": [
     {
-      "package": "pkg:pypi/[email protected].0",
+      "package": "pkg:pypi/[email protected].1",
       "dependencies": []
     }
   ]