diff --git a/news/5398.feature b/news/5398.feature new file mode 100644 index 00000000000..a3f92b92769 --- /dev/null +++ b/news/5398.feature @@ -0,0 +1 @@ +Add machine readable --json to pip download and add --log-stderr diff --git a/src/pip/_internal/basecommand.py b/src/pip/_internal/basecommand.py index 5f8f6422b0a..7165f019dbe 100644 --- a/src/pip/_internal/basecommand.py +++ b/src/pip/_internal/basecommand.py @@ -135,7 +135,9 @@ def main(self, args): logger_class = "pip._internal.utils.logging.ColorizedStreamHandler" handler_class = "pip._internal.utils.logging.BetterRotatingFileHandler" - logging.config.dictConfig({ + stdout, stderr = self.log_streams + + config = { "version": 1, "disable_existing_loggers": False, "filters": { @@ -155,7 +157,7 @@ def main(self, args): "level": level, "class": logger_class, "no_color": options.no_color, - "stream": self.log_streams[0], + "stream": stderr if options.log_stderr else stdout, "filters": ["exclude_warnings"], "formatter": "indent", }, @@ -163,7 +165,7 @@ def main(self, args): "level": "WARNING", "class": logger_class, "no_color": options.no_color, - "stream": self.log_streams[1], + "stream": stderr, "formatter": "indent", }, "user_log": { @@ -173,6 +175,13 @@ def main(self, args): "delay": True, "formatter": "indent", }, + "structured_output": { + "level": "DEBUG", + "class": logger_class, + "no_color": True, + "stream": stdout, + "formatter": "indent", + }, }, "root": { "level": root_level, @@ -194,7 +203,11 @@ def main(self, args): "pip._vendor", "distlib", "requests", "urllib3" ] }, - }) + } + config["loggers"]["pip.__structured_output"] = { + "handlers": ["structured_output"], + } + logging.config.dictConfig(config) # TODO: try to get these passing down from the command? # without resorting to os.environ to hold these. diff --git a/src/pip/_internal/cmdoptions.py b/src/pip/_internal/cmdoptions.py index eb113457f30..c4ee36ce89d 100644 --- a/src/pip/_internal/cmdoptions.py +++ b/src/pip/_internal/cmdoptions.py @@ -274,6 +274,15 @@ def extra_index_url(): help='Ignore package index (only looking at --find-links URLs instead).', ) # type: Any +log_stderr = partial( + Option, + '--log-stderr', + dest='log_stderr', + action='store_true', + default=False, + help="Log logger warnings to stderr", +) + def find_links(): return Option( @@ -604,6 +613,7 @@ def _merge_hash(option, opt_str, value, parser): no_cache, disable_pip_version_check, no_color, + log_stderr, ] } diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 66bcbd5c822..0bdc1157ef0 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import json import logging import os @@ -115,6 +116,15 @@ def __init__(self, *args, **kw): "this option."), ) + cmd_opts.add_option( + '--json', + dest='json', + action='store_true', + default=False, + help=("Output information about downloaded packages as json. " + "See documentation for caveats."), + ) + index_opts = cmdoptions.make_option_group( cmdoptions.index_group, self.parser, @@ -227,8 +237,34 @@ def run(self, options, args): if downloaded: logger.info('Successfully downloaded %s', downloaded) + if options.json: + details = self._get_download_details( + resolver, requirement_set, options.download_dir) + logging.getLogger('pip.__structured_output').info( + json.dumps(details)) + # Clean up if not options.no_clean: requirement_set.cleanup_files() return requirement_set + + def _get_download_details(self, resolver, requirement_set, download_dir): + downloaded = [] + download_dir = os.path.abspath(download_dir) + for req in requirement_set.successfully_downloaded: + deps = resolver.get_dependencies().get(req.name, []) + download_path = os.path.join(download_dir, req.link.filename) + downloaded.append( + { + 'name': req.name, + 'download_path': download_path, + 'url': req.link.url, + 'version': req.version, + 'dependencies': [ + {'name': dep.name, 'version': dep.version} + for dep in deps + ], + } + ) + return downloaded diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index 17aee7246c4..7ce35ca47e1 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -486,7 +486,7 @@ def find_requirement(self, req, upgrade): """Try to find a Link matching req Expects req, an InstallRequirement and upgrade, a boolean - Returns a Link if found, + Returns an InstallationCandidate if found, Raises DistributionNotFound or BestVersionAlreadyInstalled otherwise """ all_candidates = self.find_all_candidates(req.name) @@ -579,7 +579,7 @@ def find_requirement(self, req, upgrade): best_candidate.version, ', '.join(sorted(compatible_versions, key=parse_version)) ) - return best_candidate.location + return best_candidate def _get_pages(self, locations, project_name): """ diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index f3515e40ce2..11e09a4b829 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -27,7 +27,9 @@ from pip._internal.download import ( is_archive_file, is_url, path_to_url, url_to_path, ) -from pip._internal.exceptions import InstallationError, UninstallationError +from pip._internal.exceptions import ( + InstallationError, InvalidWheelFilename, UninstallationError, +) from pip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) @@ -304,7 +306,9 @@ def populate_link(self, finder, upgrade, require_hashes): to file modification times. """ if self.link is None: - self.link = finder.find_requirement(self, upgrade) + candidate = finder.find_requirement(self, upgrade) + self.link = candidate.location + self._found_version = candidate.version if self._wheel_cache is not None and not require_hashes: old_link = self.link self.link = self._wheel_cache.get(self.link, self.name) @@ -325,6 +329,27 @@ def is_pinned(self): return (len(specifiers) == 1 and next(iter(specifiers)).operator in {'==', '==='}) + _found_version = None + + @property + def version(self): + """ The version if available, else None + + The version determined during requirement resolution if available, + the wheel version from the filename if available, or None if no + version information could be obtained + """ + if self._found_version: + return str(self._found_version) + # If we didn't lookup the version from the internet/a finder, try to + # guess it + if self.is_wheel: + try: + return Wheel(self.link.filename).version + except InvalidWheelFilename: + pass + return None + def from_path(self): if self.req is None: return None diff --git a/src/pip/_internal/resolve.py b/src/pip/_internal/resolve.py index 3200fca8ad7..f58ab6a4399 100644 --- a/src/pip/_internal/resolve.py +++ b/src/pip/_internal/resolve.py @@ -352,3 +352,10 @@ def schedule(req): for install_req in req_set.requirements.values(): schedule(install_req) return order + + def get_dependencies(self): + """ Gets dependencies discovered after resolution + + Returns a mapping of package names to lists of requirement objects + """ + return self._discovered_dependencies diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 4cbfb5665a3..bde298e1824 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -1,7 +1,10 @@ +import json import os +import sys import textwrap import pytest +from pip._vendor.six.moves.urllib.parse import urlparse from pip._internal.status_codes import ERROR from tests.lib.path import Path @@ -88,6 +91,82 @@ def test_basic_download_should_download_dependencies(script): assert script.site_packages / 'openid' not in result.files_created +@pytest.mark.network +def test_prints_json(script): + result = script.pip( + 'download', 'flake8==3.5.0', '-d', '.', '--log-stderr', '--json', + expect_stderr=True + ) + + expected = { + 'flake8': { + 'dependencies': [ + 'pyflakes', + 'enum34', + 'configparser', + 'pycodestyle', + 'mccabe', + ], + 'filename': 'flake8-{version}-py2.py3-none-any.whl' + }, + 'pyflakes': { + 'dependencies': [], + 'filename': 'pyflakes-{version}-py2.py3-none-any.whl' + }, + 'pycodestyle': { + 'dependencies': [], + 'filename': 'pycodestyle-{version}-py2.py3-none-any.whl' + }, + 'mccabe': { + 'dependencies': [], + 'filename': 'mccabe-{version}-py2.py3-none-any.whl' + }, + 'configparser': { + 'dependencies': [], + 'filename': 'configparser-{version}.tar.gz', + }, + 'enum34': { + 'dependencies': [], + 'filename': 'enum34-{version}-py{py_version}-none-any.whl', + }, + } + expected_keys = ['dependencies', 'download_path', 'name', 'url', 'version'] + + actual = json.loads(result.stdout) + transformed = {package['name']: package for package in actual} + + assert 'flake8' in transformed + assert 'pyflakes' in transformed + + for package in actual: + assert sorted(package.keys()) == expected_keys + + expected_package = expected[package['name']] + version = package['version'] + url = urlparse(package['url']) + filename = expected_package['filename'].format( + version=version, py_version=sys.version_info[0]) + + created = result.files_created[Path('scratch') / filename] + created_path = os.path.join(created.base_path, created.path) + + # Windows likes to spit this path out lowercase. + assert package['download_path'].lower() == created_path.lower() + assert url.scheme == 'https' + assert url.hostname == 'files.pythonhosted.org' + assert Path(url.path).name == filename + if package['dependencies']: + # Dependencies can change between python versions. Just try to get + # /something/ matched + assert any( + (dep['name'] in expected for dep in package['dependencies'])) + for dep in package['dependencies']: + assert sorted(dep.keys()) == ['name', 'version'] + if dep['name'] in expected: + expected_version = transformed[dep['name']]['version'] + assert dep['version'] == expected_version + + def test_download_wheel_archive(script, data): """ It should download a wheel archive path diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 51fccfae1f1..76b3c3f0468 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -21,7 +21,7 @@ def test_no_mpkg(data): """Finder skips zipfiles with "macosx10" in the name.""" finder = PackageFinder([data.find_links], [], session=PipSession()) req = InstallRequirement.from_line("pkgwithmpkg") - found = finder.find_requirement(req, False) + found = finder.find_requirement(req, False).location assert found.url.endswith("pkgwithmpkg-1.0.tar.gz"), found @@ -30,7 +30,7 @@ def test_no_partial_name_match(data): """Finder requires the full project name to match, not just beginning.""" finder = PackageFinder([data.find_links], [], session=PipSession()) req = InstallRequirement.from_line("gmpy") - found = finder.find_requirement(req, False) + found = finder.find_requirement(req, False).location assert found.url.endswith("gmpy-1.15.tar.gz"), found @@ -42,7 +42,7 @@ def test_tilde(): finder = PackageFinder(['~/python-pkgs'], [], session=session) req = InstallRequirement.from_line("gmpy") with pytest.raises(DistributionNotFound): - finder.find_requirement(req, False) + finder.find_requirement(req, False).location def test_duplicates_sort_ok(data): @@ -54,7 +54,7 @@ def test_duplicates_sort_ok(data): session=PipSession(), ) req = InstallRequirement.from_line("duplicate") - found = finder.find_requirement(req, False) + found = finder.find_requirement(req, False).location assert found.url.endswith("duplicate-1.0.tar.gz"), found @@ -63,7 +63,7 @@ def test_finder_detects_latest_find_links(data): """Test PackageFinder detects latest using find-links""" req = InstallRequirement.from_line('simple', None) finder = PackageFinder([data.find_links], [], session=PipSession()) - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.endswith("simple-3.0.tar.gz") @@ -71,7 +71,7 @@ def test_incorrect_case_file_index(data): """Test PackageFinder detects latest using wrong case""" req = InstallRequirement.from_line('dinner', None) finder = PackageFinder([], [data.find_links3], session=PipSession()) - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.endswith("Dinner-2.0.tar.gz") @@ -90,7 +90,7 @@ def test_finder_detects_latest_already_satisfied_find_links(data): finder = PackageFinder([data.find_links], [], session=PipSession()) with pytest.raises(BestVersionAlreadyInstalled): - finder.find_requirement(req, True) + finder.find_requirement(req, True).location @pytest.mark.network @@ -112,7 +112,7 @@ def test_finder_detects_latest_already_satisfied_pypi_links(): ) with pytest.raises(BestVersionAlreadyInstalled): - finder.find_requirement(req, True) + finder.find_requirement(req, True).location class TestWheel: @@ -131,7 +131,7 @@ def test_skip_invalid_wheel_link(self, caplog, data): session=PipSession(), ) with pytest.raises(DistributionNotFound): - finder.find_requirement(req, True) + finder.find_requirement(req, True).location assert ( "invalid.whl; invalid wheel filename" @@ -157,7 +157,7 @@ def test_not_find_wheel_not_supported(self, data, monkeypatch): finder.valid_tags = pip._internal.pep425tags.get_supported() with pytest.raises(DistributionNotFound): - finder.find_requirement(req, True) + finder.find_requirement(req, True).location def test_find_wheel_supported(self, data, monkeypatch): """ @@ -175,7 +175,7 @@ def test_find_wheel_supported(self, data, monkeypatch): [], session=PipSession(), ) - found = finder.find_requirement(req, True) + found = finder.find_requirement(req, True).location assert ( found.url.endswith("simple.dist-0.1-py2.py3-none-any.whl") ), found @@ -191,7 +191,7 @@ def test_wheel_over_sdist_priority(self, data): [], session=PipSession(), ) - found = finder.find_requirement(req, True) + found = finder.find_requirement(req, True).location assert found.url.endswith("priority-1.0-py2.py3-none-any.whl"), found def test_existing_over_wheel_priority(self, data): @@ -214,7 +214,7 @@ def test_existing_over_wheel_priority(self, data): ) with pytest.raises(BestVersionAlreadyInstalled): - finder.find_requirement(req, True) + finder.find_requirement(req, True).location def test_link_sorting(self): """ @@ -296,7 +296,7 @@ def test_finder_priority_file_over_page(data): assert all(version.location.scheme == 'https' for version in all_versions[1:]), all_versions - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.startswith("file://") @@ -314,7 +314,7 @@ def test_finder_deplink(): finder.add_dependency_links( ['https://files.pythonhosted.org/packages/source/g/gmpy/gmpy-1.15.zip'] ) - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.startswith("https://files.pythonhosted.org/"), link @@ -338,7 +338,7 @@ def test_finder_priority_page_over_deplink(): assert all_versions[-1].location.url.startswith( 'https://files.pythonhosted.org/' ) - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.startswith( "https://files.pythonhosted.org/packages/3f/08/7347ca4" ), link @@ -356,7 +356,7 @@ def test_finder_priority_nonegg_over_eggfragments(): assert all_versions[0].location.url.endswith('tar.gz') assert all_versions[1].location.url.endswith('#egg=bar-1.0') - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.endswith('tar.gz') @@ -367,7 +367,7 @@ def test_finder_priority_nonegg_over_eggfragments(): all_versions = finder.find_all_candidates(req.name) assert all_versions[0].location.url.endswith('tar.gz') assert all_versions[1].location.url.endswith('#egg=bar-1.0') - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.endswith('tar.gz') @@ -381,7 +381,7 @@ def test_finder_only_installs_stable_releases(data): # using a local index (that has pre & dev releases) finder = PackageFinder([], [data.index_url("pre")], session=PipSession()) - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.endswith("bar-1.0.tar.gz"), link.url # using find-links @@ -389,14 +389,14 @@ def test_finder_only_installs_stable_releases(data): finder = PackageFinder(links, [], session=PipSession()) with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url == "https://foo/bar-1.0.tar.gz" links.reverse() finder = PackageFinder(links, [], session=PipSession()) with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url == "https://foo/bar-1.0.tar.gz" @@ -439,7 +439,7 @@ def test_finder_installs_pre_releases(data): allow_all_prereleases=True, session=PipSession(), ) - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.endswith("bar-2.0b1.tar.gz"), link.url # using find-links @@ -451,7 +451,7 @@ def test_finder_installs_pre_releases(data): ) with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url == "https://foo/bar-2.0b1.tar.gz" links.reverse() @@ -462,7 +462,7 @@ def test_finder_installs_pre_releases(data): ) with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url == "https://foo/bar-2.0b1.tar.gz" @@ -479,7 +479,7 @@ def test_finder_installs_dev_releases(data): allow_all_prereleases=True, session=PipSession(), ) - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url.endswith("bar-2.0.dev1.tar.gz"), link.url @@ -493,14 +493,14 @@ def test_finder_installs_pre_releases_with_version_spec(): finder = PackageFinder(links, [], session=PipSession()) with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url == "https://foo/bar-2.0b1.tar.gz" links.reverse() finder = PackageFinder(links, [], session=PipSession()) with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False).location assert link.url == "https://foo/bar-2.0b1.tar.gz" diff --git a/tests/unit/test_req_install.py b/tests/unit/test_req_install.py index 17af7d5f526..b7b70f85101 100644 --- a/tests/unit/test_req_install.py +++ b/tests/unit/test_req_install.py @@ -3,9 +3,19 @@ import pytest +from pip._internal.index import InstallationCandidate from pip._internal.req.req_install import InstallRequirement +class MockedFinder(object): + + def __init__(self, ret): + self.ret = ret + + def find_requirement(self, req, upgrade): + return self.ret + + class TestInstallRequirementBuildDirectory(object): # no need to test symlinks on Windows @pytest.mark.skipif("sys.platform == 'win32'") @@ -41,3 +51,33 @@ def test_forward_slash_results_in_a_link(self, tmpdir): ) assert requirement.link is not None + + def test_install_requirement_version_is_correct(self, tmpdir): + install_dir = tmpdir / "foo" / "bar" + + # Just create a file for letting the logic work + setup_py_path = install_dir / "setup.py" + os.makedirs(str(install_dir)) + with open(setup_py_path, 'w') as f: + f.write('') + + requirement1 = InstallRequirement.from_line( + 'https://example.com/urllib3.tar.gz', + ) + requirement2 = InstallRequirement.from_line( + 'https://example.com/urllib3-1.22-py2.py3-none-any.whl', + ) + requirement3 = InstallRequirement.from_line( + 'urllib3==1.22', + ) + mocked_finder = MockedFinder( + InstallationCandidate( + 'urllib3', '1.4', 'https://example.com/urllib3.tar.gz')) + requirement3.populate_link(mocked_finder, False, False) + + assert requirement1.link is not None + assert requirement1.version is None + assert requirement2.link is not None + assert requirement2.version == "1.22" + assert requirement3.link is not None + assert requirement3.version == "1.4"