diff --git a/.appveyor.yml b/.appveyor.yml index f07fba762fd..4abddac4c4c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -17,7 +17,7 @@ test_script: - "tools/build.cmd %PYTHON%\\python.exe setup.py test" after_test: - - "tools/build.cmd %PYTHON%\\python.exe setup.py bdist_wheel" + - "tools/build.cmd %PYTHON%\\python.exe setup.py sdist bdist_wheel" artifacts: - path: dist\* diff --git a/CHANGES.rst b/CHANGES.rst index 17ceb3f5280..b45122c6af7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,6 @@ -======= -Changes -======= +========= +Changelog +========= .. You should *NOT* be adding new change log entries to this file, this diff --git a/CHANGES/2408.bugfix b/CHANGES/2408.bugfix new file mode 100644 index 00000000000..4fec0c8b07c --- /dev/null +++ b/CHANGES/2408.bugfix @@ -0,0 +1 @@ +Fix ClientConnectorSSLError and ClientProxyConnectionError for proxy connector diff --git a/CHANGES/2423.bugfix b/CHANGES/2423.bugfix new file mode 100644 index 00000000000..08823f4f69f --- /dev/null +++ b/CHANGES/2423.bugfix @@ -0,0 +1 @@ +Fix connector convert OSError to ClientConnectorError diff --git a/CHANGES/2424.bugfix b/CHANGES/2424.bugfix new file mode 100644 index 00000000000..f06477b975d --- /dev/null +++ b/CHANGES/2424.bugfix @@ -0,0 +1 @@ +Fix connection attempts for multiple dns hosts diff --git a/CHANGES/2451.doc b/CHANGES/2451.doc new file mode 100644 index 00000000000..386917d1926 --- /dev/null +++ b/CHANGES/2451.doc @@ -0,0 +1 @@ +Rename `from_env` to `trust_env` in client reference. diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 21a8d4cb8e7..602f9cfdcd0 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.3.1' +__version__ = '2.3.2b1' # This relies on each of the submodules having an __all__ variable. diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 8b0ef4bbd1d..a74015e5f28 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -773,49 +773,73 @@ def _get_fingerprint_and_hashfunc(self, req): else: return (None, None) - async def _create_direct_connection(self, req): + async def _wrap_create_connection(self, *args, + req, client_error=ClientConnectorError, + **kwargs): + try: + return await self._loop.create_connection(*args, **kwargs) + except certificate_errors as exc: + raise ClientConnectorCertificateError( + req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + raise client_error(req.connection_key, exc) from exc + + async def _create_direct_connection(self, req, + *, client_error=ClientConnectorError): sslcontext = self._get_ssl_context(req) fingerprint, hashfunc = self._get_fingerprint_and_hashfunc(req) - hosts = await self._resolve_host(req.url.raw_host, req.port) + try: + hosts = await self._resolve_host(req.url.raw_host, req.port) + except OSError as exc: + # in case of proxy it is not ClientProxyConnectionError + # it is problem of resolving proxy ip itself + raise ClientConnectorError(req.connection_key, exc) from exc + + last_exc = None for hinfo in hosts: + host = hinfo['host'] + port = hinfo['port'] + try: - host = hinfo['host'] - port = hinfo['port'] - transp, proto = await self._loop.create_connection( + transp, proto = await self._wrap_create_connection( self._factory, host, port, ssl=sslcontext, family=hinfo['family'], proto=hinfo['proto'], flags=hinfo['flags'], server_hostname=hinfo['hostname'] if sslcontext else None, - local_addr=self._local_addr) - has_cert = transp.get_extra_info('sslcontext') - if has_cert and fingerprint: - sock = transp.get_extra_info('socket') - if not hasattr(sock, 'getpeercert'): - # Workaround for asyncio 3.5.0 - # Starting from 3.5.1 version - # there is 'ssl_object' extra info in transport - sock = transp._ssl_protocol._sslpipe.ssl_object - # gives DER-encoded cert as a sequence of bytes (or None) - cert = sock.getpeercert(binary_form=True) - assert cert - got = hashfunc(cert).digest() - expected = fingerprint - if got != expected: - transp.close() - if not self._cleanup_closed_disabled: - self._cleanup_closed_transports.append(transp) - raise ServerFingerprintMismatch( - expected, got, host, port) - return transp, proto - except certificate_errors as exc: - raise ClientConnectorCertificateError( - req.connection_key, exc) from exc - except ssl_errors as exc: - raise ClientConnectorSSLError(req.connection_key, exc) from exc - except OSError as exc: - raise ClientConnectorError(req.connection_key, exc) from exc + local_addr=self._local_addr, + req=req, client_error=client_error) + except ClientConnectorError as exc: + last_exc = exc + continue + + has_cert = transp.get_extra_info('sslcontext') + if has_cert and fingerprint: + sock = transp.get_extra_info('socket') + if not hasattr(sock, 'getpeercert'): + # Workaround for asyncio 3.5.0 + # Starting from 3.5.1 version + # there is 'ssl_object' extra info in transport + sock = transp._ssl_protocol._sslpipe.ssl_object + # gives DER-encoded cert as a sequence of bytes (or None) + cert = sock.getpeercert(binary_form=True) + assert cert + got = hashfunc(cert).digest() + expected = fingerprint + if got != expected: + transp.close() + if not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(transp) + last_exc = ServerFingerprintMismatch( + expected, got, host, port) + continue + + return transp, proto + else: + raise last_exc async def _create_proxy_connection(self, req): headers = {} @@ -831,12 +855,10 @@ async def _create_proxy_connection(self, req): verify_ssl=req.verify_ssl, fingerprint=req.fingerprint, ssl_context=req.ssl_context) - try: - # create connection to proxy server - transport, proto = await self._create_direct_connection( - proxy_req) - except OSError as exc: - raise ClientProxyConnectionError(proxy_req, exc) from exc + + # create connection to proxy server + transport, proto = await self._create_direct_connection( + proxy_req, client_error=ClientProxyConnectionError) auth = proxy_req.headers.pop(hdrs.AUTHORIZATION, None) if auth is not None: @@ -887,9 +909,10 @@ async def _create_proxy_connection(self, req): finally: transport.close() - transport, proto = await self._loop.create_connection( + transport, proto = await self._wrap_create_connection( self._factory, ssl=sslcontext, sock=rawsock, - server_hostname=req.host) + server_hostname=req.host, + req=req) finally: proxy_resp.close() @@ -921,6 +944,10 @@ def path(self): return self._path async def _create_connection(self, req): - _, proto = await self._loop.create_unix_connection( - self._factory, self._path) + try: + _, proto = await self._loop.create_unix_connection( + self._factory, self._path) + except OSError as exc: + raise ClientConnectorError(req.connection_key, exc) from exc + return proto diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 101aa3163d7..8f54b6391c2 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -143,7 +143,7 @@ The client session supports the context manager protocol for self closing. .. versionadded:: 2.3 - :param bool from_env: Get proxies information from *HTTP_PROXY* / + :param bool trust_env: Get proxies information from *HTTP_PROXY* / *HTTPS_PROXY* environment variables if the parameter is ``True`` (``False`` by default). diff --git a/docs/conf.py b/docs/conf.py index 1e704f315c5..ae54c9490ff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -87,7 +87,7 @@ # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = 'toc' # General information about the project. project = 'aiohttp' diff --git a/docs/deployment.rst b/docs/deployment.rst index 0acab53ef13..738a488a00c 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -1,8 +1,8 @@ .. _aiohttp-deployment: -========================= -aiohttp server deployment -========================= +================= +Server Deployment +================= There are several options for aiohttp server deployment: diff --git a/docs/index.rst b/docs/index.rst index ff82bc2dad9..bf7ae319fbf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,17 +3,18 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -aiohttp: Asynchronous HTTP Client/Server -======================================== +=============================================================== +aiohttp: Asynchronous HTTP Client/Server for Python and asyncio +=============================================================== -HTTP client/server for :term:`asyncio` (:pep:`3156`). +HTTP client/server for :term:`asyncio` and Python. .. _GitHub: https://github.com/aio-libs/aiohttp .. _Freenode: http://freenode.net Key Features ------------- +============ - Supports both :ref:`aiohttp-client` and :ref:`HTTP Server `. - Supports both :ref:`Server WebSockets ` and @@ -22,7 +23,7 @@ Key Features :ref:`aiohttp-web-signals` and pluggable routing. Library Installation --------------------- +==================== .. code-block:: bash @@ -44,7 +45,7 @@ This option is highly recommended: $ pip install aiodns Getting Started ---------------- +=============== Client example:: @@ -82,13 +83,13 @@ Server example:: Tutorial --------- +======== :ref:`Polls tutorial ` Source code ------------ +=========== The project is hosted on GitHub_ @@ -101,7 +102,7 @@ Continuous Integration. Dependencies ------------- +============ - Python 3.4.2+ - *chardet* @@ -126,7 +127,7 @@ Dependencies Communication channels ----------------------- +====================== *aio-libs* google group: https://groups.google.com/forum/#!forum/aio-libs @@ -139,14 +140,14 @@ We support `Stack Overflow Please add *aiohttp* tag to your question there. Contributing ------------- +============ Please read the :ref:`instructions for contributors` before making a Pull Request. Authors and License -------------------- +=================== The ``aiohttp`` package is written mostly by Nikolay Kim and Andrew Svetlov. @@ -158,7 +159,7 @@ Feel free to improve this package and send a pull request to GitHub_. .. _aiohttp-backward-compatibility-policy: Policy for Backward Incompatible Changes ----------------------------------------- +======================================== *aiohttp* keeps backward compatibility. @@ -176,35 +177,7 @@ solved without major API change, but we are working hard for keeping these changes as rare as possible. -Contents --------- - -.. toctree:: - - client - client_reference - tutorial - web - web_reference - web_lowlevel - abc - multipart - multipart_reference - streams - api - logging - testing - deployment - faq - external - essays - contributing - changes - glossary - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` +Table Of Contents +================= + +To see the full table of contents open the :ref:`link `. diff --git a/docs/toc.rst b/docs/toc.rst new file mode 100644 index 00000000000..02b0de633ab --- /dev/null +++ b/docs/toc.rst @@ -0,0 +1,36 @@ +Table of Contents +================= + +.. toctree:: + :caption: Table of Contents + :name: mastertoc + + Introduction + client + client_reference + tutorial + web + web_reference + web_lowlevel + abc + multipart + multipart_reference + streams + api + logging + testing + deployment + faq + external + essays + contributing + changes + glossary + Sitemap + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tests/test_connector.py b/tests/test_connector.py index 8edb1c0cd12..761f3cf41be 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -2,6 +2,7 @@ import asyncio import gc +import hashlib import os.path import platform import shutil @@ -9,6 +10,7 @@ import ssl import tempfile import unittest +import uuid from unittest import mock import pytest @@ -18,7 +20,7 @@ from aiohttp import client, web from aiohttp.client import ClientRequest from aiohttp.connector import Connection, _DNSCacheTable -from aiohttp.test_utils import unused_port +from aiohttp.test_utils import make_mocked_coro, unused_port @pytest.fixture() @@ -358,6 +360,108 @@ async def certificate_error(*args, **kwargs): '[CertificateError: ()]') +async def test_tcp_connector_multiple_hosts_errors(loop): + conn = aiohttp.TCPConnector(loop=loop) + + ip1 = '192.168.1.1' + ip2 = '192.168.1.2' + ip3 = '192.168.1.3' + ip4 = '192.168.1.4' + ip5 = '192.168.1.5' + ips = [ip1, ip2, ip3, ip4, ip5] + ips_tried = [] + + fingerprint = hashlib.sha256(b'foo').digest() + + req = ClientRequest('GET', URL('https://mocked.host'), + fingerprint=fingerprint, + loop=loop) + + async def _resolve_host(host, port): + return [{ + 'hostname': host, + 'host': ip, + 'port': port, + 'family': socket.AF_INET, + 'proto': 0, + 'flags': socket.AI_NUMERICHOST} + for ip in ips] + + conn._resolve_host = _resolve_host + + os_error = certificate_error = ssl_error = fingerprint_error = False + connected = False + + async def create_connection(*args, **kwargs): + nonlocal os_error, certificate_error, ssl_error, fingerprint_error + nonlocal connected + + ip = args[1] + + ips_tried.append(ip) + + if ip == ip1: + os_error = True + raise OSError + + if ip == ip2: + certificate_error = True + raise ssl.CertificateError + + if ip == ip3: + ssl_error = True + raise ssl.SSLError + + if ip == ip4: + fingerprint_error = True + tr, pr = mock.Mock(), None + + def get_extra_info(param): + if param == 'sslcontext': + return True + + if param == 'socket': + s = mock.Mock() + s.getpeercert.return_value = b'not foo' + return s + + assert False + + tr.get_extra_info = get_extra_info + return tr, pr + + if ip == ip5: + connected = True + tr, pr = mock.Mock(), None + + def get_extra_info(param): + if param == 'sslcontext': + return True + + if param == 'socket': + s = mock.Mock() + s.getpeercert.return_value = b'foo' + return s + + assert False + + tr.get_extra_info = get_extra_info + return tr, pr + + assert False + + conn._loop.create_connection = create_connection + + await conn.connect(req) + assert ips == ips_tried + + assert os_error + assert certificate_error + assert ssl_error + assert fingerprint_error + assert connected + + async def test_tcp_connector_resolve_host(loop): conn = aiohttp.TCPConnector(loop=loop, use_dns_cache=True) @@ -495,6 +599,19 @@ async def test_tcp_connector_dns_throttle_requests_cancelled_when_close( await f +def test_dns_error(loop): + connector = aiohttp.TCPConnector(loop=loop) + connector._resolve_host = make_mocked_coro( + raise_exception=OSError('dont take it serious')) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + loop=loop, + ) + with pytest.raises(aiohttp.ClientConnectorError): + loop.run_until_complete(connector.connect(req)) + + def test_get_pop_empty_conns(loop): # see issue #473 conn = aiohttp.BaseConnector(loop=loop) @@ -1247,6 +1364,34 @@ async def handler(request): assert r.status == 200 +@pytest.mark.skipif(not hasattr(socket, 'AF_UNIX'), + reason="requires unix socket") +def test_unix_connector_not_found(loop): + connector = aiohttp.UnixConnector('/' + uuid.uuid4().hex, loop=loop) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + loop=loop, + ) + with pytest.raises(aiohttp.ClientConnectorError): + loop.run_until_complete(connector.connect(req)) + + +@pytest.mark.skipif(not hasattr(socket, 'AF_UNIX'), + reason="requires unix socket") +def test_unix_connector_permission(loop): + loop.create_unix_connection = make_mocked_coro( + raise_exception=PermissionError()) + connector = aiohttp.UnixConnector('/' + uuid.uuid4().hex, loop=loop) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + loop=loop, + ) + with pytest.raises(aiohttp.ClientConnectorError): + loop.run_until_complete(connector.connect(req)) + + def test_default_use_dns_cache(loop): conn = aiohttp.TCPConnector(loop=loop) assert conn.use_dns_cache diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 9a0380f520f..cc066904e2b 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -2,6 +2,7 @@ import gc import hashlib import socket +import ssl import unittest from unittest import mock @@ -215,7 +216,7 @@ def _test_connect_request_with_unicode_host(self, Request_mock): Request_mock.assert_called_with(mock.ANY, mock.ANY, "xn--9caa.com:80", mock.ANY, loop=loop) - def test_proxy_connection_error(self): + def test_proxy_dns_error(self): connector = aiohttp.TCPConnector(loop=self.loop) connector._resolve_host = make_mocked_coro( raise_exception=OSError('dont take it serious')) @@ -231,6 +232,24 @@ def test_proxy_connection_error(self): self.assertEqual(req.url.path, '/') self.assertEqual(dict(req.headers), expected_headers) + def test_proxy_connection_error(self): + connector = aiohttp.TCPConnector(loop=self.loop) + connector._resolve_host = make_mocked_coro([{ + 'hostname': 'www.python.org', + 'host': '127.0.0.1', 'port': 80, + 'family': socket.AF_INET, 'proto': 0, + 'flags': socket.AI_NUMERICHOST}]) + connector._loop.create_connection = make_mocked_coro( + raise_exception=OSError('dont take it serious')) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + proxy=URL('http://proxy.example.com'), + loop=self.loop, + ) + with self.assertRaises(aiohttp.ClientProxyConnectionError): + self.loop.run_until_complete(connector.connect(req)) + @mock.patch('aiohttp.connector.ClientRequest') def test_auth(self, ClientRequestMock): proxy_req = ClientRequest( @@ -314,30 +333,6 @@ def test_auth_from_url(self, ClientRequestMock): ssl_context=None, verify_ssl=None) conn.close() - @mock.patch('aiohttp.connector.ClientRequest') - def test_auth__not_modifying_request(self, ClientRequestMock): - proxy_req = ClientRequest('GET', - URL('http://user:pass@proxy.example.com'), - loop=self.loop) - ClientRequestMock.return_value = proxy_req - proxy_req_headers = dict(proxy_req.headers) - - connector = aiohttp.TCPConnector(loop=self.loop) - connector._resolve_host = make_mocked_coro( - raise_exception=OSError('nothing personal')) - - req = ClientRequest( - 'GET', URL('http://www.python.org'), - proxy=URL('http://user:pass@proxy.example.com'), - loop=self.loop, - ) - req_headers = dict(req.headers) - with self.assertRaises(aiohttp.ClientConnectorError): - self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.headers, req_headers) - self.assertEqual(req.url.path, '/') - self.assertEqual(proxy_req.headers, proxy_req_headers) - @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect(self, ClientRequestMock): proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), @@ -375,6 +370,92 @@ def test_https_connect(self, ClientRequestMock): proxy_resp.close() self.loop.run_until_complete(req.close()) + @mock.patch('aiohttp.connector.ClientRequest') + def test_https_connect_certificate_error(self, ClientRequestMock): + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), + loop=self.loop) + ClientRequestMock.return_value = proxy_req + + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) + proxy_resp._loop = self.loop + proxy_req.send = send_mock = mock.Mock() + send_mock.return_value = proxy_resp + proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) + + connector = aiohttp.TCPConnector(loop=self.loop) + connector._resolve_host = make_mocked_coro( + [{'hostname': 'hostname', 'host': '127.0.0.1', 'port': 80, + 'family': socket.AF_INET, 'proto': 0, 'flags': 0}]) + + seq = 0 + + @asyncio.coroutine + def create_connection(*args, **kwargs): + nonlocal seq + seq += 1 + + # connection to http://proxy.example.com + if seq == 1: + return mock.Mock(), mock.Mock() + # connection to https://www.python.org + elif seq == 2: + raise ssl.CertificateError + else: + assert False + + self.loop.create_connection = create_connection + + req = ClientRequest( + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), + loop=self.loop, + ) + with self.assertRaises(aiohttp.ClientConnectorCertificateError): + self.loop.run_until_complete(connector._create_connection(req)) + + @mock.patch('aiohttp.connector.ClientRequest') + def test_https_connect_ssl_error(self, ClientRequestMock): + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), + loop=self.loop) + ClientRequestMock.return_value = proxy_req + + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) + proxy_resp._loop = self.loop + proxy_req.send = send_mock = mock.Mock() + send_mock.return_value = proxy_resp + proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) + + connector = aiohttp.TCPConnector(loop=self.loop) + connector._resolve_host = make_mocked_coro( + [{'hostname': 'hostname', 'host': '127.0.0.1', 'port': 80, + 'family': socket.AF_INET, 'proto': 0, 'flags': 0}]) + + seq = 0 + + @asyncio.coroutine + def create_connection(*args, **kwargs): + nonlocal seq + seq += 1 + + # connection to http://proxy.example.com + if seq == 1: + return mock.Mock(), mock.Mock() + # connection to https://www.python.org + elif seq == 2: + raise ssl.SSLError + else: + assert False + + self.loop.create_connection = create_connection + + req = ClientRequest( + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), + loop=self.loop, + ) + with self.assertRaises(aiohttp.ClientConnectorSSLError): + self.loop.run_until_complete(connector._create_connection(req)) + @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect_runtime_error(self, ClientRequestMock): proxy_req = ClientRequest('GET', URL('http://proxy.example.com'),