Merge master to 2.3 (aio-libs#2456)

* Fix wrap oserrors. (aio-libs#2442)

* Fix wrap oserrors.

* Skip unix socket tests on windows

* Fix wrap ssl errors for proxy connector. (aio-libs#2446)

* Fix 2451: Rename from_env to trust_env in client reference.

* Fix connection for multiple dns hosts (aio-libs#2447)

* Proof of concept fix for multiple dns hosts

* Revert getpeercert

* Added test for multiple dns hosts and errors.

* Added change notes.

* Remove redunant assert.

* Minor docs formatting

* Move TOC to separate RST file

* Build tarball on appveyor

* Bump to 2.3.2b1

* @asyncio.coroutine -> async

* Update index.rst

* Update toc.rst

.appveyor.yml

@@ -17,7 +17,7 @@ test_script:
   - "tools/build.cmd %PYTHON%\\python.exe setup.py test"
 
 after_test:
-  - "tools/build.cmd %PYTHON%\\python.exe setup.py bdist_wheel"
+  - "tools/build.cmd %PYTHON%\\python.exe setup.py sdist bdist_wheel"
 
 artifacts:
   - path: dist\*

CHANGES.rst

@@ -1,6 +1,6 @@
-=======
-Changes
-=======
+=========
+Changelog
+=========
 
 ..
     You should *NOT* be adding new change log entries to this file, this

CHANGES/2408.bugfix

@@ -0,0 +1 @@
+Fix ClientConnectorSSLError and ClientProxyConnectionError for proxy connector

CHANGES/2423.bugfix

@@ -0,0 +1 @@
+Fix connector convert OSError to ClientConnectorError

CHANGES/2424.bugfix

@@ -0,0 +1 @@
+Fix connection attempts for multiple dns hosts

CHANGES/2451.doc

@@ -0,0 +1 @@
+Rename `from_env` to  `trust_env` in client reference.

aiohttp/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.3.1'
+__version__ = '2.3.2b1'
 
 # This relies on each of the submodules having an __all__ variable.
 

aiohttp/connector.py

@@ -773,49 +773,73 @@ def _get_fingerprint_and_hashfunc(self, req):
         else:
             return (None, None)
 
-    async def _create_direct_connection(self, req):
+    async def _wrap_create_connection(self, *args,
+                                      req, client_error=ClientConnectorError,
+                                      **kwargs):
+        try:
+            return await self._loop.create_connection(*args, **kwargs)
+        except certificate_errors as exc:
+            raise ClientConnectorCertificateError(
+                req.connection_key, exc) from exc
+        except ssl_errors as exc:
+            raise ClientConnectorSSLError(req.connection_key, exc) from exc
+        except OSError as exc:
+            raise client_error(req.connection_key, exc) from exc
+
+    async def _create_direct_connection(self, req,
+                                        *, client_error=ClientConnectorError):
         sslcontext = self._get_ssl_context(req)
         fingerprint, hashfunc = self._get_fingerprint_and_hashfunc(req)
 
-        hosts = await self._resolve_host(req.url.raw_host, req.port)
+        try:
+            hosts = await self._resolve_host(req.url.raw_host, req.port)
+        except OSError as exc:
+            # in case of proxy it is not ClientProxyConnectionError
+            # it is problem of resolving proxy ip itself
+            raise ClientConnectorError(req.connection_key, exc) from exc
+
+        last_exc = None
 
         for hinfo in hosts:
+            host = hinfo['host']
+            port = hinfo['port']
+
             try:
-                host = hinfo['host']
-                port = hinfo['port']
-                transp, proto = await self._loop.create_connection(
+                transp, proto = await self._wrap_create_connection(
                     self._factory, host, port,
                     ssl=sslcontext, family=hinfo['family'],
                     proto=hinfo['proto'], flags=hinfo['flags'],
                     server_hostname=hinfo['hostname'] if sslcontext else None,
-                    local_addr=self._local_addr)
-                has_cert = transp.get_extra_info('sslcontext')
-                if has_cert and fingerprint:
-                    sock = transp.get_extra_info('socket')
-                    if not hasattr(sock, 'getpeercert'):
-                        # Workaround for asyncio 3.5.0
-                        # Starting from 3.5.1 version
-                        # there is 'ssl_object' extra info in transport
-                        sock = transp._ssl_protocol._sslpipe.ssl_object
-                    # gives DER-encoded cert as a sequence of bytes (or None)
-                    cert = sock.getpeercert(binary_form=True)
-                    assert cert
-                    got = hashfunc(cert).digest()
-                    expected = fingerprint
-                    if got != expected:
-                        transp.close()
-                        if not self._cleanup_closed_disabled:
-                            self._cleanup_closed_transports.append(transp)
-                        raise ServerFingerprintMismatch(
-                            expected, got, host, port)
-                return transp, proto
-            except certificate_errors as exc:
-                raise ClientConnectorCertificateError(
-                    req.connection_key, exc) from exc
-            except ssl_errors as exc:
-                raise ClientConnectorSSLError(req.connection_key, exc) from exc
-            except OSError as exc:
-                raise ClientConnectorError(req.connection_key, exc) from exc
+                    local_addr=self._local_addr,
+                    req=req, client_error=client_error)
+            except ClientConnectorError as exc:
+                last_exc = exc
+                continue
+
+            has_cert = transp.get_extra_info('sslcontext')
+            if has_cert and fingerprint:
+                sock = transp.get_extra_info('socket')
+                if not hasattr(sock, 'getpeercert'):
+                    # Workaround for asyncio 3.5.0
+                    # Starting from 3.5.1 version
+                    # there is 'ssl_object' extra info in transport
+                    sock = transp._ssl_protocol._sslpipe.ssl_object
+                # gives DER-encoded cert as a sequence of bytes (or None)
+                cert = sock.getpeercert(binary_form=True)
+                assert cert
+                got = hashfunc(cert).digest()
+                expected = fingerprint
+                if got != expected:
+                    transp.close()
+                    if not self._cleanup_closed_disabled:
+                        self._cleanup_closed_transports.append(transp)
+                    last_exc = ServerFingerprintMismatch(
+                        expected, got, host, port)
+                    continue
+
+            return transp, proto
+        else:
+            raise last_exc
 
     async def _create_proxy_connection(self, req):
         headers = {}
@@ -831,12 +855,10 @@ async def _create_proxy_connection(self, req):
             verify_ssl=req.verify_ssl,
             fingerprint=req.fingerprint,
             ssl_context=req.ssl_context)
-        try:
-            # create connection to proxy server
-            transport, proto = await self._create_direct_connection(
-                proxy_req)
-        except OSError as exc:
-            raise ClientProxyConnectionError(proxy_req, exc) from exc
+
+        # create connection to proxy server
+        transport, proto = await self._create_direct_connection(
+            proxy_req, client_error=ClientProxyConnectionError)
 
         auth = proxy_req.headers.pop(hdrs.AUTHORIZATION, None)
         if auth is not None:
@@ -887,9 +909,10 @@ async def _create_proxy_connection(self, req):
                 finally:
                     transport.close()
 
-                transport, proto = await self._loop.create_connection(
+                transport, proto = await self._wrap_create_connection(
                     self._factory, ssl=sslcontext, sock=rawsock,
-                    server_hostname=req.host)
+                    server_hostname=req.host,
+                    req=req)
             finally:
                 proxy_resp.close()
 
@@ -921,6 +944,10 @@ def path(self):
         return self._path
 
     async def _create_connection(self, req):
-        _, proto = await self._loop.create_unix_connection(
-            self._factory, self._path)
+        try:
+            _, proto = await self._loop.create_unix_connection(
+                self._factory, self._path)
+        except OSError as exc:
+            raise ClientConnectorError(req.connection_key, exc) from exc
+
         return proto

docs/client_reference.rst

@@ -143,7 +143,7 @@ The client session supports the context manager protocol for self closing.
 
       .. versionadded:: 2.3
 
-   :param bool from_env: Get proxies information from *HTTP_PROXY* /
+   :param bool trust_env: Get proxies information from *HTTP_PROXY* /
                          *HTTPS_PROXY* environment variables if the
                          parameter is ``True`` (``False`` by default).
 

docs/conf.py

@@ -87,7 +87,7 @@
 # source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = 'toc'
 
 # General information about the project.
 project = 'aiohttp'

docs/deployment.rst

@@ -1,8 +1,8 @@
 .. _aiohttp-deployment:
 
-=========================
-aiohttp server deployment
-=========================
+=================
+Server Deployment
+=================
 
 There are several options for aiohttp server deployment:
 

docs/index.rst

@@ -3,17 +3,18 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-aiohttp: Asynchronous HTTP Client/Server
-========================================
+===============================================================
+aiohttp: Asynchronous HTTP Client/Server for Python and asyncio
+===============================================================
 
-HTTP client/server for :term:`asyncio` (:pep:`3156`).
+HTTP client/server for :term:`asyncio` and Python.
 
 .. _GitHub: https://github.com/aio-libs/aiohttp
 .. _Freenode: http://freenode.net
 
 
 Key Features
-------------
+============
 
 - Supports both :ref:`aiohttp-client` and :ref:`HTTP Server <aiohttp-web>`.
 - Supports both :ref:`Server WebSockets <aiohttp-web-websockets>` and
@@ -22,7 +23,7 @@ Key Features
   :ref:`aiohttp-web-signals` and pluggable routing.
 
 Library Installation
---------------------
+====================
 
 .. code-block:: bash
 
@@ -44,7 +45,7 @@ This option is highly recommended:
    $ pip install aiodns
 
 Getting Started
----------------
+===============
 
 Client example::
 
@@ -82,13 +83,13 @@ Server example::
 
 
 Tutorial
---------
+========
 
 :ref:`Polls tutorial <aiohttp-tutorial>`
 
 
 Source code
------------
+===========
 
 The project is hosted on GitHub_
 
@@ -101,7 +102,7 @@ Continuous Integration.
 
 
 Dependencies
-------------
+============
 
 - Python 3.4.2+
 - *chardet*
@@ -126,7 +127,7 @@ Dependencies
 
 
 Communication channels
-----------------------
+======================
 
 *aio-libs* google group: https://groups.google.com/forum/#!forum/aio-libs
 
@@ -139,14 +140,14 @@ We support `Stack Overflow
 Please add *aiohttp* tag to your question there.
 
 Contributing
-------------
+============
 
 Please read the :ref:`instructions for contributors<aiohttp-contributing>`
 before making a Pull Request.
 
 
 Authors and License
--------------------
+===================
 
 The ``aiohttp`` package is written mostly by Nikolay Kim and Andrew Svetlov.
 
@@ -158,7 +159,7 @@ Feel free to improve this package and send a pull request to GitHub_.
 .. _aiohttp-backward-compatibility-policy:
 
 Policy for Backward Incompatible Changes
-----------------------------------------
+========================================
 
 *aiohttp* keeps backward compatibility.
 
@@ -176,35 +177,7 @@ solved without major API change, but we are working hard for keeping
 these changes as rare as possible.
 
 
-Contents
---------
-
-.. toctree::
-
-   client
-   client_reference
-   tutorial
-   web
-   web_reference
-   web_lowlevel
-   abc
-   multipart
-   multipart_reference
-   streams
-   api
-   logging
-   testing
-   deployment
-   faq
-   external
-   essays
-   contributing
-   changes
-   glossary
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+Table Of Contents
+=================
+
+To see the full table of contents open the :ref:`link <mastertoc>`.

docs/toc.rst

@@ -0,0 +1,36 @@
+Table of Contents
+=================
+
+.. toctree::
+   :caption: Table of Contents
+   :name: mastertoc
+
+   Introduction <index>
+   client
+   client_reference
+   tutorial
+   web
+   web_reference
+   web_lowlevel
+   abc
+   multipart
+   multipart_reference
+   streams
+   api
+   logging
+   testing
+   deployment
+   faq
+   external
+   essays
+   contributing
+   changes
+   glossary
+   Sitemap <self>
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`