Fix timeout when retrying uploads (#90)

.github/workflows/release.yml

@@ -0,0 +1,84 @@
+name: Build and publish
+
+# Run on PR requests. And on master itself.
+on:
+  push:
+    branches:
+      - main  # just build the sdist skip release
+    tags:
+      - "*"
+  pull_request:  # also build on PRs touching some files
+    paths:
+      - ".github/workflows/release.yml"
+      - "MANIFEST.in"
+      - "setup.cfg"
+      - "setup.py"
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Build a source tarball
+        run: |
+          python -m pip install --upgrade setuptools wheel
+          python setup.py sdist bdist_wheel
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./dist/*
+          retention-days: 5
+
+  publish:
+    name: Publish on GitHub and PyPI
+    needs: [build]
+    runs-on: ubuntu-latest
+    # release on every tag
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact
+          path: dist
+
+      - name: Create GitHub Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+          draft: false
+          prerelease: false
+
+      - name: Get Asset name
+        run: |
+          export PKG=$(ls dist/ | grep tar)
+          set -- $PKG
+          echo "name=$1" >> $GITHUB_ENV
+
+      - name: Upload Release Asset (sdist) to GitHub
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: dist/${{ env.name }}
+          asset_name: ${{ env.name }}
+          asset_content_type: application/zip
+
+      - name: Upload Release Assets to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_UPLOAD_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,49 @@
+name: Linux
+
+# Run on PR requests. And on master itself.
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  TestLinux:
+    name: Linux, Python ${{ matrix.python }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # 2019
+          - python: 3.8
+            pins: "certifi==2019.* urllib3==1.24.* aiohttp==3.6.3 aiofiles==0.6.*"
+          # 2021
+          - python: 3.9
+            pins: "certifi==2021.* urllib3==1.26.6 aiohttp==3.7.* aiofiles==0.7.*"
+          # 2022
+          - python: "3.10"
+            pins: "certifi==2022.* urllib3==1.26.* aiohttp==3.8.* aiofiles==0.8.*"
+          # current
+          - python: "3.11"
+            pins: ""
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install python dependencies
+        shell: bash
+        run: |
+          pip install --disable-pip-version-check --upgrade pip setuptools wheel
+          pip install ${{ matrix.pins }} .[aio,test]
+          pip list
+
+      - name: Run tests
+        shell: bash
+        run: |
+          pytest

HISTORY.rst

@@ -5,7 +5,7 @@ History
 4.1.2 (unreleased)
 ------------------
 
-- Nothing changed yet.
+- Fix timeout when retrying uploads.
 
 
 4.1.1 (2022-11-21)

README.rst

@@ -9,8 +9,8 @@ threedi-api-client
 .. image:: https://img.shields.io/pypi/v/threedi-api-client.svg
         :target: https://pypi.python.org/pypi/threedi-api-client
 
-.. image:: https://github.com/nens/threedi-api-client/actions/workflows/main.yml/badge.svg
-        :target: https://github.com/nens/threedi-api-client/actions/workflows/main.yml
+.. image:: https://github.com/nens/threedi-api-client/actions/workflows/test.yml/badge.svg
+        :target: https://github.com/nens/threedi-api-client/actions/workflows/test.yml
 
 
 * A Python library for interfacing with the 3Di API

setup.py

@@ -5,7 +5,6 @@
 
 from setuptools import setup, find_packages
 import codecs
-import re
 import os
 import pathlib
 
@@ -37,15 +36,15 @@ def get_version():
 
 requirements = [
     'certifi>=2019.3.9',
-    'urllib3>=1.15',
+    'urllib3>=1.24,<2.1',
     'six>=1.10',
     'python-dateutil',
 ]
 
-aio_requirements = ["aiohttp>=3.6.3", "aiofiles"]
+aio_requirements = ["aiohttp>=3.6.3", "aiofiles>=0.6"]
 
 # Note: mock contains a backport of AsyncMock
-test_requirements = ["pytest", "pytest-asyncio<0.19", "mock", 'pyjwt']
+test_requirements = ["pytest", "pytest-asyncio", "mock ; python_version<'3.8'", 'pyjwt']
 
 
 setup(
@@ -56,10 +55,7 @@ def get_version():
         "Intended Audience :: Developers",
         "License :: OSI Approved :: BSD License",
         "Natural Language :: English",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3",
     ],
     description="client for the threedi API",
     install_requires=requirements,
@@ -76,7 +72,7 @@ def get_version():
             "threedi_api_client.*",
         ]
     ),
-    python_requires=">=3.6",
+    python_requires=">=3.7",
     extras_require={
         "aio": aio_requirements,
         "test": test_requirements,

tests/test_files_aio.py

@@ -2,11 +2,15 @@
 from concurrent.futures import ThreadPoolExecutor
 
 import pytest
+import pytest_asyncio
 from aiofiles.threadpool import AsyncBufferedIOBase
 
-# note: unittest.mock has no asyncio support in Python < 3.7,
-# but luckily mock backported it:
-from mock import AsyncMock, DEFAULT, Mock, patch
+try:
+    from unittest.mock import AsyncMock, DEFAULT, Mock, patch
+except ImportError:
+    # Python 3.7
+    from mock.mock import AsyncMock, DEFAULT, Mock, patch
+
 
 from threedi_api_client.openapi import ApiException
 from threedi_api_client.aio.files import (
@@ -38,21 +42,21 @@ async def write(self, *args, **kwargs):
         return self._io.write(*args, **kwargs)
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
 async def aio_request():
     with patch("aiohttp.ClientSession.request", new_callable=AsyncMock) as aio_request:
         yield aio_request
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
 async def response_error():
     # mimics aiohttp.ClientResponse
     response = AsyncMock()
     response.status = 503
     return response
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
 async def responses_single():
     # mimics aiohttp.ClientResponse
     response = AsyncMock()
@@ -62,7 +66,7 @@ async def responses_single():
     return [response]
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
 async def responses_double():
     # mimics aiohttp.ClientResponse
     response1 = AsyncMock()
@@ -231,14 +235,14 @@ async def test_download_file_directory(mocked_download_fileobj, tmp_path):
     assert args[1].name == str(tmp_path / "a.b")
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
 async def upload_response():
     response = AsyncMock()
     response.status = 200
     return response
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
 async def fileobj():
     stream = AsyncBytesIO()
     await stream.write(b"X" * 39)

tests/test_threedi_api.py

@@ -1,7 +1,11 @@
 from unittest import mock
 
 import pytest
-from mock import AsyncMock
+try:
+    from unittest.mock import AsyncMock
+except ImportError:
+    # Python 3.7
+    from mock.mock import AsyncMock
 
 from threedi_api_client import ThreediApi
 from threedi_api_client.aio.openapi.api_client import ApiClient as AsyncApiClient

threedi_api_client/files.py

@@ -271,6 +271,28 @@ def _iter_chunks(
         yield data
 
 
+class _SeekableChunkIterator:
+    """A chunk iterator that can be rewinded in case of urllib3 retries."""
+    def __init__(
+        self,
+        fileobj: BinaryIO,
+        chunk_size: int,
+        callback_func: Optional[Callable[[int], None]] = None,
+    ):
+        self.fileobj = fileobj
+        self.chunk_size = chunk_size
+        self.callback_func = callback_func
+
+    def seek(self, pos: int):
+        return self.fileobj.seek(pos)
+
+    def tell(self):
+        return self.fileobj.tell()
+
+    def __iter__(self):
+        return _iter_chunks(self.fileobj, self.chunk_size, self.callback_func)
+
+
 def upload_fileobj(
     url: str,
     fileobj: BinaryIO,
@@ -337,7 +359,9 @@ def callback(uploaded_bytes: int):
                 uploaded_bytes = file_size
             callback_func(uploaded_bytes, file_size)
 
-    iterable = _iter_chunks(fileobj, chunk_size=chunk_size, callback_func=callback)
+    iterable = _SeekableChunkIterator(
+        fileobj, chunk_size=chunk_size, callback_func=callback,
+    )
 
     # Tested: both Content-Length and Content-MD5 are checked by Minio
     headers = {