Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add async support for temporary file handling. Closes #344 #867

Merged
merged 30 commits into from
Mar 2, 2025
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
beb8285
feat: Add async support for temporary file handling. Closes #344
11kkw Feb 10, 2025
07730ef
Merge branch 'master' into async-tempfile-support
agronholm Feb 10, 2025
93c9ae4
Fix: Apply requested changes from @agronholm
11kkw Feb 11, 2025
546a462
Merge branch 'master' into async-tempfile-support
agronholm Feb 11, 2025
684176b
fix: add blank lines after control blocks, fix Pyright errors, and ig…
11kkw Feb 11, 2025
2c89b61
fix(tempfile): apply review feedback and improve async tempfile
11kkw Feb 12, 2025
2e6ea50
Update src/anyio/_core/_tempfile.py
11kkw Feb 14, 2025
db3c2cd
Update src/anyio/_core/_tempfile.py
11kkw Feb 14, 2025
ad12c85
tempfile: update type annotations, fix Py3.14 compatibility, and upda…
11kkw Feb 14, 2025
50c498f
Merge branch 'master' into async-tempfile-support
agronholm Feb 15, 2025
b7a9722
Removed redundant py3.14 fixes
agronholm Feb 15, 2025
2a1a1cf
Update src/anyio/_core/_tempfile.py
11kkw Feb 16, 2025
e608cdc
Update src/anyio/_core/_tempfile.py
11kkw Feb 16, 2025
4cdadc6
Update tests/test_tempfile.py
11kkw Feb 16, 2025
bd36d3c
Update tests/test_tempfile.py
11kkw Feb 16, 2025
86719ec
Update tests/test_tempfile.py
11kkw Feb 16, 2025
c129057
Update tests/test_tempfile.py
11kkw Feb 16, 2025
6551774
Update tests/test_tempfile.py
11kkw Feb 16, 2025
f730c72
Update tests/test_tempfile.py
11kkw Feb 16, 2025
4dc6c71
Update tests/test_tempfile.py
11kkw Feb 16, 2025
bd61572
Update tests/test_tempfile.py
11kkw Feb 16, 2025
1509c28
docs: Include TemporaryFile-related classes in API reference
11kkw Feb 16, 2025
78bb1d1
api.rst update
11kkw Feb 16, 2025
9eaa5c5
Merge branch 'master' into async-tempfile-support
agronholm Feb 22, 2025
77e7259
Reimplemented SpooledTemporaryFile and improved type annotations
agronholm Feb 23, 2025
058ab8a
Updated tests
agronholm Feb 23, 2025
e88499e
Reduced the number of mypy errors
agronholm Mar 1, 2025
e57dd1d
Silenced the remaining mypy errors
agronholm Mar 1, 2025
8b59b66
Merge branch 'master' into async-tempfile-support
agronholm Mar 1, 2025
7d8f4b7
Added some finishing touches
agronholm Mar 2, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ extend-select = [
[tool.ruff.lint.isort]
"required-imports" = ["from __future__ import annotations"]

[tool.ruff.lint.per-file-ignores]
"tests/test_tempfile.py" = ["ASYNC230"]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Which open call is blocking?

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The one in test_mkstemp().


[tool.mypy]
python_version = "3.13"
strict = true
Expand Down
8 changes: 8 additions & 0 deletions src/anyio/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,14 @@
from ._core._tasks import current_effective_deadline as current_effective_deadline
from ._core._tasks import fail_after as fail_after
from ._core._tasks import move_on_after as move_on_after
from ._core._tempfile import NamedTemporaryFile as NamedTemporaryFile
from ._core._tempfile import SpooledTemporaryFile as SpooledTemporaryFile
from ._core._tempfile import TemporaryDirectory as TemporaryDirectory
from ._core._tempfile import TemporaryFile as TemporaryFile
from ._core._tempfile import gettempdir as gettempdir
from ._core._tempfile import gettempdirb as gettempdirb
from ._core._tempfile import mkdtemp as mkdtemp
from ._core._tempfile import mkstemp as mkstemp
from ._core._testing import TaskInfo as TaskInfo
from ._core._testing import get_current_task as get_current_task
from ._core._testing import get_running_tasks as get_running_tasks
Expand Down
276 changes: 276 additions & 0 deletions src/anyio/_core/_tempfile.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,276 @@
from __future__ import annotations

import sys
import tempfile
from functools import partial
from types import TracebackType
from typing import Any, AnyStr, Callable, Generic, cast

from .. import to_thread
from .._core._fileio import AsyncFile


class TemporaryFile(Generic[AnyStr]):
def __init__(
self,
mode: str = "w+b",
buffering: int = -1,
encoding: str | None = None,
newline: str | None = None,
suffix: AnyStr | None = None,
prefix: AnyStr | None = None,
dir: AnyStr | None = None,
*,
errors: str | None = None,
) -> None:
self.mode = mode
self.buffering = buffering
self.encoding = encoding
self.newline = newline
self.suffix: AnyStr | None = suffix
self.prefix: AnyStr | None = prefix
self.dir: AnyStr | None = dir
self.errors = errors

self._async_file: AsyncFile | None = None

async def __aenter__(self) -> AsyncFile[AnyStr]:
fp = await to_thread.run_sync(
lambda: tempfile.TemporaryFile(
self.mode,
self.buffering,
self.encoding,
self.newline,
self.suffix,
self.prefix,
self.dir,
errors=self.errors,
)
)
self._async_file = AsyncFile(fp)
return self._async_file

async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
if self._async_file is not None:
await self._async_file.aclose()
self._async_file = None


class NamedTemporaryFile(Generic[AnyStr]):
def __init__(
self,
mode: str = "w+b",
buffering: int = -1,
encoding: str | None = None,
newline: str | None = None,
suffix: AnyStr | None = None,
prefix: AnyStr | None = None,
dir: AnyStr | None = None,
delete: bool = True,
*,
errors: str | None = None,
delete_on_close: bool = True,
) -> None:
self.mode = mode
self.buffering = buffering
self.encoding = encoding
self.newline = newline
self.suffix: AnyStr | None = suffix
self.prefix: AnyStr | None = prefix
self.dir: AnyStr | None = dir
self.delete = delete
self.errors = errors
self.delete_on_close = delete_on_close

self._async_file: AsyncFile | None = None

async def __aenter__(self) -> AsyncFile[AnyStr]:
params: dict[str, Any] = {
"mode": self.mode,
"buffering": self.buffering,
"encoding": self.encoding,
"newline": self.newline,
"suffix": self.suffix,
"prefix": self.prefix,
"dir": self.dir,
"delete": self.delete,
"errors": self.errors,
}
if sys.version_info >= (3, 12):
params["delete_on_close"] = self.delete_on_close

fp = await to_thread.run_sync(lambda: tempfile.NamedTemporaryFile(**params))
self._async_file = AsyncFile(fp)
return self._async_file

async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
if self._async_file is not None:
await self._async_file.aclose()
self._async_file = None


class SpooledTemporaryFile(Generic[AnyStr]):
Copy link
Collaborator

@graingert graingert Feb 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally for SpooledTemporaryFile is that writes/reads to/from the BytesIO buffer will not go to a thread

SpooledTemporaryFile needs to not go to a thread for these 'small' file things because the usecase is starlette multipart form file uploads which needs to remain fast for tiny uploads

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Kludex would you like to participate in the review of this feature too?

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've made the necessary changes based on your feedback, but I'm not entirely sure if I've implemented them correctly due to my limited experience. I'd really appreciate it if you could review my changes and provide any further suggestions. Thank you for your guidance!"

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This class needs an aclose()

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, it needs the aclose().

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this a generic? The tempfile.SpooledTemporaryFile is not a generic.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's something different happening here...

from tempfile import SpooledTemporaryFile

from anyio import SpooledTemporaryFile as anyio_SpooledTemporaryFile

a = SpooledTemporaryFile()
b = anyio_SpooledTemporaryFile()

I think it defaults to bytes. My IDE is running pyright (strict mode):

Screenshot 2025-02-22 at 13 53 19

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Curious, mypy reports the type as SpooledTemporaryFile[builtins.str] while pyright reports it as SpooledTemporaryFile[Unknown].

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

from tempfile import SpooledTemporaryFile

from anyio import SpooledTemporaryFile as AnyIOSpooledTemporaryFile

async def main() -> None:
    with SpooledTemporaryFile() as f1:
        reveal_type(f1)

    async with AnyIOSpooledTemporaryFile() as f2:
        reveal_type(f2)

Gives me:

  /home/alex/workspace/anyio/client.py:7:21 - information: Type of "f1" is "SpooledTemporaryFile[bytes]"
  /home/alex/workspace/anyio/client.py:10:21 - information: Type of "f2" is "SpooledTemporaryFile[Unknown]"

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I refactored SpooledTemporaryFile in a way that should satisfy @graingert. However, this caused a lot fo mypy errors that I can't understand.

Copy link
Contributor Author

@11kkw 11kkw Feb 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm sorry for bringing this up so late. However, since the official documentation states, The dir, prefix and suffix parameters have the same meaning and defaults as with mkstemp() I assumed that the parameters for mkstemp are of type AnyStr, and I proceeded with that approach.

https://docs.python.org/3/library/tempfile.html#tempfile.mkstemp

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Whatever the reason, using AnyStr in both the class type argument AND the initializer arguments makes no sense because the type parameter in the class is supposed to determine whether the input and output type is str or bytes. But the reason for the initializer arguments to be parametrized is because they in part determine file name and you want them to be either str or bytes. Two different, unrelated uses.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Whatever the reason, using AnyStr in both the class type argument AND the initializer arguments makes no sense because the type parameter in the class is supposed to determine whether the input and output type is str or bytes. But the reason for the initializer arguments to be parametrized is because they in part determine file name and you want them to be either str or bytes. Two different, unrelated uses.

I agree with this change

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Based on the discussions I had on Discord, it would appear that neither mypy or pyright can handle overriding methods like write() or writelines() correctly. I'm therefore leaning towards adding # type: ignore comments to mute the errors about methods like write() and writelines().

def __init__(
self,
max_size: int = 0,
mode: str = "w+b",
buffering: int = -1,
encoding: str | None = None,
newline: str | None = None,
suffix: AnyStr | None = None,
prefix: AnyStr | None = None,
dir: AnyStr | None = None,
*,
errors: str | None = None,
) -> None:
self.max_size = max_size
self.mode = mode
self.buffering = buffering
self.encoding = encoding
self.newline = newline
self.suffix: AnyStr | None = suffix
self.prefix: AnyStr | None = prefix
self.dir: AnyStr | None = dir
self.errors = errors

self._async_file: AsyncFile | None = None

async def __aenter__(
self: SpooledTemporaryFile[AnyStr],
) -> SpooledTemporaryFile[AnyStr]:
fp = await to_thread.run_sync(
partial(
tempfile.SpooledTemporaryFile,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Creating the SpooledTemporyFile doesn't perform blocking disk IO so this shouldn't need a thread

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, you already mentioned that ^ here.

self.max_size,
self.mode,
self.buffering,
self.encoding,
self.newline,
self.suffix,
self.prefix,
self.dir,
errors=self.errors,
)
)
self._async_file = AsyncFile(fp)
return self

async def rollover(self) -> None:
if self._async_file is None:
raise RuntimeError("Internal file is not initialized.")

await to_thread.run_sync(cast(Callable[[], None], self._async_file.rollover))

@property
def closed(self) -> bool:
if self._async_file is not None:
try:
return bool(self._async_file.closed)
except AttributeError:
f = getattr(self._async_file, "_f", None)
if f is None:
return True
return bool(f.closed)
return True

def __getattr__(self, attr: str) -> Any:
if self._async_file is not None:
return getattr(self._async_file, attr)

raise AttributeError(f"{self.__class__.__name__} has no attribute {attr}")

async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
if self._async_file is not None:
await self._async_file.aclose()
self._async_file = None


class TemporaryDirectory(Generic[AnyStr]):
def __init__(
self,
suffix: AnyStr | None = None,
prefix: AnyStr | None = None,
dir: AnyStr | None = None,
*,
ignore_cleanup_errors: bool = False,
delete: bool = True,
) -> None:
self.suffix: AnyStr | None = suffix
self.prefix: AnyStr | None = prefix
self.dir: AnyStr | None = dir
self.ignore_cleanup_errors = ignore_cleanup_errors
self.delete = delete

self._tempdir: tempfile.TemporaryDirectory | None = None

async def __aenter__(self) -> str:
params: dict[str, Any] = {
"suffix": self.suffix,
"prefix": self.prefix,
"dir": self.dir,
}
if sys.version_info >= (3, 10):
params["ignore_cleanup_errors"] = self.ignore_cleanup_errors

if sys.version_info >= (3, 12):
params["delete"] = self.delete

self._tempdir = await to_thread.run_sync(
lambda: tempfile.TemporaryDirectory(**params)
)
return await to_thread.run_sync(self._tempdir.__enter__)

async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
if self._tempdir is not None:
await to_thread.run_sync(
self._tempdir.__exit__, exc_type, exc_value, traceback
)

async def cleanup(self) -> None:
if self._tempdir is not None:
await to_thread.run_sync(self._tempdir.cleanup)


async def mkstemp(
suffix: AnyStr | None = None,
prefix: AnyStr | None = None,
dir: AnyStr | None = None,
text: bool = False,
) -> tuple[int, str | bytes]:
return await to_thread.run_sync(lambda: tempfile.mkstemp(suffix, prefix, dir, text))


async def mkdtemp(
suffix: AnyStr | None = None,
prefix: AnyStr | None = None,
dir: AnyStr | None = None,
) -> str | bytes:
return await to_thread.run_sync(lambda: tempfile.mkdtemp(suffix, prefix, dir))


async def gettempdir() -> str:
return await to_thread.run_sync(tempfile.gettempdir)


async def gettempdirb() -> bytes:
return await to_thread.run_sync(tempfile.gettempdirb)
Loading
Loading