-
Notifications
You must be signed in to change notification settings - Fork 146
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add async support for temporary file handling. Closes #344 #867
Changes from 5 commits
beb8285
07730ef
93c9ae4
546a462
684176b
2c89b61
2e6ea50
db3c2cd
ad12c85
50c498f
b7a9722
2a1a1cf
e608cdc
4cdadc6
bd36d3c
86719ec
c129057
6551774
f730c72
4dc6c71
bd61572
1509c28
78bb1d1
9eaa5c5
77e7259
058ab8a
e88499e
e57dd1d
8b59b66
7d8f4b7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,276 @@ | ||
from __future__ import annotations | ||
|
||
import sys | ||
import tempfile | ||
from functools import partial | ||
from types import TracebackType | ||
from typing import Any, AnyStr, Callable, Generic, cast | ||
|
||
from .. import to_thread | ||
from .._core._fileio import AsyncFile | ||
|
||
|
||
class TemporaryFile(Generic[AnyStr]): | ||
def __init__( | ||
self, | ||
mode: str = "w+b", | ||
buffering: int = -1, | ||
encoding: str | None = None, | ||
newline: str | None = None, | ||
suffix: AnyStr | None = None, | ||
prefix: AnyStr | None = None, | ||
dir: AnyStr | None = None, | ||
*, | ||
errors: str | None = None, | ||
) -> None: | ||
self.mode = mode | ||
self.buffering = buffering | ||
self.encoding = encoding | ||
self.newline = newline | ||
self.suffix: AnyStr | None = suffix | ||
self.prefix: AnyStr | None = prefix | ||
self.dir: AnyStr | None = dir | ||
self.errors = errors | ||
|
||
self._async_file: AsyncFile | None = None | ||
|
||
async def __aenter__(self) -> AsyncFile[AnyStr]: | ||
fp = await to_thread.run_sync( | ||
lambda: tempfile.TemporaryFile( | ||
self.mode, | ||
self.buffering, | ||
self.encoding, | ||
self.newline, | ||
self.suffix, | ||
self.prefix, | ||
self.dir, | ||
errors=self.errors, | ||
) | ||
) | ||
self._async_file = AsyncFile(fp) | ||
return self._async_file | ||
|
||
async def __aexit__( | ||
self, | ||
exc_type: type[BaseException] | None, | ||
exc_value: BaseException | None, | ||
traceback: TracebackType | None, | ||
) -> None: | ||
if self._async_file is not None: | ||
await self._async_file.aclose() | ||
self._async_file = None | ||
|
||
|
||
class NamedTemporaryFile(Generic[AnyStr]): | ||
def __init__( | ||
self, | ||
mode: str = "w+b", | ||
buffering: int = -1, | ||
encoding: str | None = None, | ||
newline: str | None = None, | ||
suffix: AnyStr | None = None, | ||
prefix: AnyStr | None = None, | ||
dir: AnyStr | None = None, | ||
delete: bool = True, | ||
*, | ||
errors: str | None = None, | ||
delete_on_close: bool = True, | ||
) -> None: | ||
self.mode = mode | ||
self.buffering = buffering | ||
self.encoding = encoding | ||
self.newline = newline | ||
self.suffix: AnyStr | None = suffix | ||
self.prefix: AnyStr | None = prefix | ||
self.dir: AnyStr | None = dir | ||
self.delete = delete | ||
self.errors = errors | ||
self.delete_on_close = delete_on_close | ||
|
||
self._async_file: AsyncFile | None = None | ||
|
||
async def __aenter__(self) -> AsyncFile[AnyStr]: | ||
params: dict[str, Any] = { | ||
"mode": self.mode, | ||
"buffering": self.buffering, | ||
"encoding": self.encoding, | ||
"newline": self.newline, | ||
"suffix": self.suffix, | ||
"prefix": self.prefix, | ||
"dir": self.dir, | ||
"delete": self.delete, | ||
"errors": self.errors, | ||
} | ||
if sys.version_info >= (3, 12): | ||
params["delete_on_close"] = self.delete_on_close | ||
|
||
fp = await to_thread.run_sync(lambda: tempfile.NamedTemporaryFile(**params)) | ||
self._async_file = AsyncFile(fp) | ||
return self._async_file | ||
|
||
async def __aexit__( | ||
self, | ||
exc_type: type[BaseException] | None, | ||
exc_value: BaseException | None, | ||
traceback: TracebackType | None, | ||
) -> None: | ||
if self._async_file is not None: | ||
await self._async_file.aclose() | ||
self._async_file = None | ||
|
||
|
||
class SpooledTemporaryFile(Generic[AnyStr]): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ideally for SpooledTemporaryFile is that writes/reads to/from the BytesIO buffer will not go to a thread SpooledTemporaryFile needs to not go to a thread for these 'small' file things because the usecase is starlette multipart form file uploads which needs to remain fast for tiny uploads There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @Kludex would you like to participate in the review of this feature too? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @11kkw I think @graingert is referring to something like this: https://github.com/Tinche/aiofiles/blob/main/src/aiofiles/tempfile/temptypes.py#L25-L55 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've made the necessary changes based on your feedback, but I'm not entirely sure if I've implemented them correctly due to my limited experience. I'd really appreciate it if you could review my changes and provide any further suggestions. Thank you for your guidance!" There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This class needs an aclose() There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep, it needs the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why is this a generic? The There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Looks pretty generic to me? https://github.com/python/typeshed/blob/ac8f2632ec37bb4a82ade0906e6ce9bdb33883d3/stdlib/tempfile.pyi#L277 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Curious, mypy reports the type as There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. from tempfile import SpooledTemporaryFile
from anyio import SpooledTemporaryFile as AnyIOSpooledTemporaryFile
async def main() -> None:
with SpooledTemporaryFile() as f1:
reveal_type(f1)
async with AnyIOSpooledTemporaryFile() as f2:
reveal_type(f2) Gives me:
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I refactored There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm sorry for bringing this up so late. However, since the official documentation states, https://docs.python.org/3/library/tempfile.html#tempfile.mkstemp There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Whatever the reason, using There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
I agree with this change There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Based on the discussions I had on Discord, it would appear that neither mypy or pyright can handle overriding methods like |
||
def __init__( | ||
self, | ||
max_size: int = 0, | ||
mode: str = "w+b", | ||
buffering: int = -1, | ||
encoding: str | None = None, | ||
newline: str | None = None, | ||
suffix: AnyStr | None = None, | ||
prefix: AnyStr | None = None, | ||
dir: AnyStr | None = None, | ||
*, | ||
errors: str | None = None, | ||
) -> None: | ||
self.max_size = max_size | ||
self.mode = mode | ||
self.buffering = buffering | ||
self.encoding = encoding | ||
self.newline = newline | ||
self.suffix: AnyStr | None = suffix | ||
self.prefix: AnyStr | None = prefix | ||
self.dir: AnyStr | None = dir | ||
self.errors = errors | ||
|
||
self._async_file: AsyncFile | None = None | ||
|
||
async def __aenter__( | ||
self: SpooledTemporaryFile[AnyStr], | ||
) -> SpooledTemporaryFile[AnyStr]: | ||
fp = await to_thread.run_sync( | ||
partial( | ||
tempfile.SpooledTemporaryFile, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Creating the SpooledTemporyFile doesn't perform blocking disk IO so this shouldn't need a thread There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Right, you already mentioned that ^ here. |
||
self.max_size, | ||
self.mode, | ||
self.buffering, | ||
self.encoding, | ||
self.newline, | ||
self.suffix, | ||
self.prefix, | ||
self.dir, | ||
errors=self.errors, | ||
) | ||
) | ||
self._async_file = AsyncFile(fp) | ||
return self | ||
|
||
async def rollover(self) -> None: | ||
if self._async_file is None: | ||
raise RuntimeError("Internal file is not initialized.") | ||
|
||
await to_thread.run_sync(cast(Callable[[], None], self._async_file.rollover)) | ||
agronholm marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
@property | ||
def closed(self) -> bool: | ||
if self._async_file is not None: | ||
try: | ||
return bool(self._async_file.closed) | ||
except AttributeError: | ||
f = getattr(self._async_file, "_f", None) | ||
if f is None: | ||
return True | ||
return bool(f.closed) | ||
return True | ||
agronholm marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
def __getattr__(self, attr: str) -> Any: | ||
if self._async_file is not None: | ||
return getattr(self._async_file, attr) | ||
|
||
raise AttributeError(f"{self.__class__.__name__} has no attribute {attr}") | ||
|
||
async def __aexit__( | ||
self, | ||
exc_type: type[BaseException] | None, | ||
exc_value: BaseException | None, | ||
traceback: TracebackType | None, | ||
) -> None: | ||
if self._async_file is not None: | ||
await self._async_file.aclose() | ||
self._async_file = None | ||
|
||
|
||
class TemporaryDirectory(Generic[AnyStr]): | ||
def __init__( | ||
self, | ||
suffix: AnyStr | None = None, | ||
prefix: AnyStr | None = None, | ||
dir: AnyStr | None = None, | ||
*, | ||
ignore_cleanup_errors: bool = False, | ||
delete: bool = True, | ||
) -> None: | ||
self.suffix: AnyStr | None = suffix | ||
self.prefix: AnyStr | None = prefix | ||
self.dir: AnyStr | None = dir | ||
self.ignore_cleanup_errors = ignore_cleanup_errors | ||
self.delete = delete | ||
|
||
self._tempdir: tempfile.TemporaryDirectory | None = None | ||
|
||
async def __aenter__(self) -> str: | ||
params: dict[str, Any] = { | ||
"suffix": self.suffix, | ||
"prefix": self.prefix, | ||
"dir": self.dir, | ||
} | ||
if sys.version_info >= (3, 10): | ||
params["ignore_cleanup_errors"] = self.ignore_cleanup_errors | ||
|
||
if sys.version_info >= (3, 12): | ||
params["delete"] = self.delete | ||
|
||
self._tempdir = await to_thread.run_sync( | ||
lambda: tempfile.TemporaryDirectory(**params) | ||
) | ||
return await to_thread.run_sync(self._tempdir.__enter__) | ||
|
||
async def __aexit__( | ||
self, | ||
exc_type: type[BaseException] | None, | ||
exc_value: BaseException | None, | ||
traceback: TracebackType | None, | ||
) -> None: | ||
if self._tempdir is not None: | ||
await to_thread.run_sync( | ||
self._tempdir.__exit__, exc_type, exc_value, traceback | ||
) | ||
|
||
async def cleanup(self) -> None: | ||
if self._tempdir is not None: | ||
await to_thread.run_sync(self._tempdir.cleanup) | ||
|
||
|
||
async def mkstemp( | ||
suffix: AnyStr | None = None, | ||
prefix: AnyStr | None = None, | ||
dir: AnyStr | None = None, | ||
text: bool = False, | ||
) -> tuple[int, str | bytes]: | ||
return await to_thread.run_sync(lambda: tempfile.mkstemp(suffix, prefix, dir, text)) | ||
11kkw marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
|
||
async def mkdtemp( | ||
suffix: AnyStr | None = None, | ||
prefix: AnyStr | None = None, | ||
dir: AnyStr | None = None, | ||
) -> str | bytes: | ||
return await to_thread.run_sync(lambda: tempfile.mkdtemp(suffix, prefix, dir)) | ||
agronholm marked this conversation as resolved.
Show resolved
Hide resolved
11kkw marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
|
||
async def gettempdir() -> str: | ||
return await to_thread.run_sync(tempfile.gettempdir) | ||
|
||
|
||
async def gettempdirb() -> bytes: | ||
return await to_thread.run_sync(tempfile.gettempdirb) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Which open call is blocking?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The one in
test_mkstemp()
.