Compare commits
No commits in common. "d29ebeb7d69c198d6d50950ff204f7264ea1cd6c" and "73133e3f74118e80f1e0d7c9edd43779b1d5062f" have entirely different histories.
d29ebeb7d6
...
73133e3f74
2
LICENSE
2
LICENSE
@ -1,6 +1,6 @@
|
|||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2022 Daniil Fajnberg
|
Copyright (c) 2021 Daniil Fajnberg
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
@ -10,14 +10,10 @@ Makes it more convenient to run awaitable objects in concurrent batches.
|
|||||||
|
|
||||||
## Decorators
|
## Decorators
|
||||||
|
|
||||||
### @in_async_session
|
### in_async_session
|
||||||
|
|
||||||
Handles starting and closing of a temporary `aiohttp.ClientSession` instance around any async function that makes use of such an object to perform requests.
|
Handles starting and closing of a temporary `aiohttp.ClientSession` instance around any async function that makes use of such an object to perform requests.
|
||||||
|
|
||||||
### @attempt
|
|
||||||
|
|
||||||
Allows defining for an async function to be called repeatedly if it throws a specific kind of exception.
|
|
||||||
|
|
||||||
## Building
|
## Building
|
||||||
|
|
||||||
Clone this repo, install `build` via pip, then run `python -m build` from the repository's root directory. This should produce a `dist/` subdirectory with a wheel (build) and archive (source) distribution.
|
Clone this repo, install `build` via pip, then run `python -m build` from the repository's root directory. This should produce a `dist/` subdirectory with a wheel (build) and archive (source) distribution.
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
[metadata]
|
[metadata]
|
||||||
name = webutils-df
|
name = webutils-df
|
||||||
version = 0.1.1
|
version = 0.0.5
|
||||||
author = Daniil F.
|
author = Daniil F.
|
||||||
author_email = mail@placeholder123.to
|
author_email = mail@placeholder123.to
|
||||||
description = Miscellaneous web utilities
|
description = Miscellaneous web utilities
|
||||||
@ -8,7 +8,7 @@ long_description = file: README.md
|
|||||||
long_description_content_type = text/markdown
|
long_description_content_type = text/markdown
|
||||||
url = https://git.fajnberg.de/daniil/webutils-df
|
url = https://git.fajnberg.de/daniil/webutils-df
|
||||||
project_urls =
|
project_urls =
|
||||||
Bug Tracker = https://git.fajnberg.de/daniil/webutils-df/issues
|
Bug Tracker = https://github.com/daniil-berg/webutils-df/issues
|
||||||
classifiers =
|
classifiers =
|
||||||
Programming Language :: Python :: 3
|
Programming Language :: Python :: 3
|
||||||
License :: OSI Approved :: MIT License
|
License :: OSI Approved :: MIT License
|
||||||
|
@ -1,5 +1,4 @@
|
|||||||
from .util import (
|
from .util import (
|
||||||
in_async_session,
|
in_async_session,
|
||||||
attempt,
|
|
||||||
gather_in_batches
|
gather_in_batches
|
||||||
)
|
)
|
@ -2,9 +2,7 @@ import logging
|
|||||||
import asyncio
|
import asyncio
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from inspect import signature
|
from inspect import signature
|
||||||
from math import inf
|
from typing import Callable, Awaitable, Dict, Tuple, Any, TypeVar
|
||||||
from timeit import default_timer
|
|
||||||
from typing import Callable, Awaitable, Dict, Tuple, Sequence, Any, Type, Union, TypeVar
|
|
||||||
|
|
||||||
from aiohttp.client import ClientSession
|
from aiohttp.client import ClientSession
|
||||||
|
|
||||||
@ -13,7 +11,6 @@ LOGGER_NAME = 'webutils'
|
|||||||
logger = logging.getLogger(LOGGER_NAME)
|
logger = logging.getLogger(LOGGER_NAME)
|
||||||
|
|
||||||
AsyncFunction = TypeVar('AsyncFunction')
|
AsyncFunction = TypeVar('AsyncFunction')
|
||||||
AttemptCallbackT = Callable[[AsyncFunction, Exception, int, float, tuple, dict], Awaitable[None]]
|
|
||||||
|
|
||||||
|
|
||||||
def _get_param_idx_and_default(function: Callable, param_name: str) -> Tuple[int, Any]:
|
def _get_param_idx_and_default(function: Callable, param_name: str) -> Tuple[int, Any]:
|
||||||
@ -88,65 +85,6 @@ def in_async_session(_func: AsyncFunction = None, *,
|
|||||||
return decorator if _func is None else decorator(_func)
|
return decorator if _func is None else decorator(_func)
|
||||||
|
|
||||||
|
|
||||||
def attempt(_func: AsyncFunction = None, *,
|
|
||||||
exception: Union[Type[Exception], Sequence[Type[Exception]]] = Exception,
|
|
||||||
max_attempts: float = inf,
|
|
||||||
timeout_seconds: float = inf,
|
|
||||||
seconds_between: float = 0,
|
|
||||||
callback: AttemptCallbackT = None) -> AsyncFunction:
|
|
||||||
"""
|
|
||||||
Decorator allowing an async function to be called repeatedly, if previous attempts cause specific exceptions.
|
|
||||||
Note: If no limiting arguments are passed to the decorator, the decorated function **will** be called repeatedly in
|
|
||||||
a potentially infinite loop, as long as it keeps throwing an exception.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
_func:
|
|
||||||
Control parameter; allows using the decorator with or without arguments.
|
|
||||||
If this decorator is used *with any* arguments, this will always be the decorated function itself.
|
|
||||||
exception (optional):
|
|
||||||
An `Exception` (sub-)class or a sequence thereof; a failed call of the decorated function will only be
|
|
||||||
repeated if it fails with a matching exception. Defaults to `Exception`, i.e. any exception.
|
|
||||||
max_attempts (optional):
|
|
||||||
The maximum number of (re-)attempts at calling the decorated function; if it is called `max_attempts` times
|
|
||||||
and fails, the exception will be propagated. The number of attempts is unlimited by default.
|
|
||||||
timeout_seconds (optional):
|
|
||||||
Defines the cutoff time (in seconds) for the entirety of attempts at executing the decorated function;
|
|
||||||
if the attempts take longer in total and fail, the exception will be propagated. No timeout by default.
|
|
||||||
seconds_between (optional):
|
|
||||||
Sets a sleep interval (in seconds) between each attempt to call the decorated function. Defaults to 0.
|
|
||||||
callback (optional):
|
|
||||||
If passed an async function (with matching parameters), a failed **and caught** attempt will call it with
|
|
||||||
the following positional arguments (in that order):
|
|
||||||
- the decorated async function itself
|
|
||||||
- the exception class encountered and caught
|
|
||||||
- the total number of failed attempts up to that point
|
|
||||||
- the `seconds_between` argument
|
|
||||||
- positional and keyword arguments (as tuple and dictionary respectively) passed to the decorated function
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
Any exceptions that do **not** match those passed to the `exception` parameter are immediately propagated.
|
|
||||||
Those that were specified in `exception` are propagated when `max_attempts` or `timeout_seconds` are reached.
|
|
||||||
"""
|
|
||||||
def decorator(function: AsyncFunction) -> AsyncFunction:
|
|
||||||
# Using `functools.wraps` to preserve information about the actual function being decorated
|
|
||||||
# More details: https://docs.python.org/3/library/functools.html#functools.wraps
|
|
||||||
@wraps(function)
|
|
||||||
async def wrapper(*args, **kwargs) -> Any:
|
|
||||||
start, failed_attempts = default_timer(), 0
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
return await function(*args, **kwargs)
|
|
||||||
except exception as e:
|
|
||||||
failed_attempts += 1
|
|
||||||
if default_timer() - start >= timeout_seconds or failed_attempts >= max_attempts:
|
|
||||||
raise e
|
|
||||||
if callback:
|
|
||||||
await callback(function, e, failed_attempts, seconds_between, args, kwargs)
|
|
||||||
await asyncio.sleep(seconds_between)
|
|
||||||
return wrapper
|
|
||||||
return decorator if _func is None else decorator(_func)
|
|
||||||
|
|
||||||
|
|
||||||
async def gather_in_batches(batch_size: int, *aws: Awaitable, return_exceptions: bool = False) -> list:
|
async def gather_in_batches(batch_size: int, *aws: Awaitable, return_exceptions: bool = False) -> list:
|
||||||
"""
|
"""
|
||||||
Simple extension of the `asyncio.gather` function to make it easy to run awaitable objects in concurrent batches.
|
Simple extension of the `asyncio.gather` function to make it easy to run awaitable objects in concurrent batches.
|
Loading…
x
Reference in New Issue
Block a user