control interface now supports TaskPool instances:

dotted paths to coroutine functions can be passed to the parser as arguments for methods like `map`;
parser supports literal evaluation for the argument iterables in methods like `map`;
minor fixes
This commit is contained in:
Daniil Fajnberg 2022-03-16 11:27:27 +01:00
parent 3503c0bf44
commit 689a74c678
7 changed files with 82 additions and 21 deletions

View File

@ -1,6 +1,6 @@
[metadata] [metadata]
name = asyncio-taskpool name = asyncio-taskpool
version = 0.6.4 version = 0.7.0
author = Daniil Fajnberg author = Daniil Fajnberg
author_email = mail@daniil.fajnberg.de author_email = mail@daniil.fajnberg.de
description = Dynamically manage pools of asyncio tasks description = Dynamically manage pools of asyncio tasks

View File

@ -20,14 +20,16 @@ This module contains the the definition of the `ControlParser` class used by a c
from argparse import Action, ArgumentParser, ArgumentDefaultsHelpFormatter, HelpFormatter, SUPPRESS from argparse import Action, ArgumentParser, ArgumentDefaultsHelpFormatter, HelpFormatter, SUPPRESS
from ast import literal_eval
from asyncio.streams import StreamWriter from asyncio.streams import StreamWriter
from inspect import Parameter, getmembers, isfunction, signature from inspect import Parameter, getmembers, isfunction, signature
from shutil import get_terminal_size from shutil import get_terminal_size
from typing import Any, Callable, Container, Dict, Set, Type, TypeVar from typing import Any, Callable, Container, Dict, Iterable, Set, Type, TypeVar
from ..constants import CLIENT_INFO, CMD, STREAM_WRITER from ..constants import CLIENT_INFO, CMD, STREAM_WRITER
from ..exceptions import HelpRequested, ParserError from ..exceptions import HelpRequested, ParserError
from ..helpers import get_first_doc_line from ..helpers import get_first_doc_line, resolve_dotted_path
from ..types import ArgsT, CancelCB, CoroutineFunc, EndCB, KwArgsT
FmtCls = TypeVar('FmtCls', bound=Type[HelpFormatter]) FmtCls = TypeVar('FmtCls', bound=Type[HelpFormatter])
@ -266,7 +268,7 @@ class ControlParser(ArgumentParser):
kwargs.setdefault('nargs', '*') kwargs.setdefault('nargs', '*')
if not kwargs.get('action') == 'store_true': if not kwargs.get('action') == 'store_true':
# Set the type from the parameter annotation. # Set the type from the parameter annotation.
kwargs.setdefault('type', _get_arg_type_wrapper(parameter.annotation)) kwargs.setdefault('type', _get_type_from_annotation(parameter.annotation))
return self.add_argument(*name_or_flags, **kwargs) return self.add_argument(*name_or_flags, **kwargs)
def add_function_args(self, function: Callable, omit: Container[str] = OMIT_PARAMS_DEFAULT) -> None: def add_function_args(self, function: Callable, omit: Container[str] = OMIT_PARAMS_DEFAULT) -> None:
@ -300,3 +302,11 @@ def _get_arg_type_wrapper(cls: Type) -> Callable[[Any], Any]:
# Copy the name of the class to maintain useful help messages when incorrect arguments are passed. # Copy the name of the class to maintain useful help messages when incorrect arguments are passed.
wrapper.__name__ = cls.__name__ wrapper.__name__ = cls.__name__
return wrapper return wrapper
def _get_type_from_annotation(annotation: Type) -> Callable[[Any], Any]:
if any(annotation is t for t in {CoroutineFunc, EndCB, CancelCB}):
annotation = resolve_dotted_path
if any(annotation is t for t in {ArgsT, KwArgsT, Iterable[ArgsT], Iterable[KwArgsT]}):
annotation = literal_eval
return _get_arg_type_wrapper(annotation)

View File

@ -37,7 +37,7 @@ from .session import ControlSession
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class ControlServer(ABC): # TODO: Implement interface for normal TaskPool instances, not just SimpleTaskPool class ControlServer(ABC):
""" """
Abstract base class for a task pool control server. Abstract base class for a task pool control server.

View File

@ -21,6 +21,7 @@ Miscellaneous helper functions. None of these should be considered part of the p
from asyncio.coroutines import iscoroutinefunction from asyncio.coroutines import iscoroutinefunction
from asyncio.queues import Queue from asyncio.queues import Queue
from importlib import import_module
from inspect import getdoc from inspect import getdoc
from typing import Any, Optional, Union from typing import Any, Optional, Union
@ -63,3 +64,22 @@ async def return_or_exception(_function_to_execute: AnyCallableT, *args, **kwarg
return _function_to_execute(*args, **kwargs) return _function_to_execute(*args, **kwargs)
except Exception as e: except Exception as e:
return e return e
def resolve_dotted_path(dotted_path: str) -> object:
"""
Resolves a dotted path to a global object and returns that object.
Algorithm shamelessly stolen from the `logging.config` module from the standard library.
"""
names = dotted_path.split('.')
module_name = names.pop(0)
found = import_module(module_name)
for name in names:
try:
found = getattr(found, name)
except AttributeError:
module_name += f'.{name}'
import_module(module_name)
found = getattr(found, name)
return found

View File

@ -20,12 +20,16 @@ Unittests for the `asyncio_taskpool.control.parser` module.
from argparse import ArgumentParser, HelpFormatter, ArgumentDefaultsHelpFormatter, RawTextHelpFormatter, SUPPRESS from argparse import ArgumentParser, HelpFormatter, ArgumentDefaultsHelpFormatter, RawTextHelpFormatter, SUPPRESS
from ast import literal_eval
from inspect import signature from inspect import signature
from unittest import TestCase from unittest import TestCase
from unittest.mock import MagicMock, call, patch from unittest.mock import MagicMock, call, patch
from typing import Iterable
from asyncio_taskpool.control import parser from asyncio_taskpool.control import parser
from asyncio_taskpool.exceptions import HelpRequested, ParserError from asyncio_taskpool.exceptions import HelpRequested, ParserError
from asyncio_taskpool.helpers import resolve_dotted_path
from asyncio_taskpool.types import ArgsT, CancelCB, CoroutineFunc, EndCB, KwArgsT
FOO, BAR = 'foo', 'bar' FOO, BAR = 'foo', 'bar'
@ -194,11 +198,11 @@ class ControlServerTestCase(TestCase):
self.parser.print_help(arg) self.parser.print_help(arg)
mock_print_help.assert_called_once_with(arg) mock_print_help.assert_called_once_with(arg)
@patch.object(parser, '_get_arg_type_wrapper') @patch.object(parser, '_get_type_from_annotation')
@patch.object(parser.ArgumentParser, 'add_argument') @patch.object(parser.ArgumentParser, 'add_argument')
def test_add_function_arg(self, mock_add_argument: MagicMock, mock__get_arg_type_wrapper: MagicMock): def test_add_function_arg(self, mock_add_argument: MagicMock, mock__get_type_from_annotation: MagicMock):
mock_add_argument.return_value = expected_output = 'action' mock_add_argument.return_value = expected_output = 'action'
mock__get_arg_type_wrapper.return_value = mock_type = 'fake' mock__get_type_from_annotation.return_value = mock_type = 'fake'
foo_type, args_type, bar_type, baz_type, boo_type = tuple, str, int, float, complex foo_type, args_type, bar_type, baz_type, boo_type = tuple, str, int, float, complex
bar_default, baz_default, boo_default = 1, 0.1, 1j bar_default, baz_default, boo_default = 1, 0.1, 1j
@ -211,42 +215,42 @@ class ControlServerTestCase(TestCase):
kwargs = {FOO + BAR: 'xyz'} kwargs = {FOO + BAR: 'xyz'}
self.assertEqual(expected_output, self.parser.add_function_arg(param_foo, **kwargs)) self.assertEqual(expected_output, self.parser.add_function_arg(param_foo, **kwargs))
mock_add_argument.assert_called_once_with('foo', type=mock_type, **kwargs) mock_add_argument.assert_called_once_with('foo', type=mock_type, **kwargs)
mock__get_arg_type_wrapper.assert_called_once_with(foo_type) mock__get_type_from_annotation.assert_called_once_with(foo_type)
mock_add_argument.reset_mock() mock_add_argument.reset_mock()
mock__get_arg_type_wrapper.reset_mock() mock__get_type_from_annotation.reset_mock()
self.assertEqual(expected_output, self.parser.add_function_arg(param_args, **kwargs)) self.assertEqual(expected_output, self.parser.add_function_arg(param_args, **kwargs))
mock_add_argument.assert_called_once_with('args', nargs='*', type=mock_type, **kwargs) mock_add_argument.assert_called_once_with('args', nargs='*', type=mock_type, **kwargs)
mock__get_arg_type_wrapper.assert_called_once_with(args_type) mock__get_type_from_annotation.assert_called_once_with(args_type)
mock_add_argument.reset_mock() mock_add_argument.reset_mock()
mock__get_arg_type_wrapper.reset_mock() mock__get_type_from_annotation.reset_mock()
self.assertEqual(expected_output, self.parser.add_function_arg(param_bar, **kwargs)) self.assertEqual(expected_output, self.parser.add_function_arg(param_bar, **kwargs))
mock_add_argument.assert_called_once_with('-b', '--bar', default=bar_default, type=mock_type, **kwargs) mock_add_argument.assert_called_once_with('-b', '--bar', default=bar_default, type=mock_type, **kwargs)
mock__get_arg_type_wrapper.assert_called_once_with(bar_type) mock__get_type_from_annotation.assert_called_once_with(bar_type)
mock_add_argument.reset_mock() mock_add_argument.reset_mock()
mock__get_arg_type_wrapper.reset_mock() mock__get_type_from_annotation.reset_mock()
self.assertEqual(expected_output, self.parser.add_function_arg(param_baz, **kwargs)) self.assertEqual(expected_output, self.parser.add_function_arg(param_baz, **kwargs))
mock_add_argument.assert_called_once_with('-B', '--baz', default=baz_default, type=mock_type, **kwargs) mock_add_argument.assert_called_once_with('-B', '--baz', default=baz_default, type=mock_type, **kwargs)
mock__get_arg_type_wrapper.assert_called_once_with(baz_type) mock__get_type_from_annotation.assert_called_once_with(baz_type)
mock_add_argument.reset_mock() mock_add_argument.reset_mock()
mock__get_arg_type_wrapper.reset_mock() mock__get_type_from_annotation.reset_mock()
self.assertEqual(expected_output, self.parser.add_function_arg(param_boo, **kwargs)) self.assertEqual(expected_output, self.parser.add_function_arg(param_boo, **kwargs))
mock_add_argument.assert_called_once_with('--boo', default=boo_default, type=mock_type, **kwargs) mock_add_argument.assert_called_once_with('--boo', default=boo_default, type=mock_type, **kwargs)
mock__get_arg_type_wrapper.assert_called_once_with(boo_type) mock__get_type_from_annotation.assert_called_once_with(boo_type)
mock_add_argument.reset_mock() mock_add_argument.reset_mock()
mock__get_arg_type_wrapper.reset_mock() mock__get_type_from_annotation.reset_mock()
self.assertEqual(expected_output, self.parser.add_function_arg(param_flag, **kwargs)) self.assertEqual(expected_output, self.parser.add_function_arg(param_flag, **kwargs))
mock_add_argument.assert_called_once_with('-f', '--flag', action='store_true', **kwargs) mock_add_argument.assert_called_once_with('-f', '--flag', action='store_true', **kwargs)
mock__get_arg_type_wrapper.assert_not_called() mock__get_type_from_annotation.assert_not_called()
@patch.object(parser.ControlParser, 'add_function_arg') @patch.object(parser.ControlParser, 'add_function_arg')
def test_add_function_args(self, mock_add_function_arg: MagicMock): def test_add_function_args(self, mock_add_function_arg: MagicMock):
@ -266,3 +270,20 @@ class RestTestCase(TestCase):
self.assertEqual('int', type_wrap.__name__) self.assertEqual('int', type_wrap.__name__)
self.assertEqual(SUPPRESS, type_wrap(SUPPRESS)) self.assertEqual(SUPPRESS, type_wrap(SUPPRESS))
self.assertEqual(13, type_wrap('13')) self.assertEqual(13, type_wrap('13'))
@patch.object(parser, '_get_arg_type_wrapper')
def test__get_type_from_annotation(self, mock__get_arg_type_wrapper: MagicMock):
mock__get_arg_type_wrapper.return_value = expected_output = FOO + BAR
dotted_path_ann = [CoroutineFunc, EndCB, CancelCB]
literal_eval_ann = [ArgsT, KwArgsT, Iterable[ArgsT], Iterable[KwArgsT]]
any_other_ann = MagicMock()
for a in dotted_path_ann:
self.assertEqual(expected_output, parser._get_type_from_annotation(a))
mock__get_arg_type_wrapper.assert_has_calls(len(dotted_path_ann) * [call(resolve_dotted_path)])
mock__get_arg_type_wrapper.reset_mock()
for a in literal_eval_ann:
self.assertEqual(expected_output, parser._get_type_from_annotation(a))
mock__get_arg_type_wrapper.assert_has_calls(len(literal_eval_ann) * [call(literal_eval)])
mock__get_arg_type_wrapper.reset_mock()
self.assertEqual(expected_output, parser._get_type_from_annotation(any_other_ann))
mock__get_arg_type_wrapper.assert_called_once_with(any_other_ann)

View File

@ -20,7 +20,7 @@ Unittests for the `asyncio_taskpool.helpers` module.
from unittest import IsolatedAsyncioTestCase from unittest import IsolatedAsyncioTestCase
from unittest.mock import MagicMock, AsyncMock, NonCallableMagicMock from unittest.mock import MagicMock, AsyncMock, NonCallableMagicMock, call, patch
from asyncio_taskpool import helpers from asyncio_taskpool import helpers
@ -118,3 +118,13 @@ class HelpersTestCase(IsolatedAsyncioTestCase):
output = await helpers.return_or_exception(mock_func, *args, **kwargs) output = await helpers.return_or_exception(mock_func, *args, **kwargs)
self.assertEqual(test_exception, output) self.assertEqual(test_exception, output)
mock_func.assert_called_once_with(*args, **kwargs) mock_func.assert_called_once_with(*args, **kwargs)
def test_resolve_dotted_path(self):
from logging import WARNING
from urllib.request import urlopen
self.assertEqual(WARNING, helpers.resolve_dotted_path('logging.WARNING'))
self.assertEqual(urlopen, helpers.resolve_dotted_path('urllib.request.urlopen'))
with patch.object(helpers, 'import_module', return_value=object) as mock_import_module:
with self.assertRaises(AttributeError):
helpers.resolve_dotted_path('foo.bar.baz')
mock_import_module.assert_has_calls([call('foo'), call('foo.bar')])

View File

@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along w
If not, see <https://www.gnu.org/licenses/>.""" If not, see <https://www.gnu.org/licenses/>."""
__doc__ = """ __doc__ = """
Working example of a UnixControlServer in combination with the SimpleTaskPool. Working example of a TCPControlServer in combination with the SimpleTaskPool.
Use the main CLI client to interface at the socket. Use the main CLI client to interface at the socket.
""" """