From c09c1010745535fecb916b149dc84043e7616e42 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 2 Oct 2017 08:38:31 +0200 Subject: [PATCH] Making config/util/structs apis available at level 2 import (x.y), implements the roots for loopbacks (recursive transformations). This still needs work, as its hard not to close an input queue as soon as the last item was read. --- bonobo/_api.py | 4 +-- bonobo/constants.py | 1 + bonobo/execution/node.py | 10 ++++--- bonobo/structs/__init__.py | 10 +++++-- bonobo/structs/bags.py | 16 +++++++---- bonobo/util/__init__.py | 24 +++++++++++++++- bonobo/util/errors.py | 7 ----- bonobo/util/inspect.py | 56 ++++++++++++++++++++++++++++++-------- 8 files changed, 96 insertions(+), 32 deletions(-) diff --git a/bonobo/_api.py b/bonobo/_api.py index 1b7e424..de75bd2 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -1,6 +1,6 @@ import logging -from bonobo.structs import Bag, Graph, Token +from bonobo.structs import Bag, ErrorBag, Graph, Token from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \ PickleReader, PickleWriter, PrettyPrinter, RateLimited, Tee, arg0_to_kwargs, count, identity, kwargs_to_arg0, noop from bonobo.strategies import create_strategy @@ -70,7 +70,7 @@ def run(graph, strategy=None, plugins=None, services=None): # bonobo.structs -register_api_group(Bag, Graph, Token) +register_api_group(Bag, ErrorBag, Graph, Token) # bonobo.strategies register_api(create_strategy) diff --git a/bonobo/constants.py b/bonobo/constants.py index 4187197..4a02f5e 100644 --- a/bonobo/constants.py +++ b/bonobo/constants.py @@ -3,6 +3,7 @@ from bonobo.structs.tokens import Token BEGIN = Token('Begin') END = Token('End') INHERIT_INPUT = Token('InheritInput') +LOOPBACK = Token('Loopback') NOT_MODIFIED = Token('NotModified') DEFAULT_SERVICES_FILENAME = '_services.py' DEFAULT_SERVICES_ATTR = 'get_services' \ No newline at end of file diff --git a/bonobo/execution/node.py b/bonobo/execution/node.py index 45691a6..6f83f04 100644 --- a/bonobo/execution/node.py +++ b/bonobo/execution/node.py @@ -2,13 +2,13 @@ import traceback from queue import Empty from time import sleep -from bonobo.constants import INHERIT_INPUT, NOT_MODIFIED +from bonobo.constants import INHERIT_INPUT, NOT_MODIFIED, BEGIN, END from bonobo.errors import InactiveReadableError, UnrecoverableError from bonobo.execution.base import LoopingExecutionContext from bonobo.structs.bags import Bag from bonobo.structs.inputs import Input from bonobo.util.compat import deprecated_alias -from bonobo.util.errors import is_error +from bonobo.util.inspect import iserrorbag, isloopbackbag from bonobo.util.iterators import iter_if_not_sequence from bonobo.util.objects import get_name from bonobo.util.statistics import WithStatistics @@ -65,8 +65,10 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext): if not _control: self.increment('out') - if is_error(value): + if iserrorbag(value): value.apply(self.handle_error) + elif isloopbackbag(value): + self.input.put(value) else: for output in self.outputs: output.put(value) @@ -137,7 +139,7 @@ def _resolve(input_bag, output): if output is NOT_MODIFIED: return input_bag - if is_error(output): + if iserrorbag(output): return output # If it does not look like a bag, let's create one for easier manipulation diff --git a/bonobo/structs/__init__.py b/bonobo/structs/__init__.py index 15e76a9..678cea1 100644 --- a/bonobo/structs/__init__.py +++ b/bonobo/structs/__init__.py @@ -1,5 +1,11 @@ -from bonobo.structs.bags import Bag +from bonobo.structs.bags import Bag, ErrorBag, LoopbackBag from bonobo.structs.graphs import Graph from bonobo.structs.tokens import Token -__all__ = ['Bag', 'Graph', 'Token'] +__all__ = [ + 'Bag', + 'ErrorBag', + 'Graph', + 'LoopbackBag', + 'Token', +] diff --git a/bonobo/structs/bags.py b/bonobo/structs/bags.py index 4ef2fa7..0c91274 100644 --- a/bonobo/structs/bags.py +++ b/bonobo/structs/bags.py @@ -1,6 +1,6 @@ import itertools -from bonobo.constants import INHERIT_INPUT +from bonobo.constants import INHERIT_INPUT, LOOPBACK __all__ = [ 'Bag', @@ -33,8 +33,10 @@ class Bag: """ + default_flags = () + def __init__(self, *args, _flags=None, _parent=None, **kwargs): - self._flags = _flags or () + self._flags = type(self).default_flags + (_flags or ()) self._parent = _parent self._args = args self._kwargs = kwargs @@ -43,7 +45,7 @@ class Bag: def args(self): if self._parent is None: return self._args - return (*self._parent.args, *self._args, ) + return (*self._parent.args, *self._args,) @property def kwargs(self): @@ -91,7 +93,7 @@ class Bag: @classmethod def inherit(cls, *args, **kwargs): - return cls(*args, _flags=(INHERIT_INPUT, ), **kwargs) + return cls(*args, _flags=(INHERIT_INPUT,), **kwargs) def __eq__(self, other): return isinstance(other, Bag) and other.args == self.args and other.kwargs == self.kwargs @@ -99,12 +101,16 @@ class Bag: def __repr__(self): return '<{} ({})>'.format( type(self).__name__, ', '. - join(itertools.chain( + join(itertools.chain( map(repr, self.args), ('{}={}'.format(k, repr(v)) for k, v in self.kwargs.items()), )) ) +class LoopbackBag(Bag): + default_flags = (LOOPBACK,) + + class ErrorBag(Bag): pass diff --git a/bonobo/util/__init__.py b/bonobo/util/__init__.py index f4b4158..4a5e8dc 100644 --- a/bonobo/util/__init__.py +++ b/bonobo/util/__init__.py @@ -1,6 +1,28 @@ +from bonobo.util.inspect import ( + inspect_node, + isbag, + isconfigurable, + isconfigurabletype, + iscontextprocessor, + iserrorbag, + isloopbackbag, + ismethod, + isoption, + istype, +) from bonobo.util.python import require # Bonobo's util API __all__ = [ - 'require' + 'require', + 'inspect_node', + 'isbag', + 'isconfigurable', + 'isconfigurabletype', + 'iscontextprocessor', + 'iserrorbag', + 'isloopbackbag', + 'ismethod', + 'isoption', + 'istype', ] diff --git a/bonobo/util/errors.py b/bonobo/util/errors.py index 0ea4e58..cae2789 100644 --- a/bonobo/util/errors.py +++ b/bonobo/util/errors.py @@ -1,13 +1,6 @@ import sys from textwrap import indent -from bonobo import settings -from bonobo.structs.bags import ErrorBag - - -def is_error(bag): - return isinstance(bag, ErrorBag) - def _get_error_message(exc): if hasattr(exc, '__str__'): diff --git a/bonobo/util/inspect.py b/bonobo/util/inspect.py index 42da6ed..1594d1e 100644 --- a/bonobo/util/inspect.py +++ b/bonobo/util/inspect.py @@ -1,5 +1,18 @@ from collections import namedtuple +from bonobo.constants import LOOPBACK + + +def isconfigurable(mixed): + """ + Check if the given argument is an instance of :class:`bonobo.config.Configurable`. + + :param mixed: + :return: bool + """ + from bonobo.config.configurables import Configurable + return isinstance(mixed, Configurable) + def isconfigurabletype(mixed): """ @@ -13,17 +26,6 @@ def isconfigurabletype(mixed): return isinstance(mixed, ConfigurableMeta) -def isconfigurable(mixed): - """ - Check if the given argument is an instance of :class:`bonobo.config.Configurable`. - - :param mixed: - :return: bool - """ - from bonobo.config.configurables import Configurable - return isinstance(mixed, Configurable) - - def isoption(mixed): """ Check if the given argument is an instance of :class:`bonobo.config.Option`. @@ -68,6 +70,38 @@ def istype(mixed): return isinstance(mixed, type) +def isbag(mixed): + """ + Check if the given argument is an instance of a :class:`bonobo.Bag`. + + :param mixed: + :return: bool + """ + from bonobo.structs.bags import Bag + return isinstance(mixed, Bag) + + +def iserrorbag(mixed): + """ + Check if the given argument is an instance of an :class:`bonobo.ErrorBag`. + + :param mixed: + :return: bool + """ + from bonobo.structs.bags import ErrorBag + return isinstance(mixed, ErrorBag) + + +def isloopbackbag(mixed): + """ + Check if the given argument is an instance of a :class:`bonobo.Bag`, marked for loopback behaviour. + + :param mixed: + :return: bool + """ + return isbag(mixed) and LOOPBACK in mixed.flags + + ConfigurableInspection = namedtuple( 'ConfigurableInspection', [ 'type',