paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

first draft of enhancers.

Browse Source
This commit is contained in:
Romain Dorgueil
2017-05-08 11:33:02 +02:00
parent 6cc3cedf1a
commit 171fa3415b
16 changed files with 197 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bonobo/core/statistics.py
View File

@ -24,7 +24,8 @@ class WithStatistics:
return ((name, self.statistics[name]) for name in self.statistics_names)
def get_statistics_as_string(self, *args, **kwargs):
return ' '.join(('{0}={1}'.format(name, cnt) for name, cnt in self.get_statistics(*args, **kwargs) if cnt > 0))
stats = tuple('{0}={1}'.format(name, cnt) for name, cnt in self.get_statistics(*args, **kwargs) if cnt > 0)
return (kwargs.get('prefix', '') + ' '.join(stats)) if len(stats) else ''
def increment(self, name):
self.statistics[name] += 1

13
bonobo/examples/__init__.py
View File

@ -0,0 +1,13 @@
def require(package, requirement=None):
requirement = requirement or package
try:
return __import__(package)
except ImportError:
from colorama import Fore, Style
print(Fore.YELLOW, 'This example requires the {!r} package. Install it using:'.format(requirement),
Style.RESET_ALL, sep='')
print()
print(Fore.YELLOW, ' $ pip install {!s}'.format(requirement), Style.RESET_ALL, sep='')
print()
raise

52
bonobo/execution/base.py
View File

@ -1,21 +1,27 @@
import traceback
from contextlib import contextmanager
from time import sleep
from bonobo.config import Container
from bonobo.config.processors import resolve_processors, ContextCurrifier
from bonobo.config.processors import ContextCurrifier
from bonobo.plugins import get_enhancers
from bonobo.util.errors import print_error
from bonobo.util.iterators import ensure_tuple
from bonobo.util.objects import Wrapper
from bonobo.util.objects import Wrapper, get_name
@contextmanager
def unrecoverable(error_handler):
try:
yield
except Exception as exc: # pylint: disable=broad-except
error_handler(exc, traceback.format_exc())
raise # raise unrecoverableerror from x ?
class LoopingExecutionContext(Wrapper):
alive = True
PERIOD = 0.25
@property
def state(self):
return self._started, self._stopped
@property
def started(self):
return self._started
@ -26,7 +32,9 @@ class LoopingExecutionContext(Wrapper):
def __init__(self, wrapped, parent, services=None):
super().__init__(wrapped)
self.parent = parent
if services:
if parent:
raise RuntimeError(
@ -36,19 +44,25 @@ class LoopingExecutionContext(Wrapper):
else:
self.services = None
self._started, self._stopped, self._stack = False, False, None
self._started, self._stopped = False, False
self._stack = None
# XXX enhancers
self._enhancers = get_enhancers(self.wrapped)
def start(self):
assert self.state == (False,
False), ('{}.start() can only be called on a new node.').format(type(self).__name__)
if self.started:
raise RuntimeError('Cannot start a node twice ({}).'.format(get_name(self)))
self._started = True
self._stack = ContextCurrifier(self.wrapped, *self._get_initial_context())
try:
with unrecoverable(self.handle_error):
self._stack.setup(self)
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
raise
for enhancer in self._enhancers:
with unrecoverable(self.handle_error):
enhancer.start(self)
def loop(self):
"""Generic loop. A bit boring. """
@ -61,16 +75,16 @@ class LoopingExecutionContext(Wrapper):
raise NotImplementedError('Abstract.')
def stop(self):
assert self._started, ('{}.stop() can only be called on a previously started node.').format(type(self).__name__)
if not self.started:
raise RuntimeError('Cannot stop an unstarted node ({}).'.format(get_name(self)))
if self._stopped:
return
self._stopped = True
try:
with unrecoverable(self.handle_error):
self._stack.teardown()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
raise
def handle_error(self, exc, trace):
return print_error(exc, trace, context=self.wrapped)

4
bonobo/execution/graph.py
View File

@ -50,7 +50,7 @@ class GraphExecutionContext:
for i in self.graph.outputs_of(BEGIN):
for message in messages:
self[i].recv(message)
self[i].write(message)
def start(self):
# todo use strategy
@ -65,4 +65,4 @@ class GraphExecutionContext:
def stop(self):
# todo use strategy
for node in self.nodes:
node.stop()
node.stop()

47
bonobo/execution/node.py
View File

@ -8,8 +8,10 @@ from bonobo.core.statistics import WithStatistics
from bonobo.errors import InactiveReadableError
from bonobo.execution.base import LoopingExecutionContext
from bonobo.structs.bags import Bag
from bonobo.util.compat import deprecated_alias
from bonobo.util.errors import is_error
from bonobo.util.iterators import iter_if_not_sequence
from bonobo.util.objects import get_name
class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
@ -22,6 +24,10 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
"""todo check if this is right, and where it is used"""
return self.input.alive and self._started and not self._stopped
@property
def alive_str(self):
return '+' if self.alive else '-'
def __init__(self, wrapped, parent=None, services=None):
LoopingExecutionContext.__init__(self, wrapped, parent=parent, services=services)
WithStatistics.__init__(self, 'in', 'out', 'err')
@ -30,18 +36,13 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
self.outputs = []
def __str__(self):
return (('+' if self.alive else '-') + ' ' + self.__name__ + ' ' + self.get_statistics_as_string()).strip()
return self.alive_str + ' ' + self.__name__ + self.get_statistics_as_string(prefix=' ')
def __repr__(self):
stats = self.get_statistics_as_string().strip()
return '<{}({}{}){}>'.format(
type(self).__name__,
'+' if self.alive else '',
self.__name__,
(' ' + stats) if stats else '',
)
name, type_name = get_name(self), get_name(type(self))
return '<{}({}{}){}>'.format(type_name, self.alive_str, name, self.get_statistics_as_string(prefix=' '))
def recv(self, *messages):
def write(self, *messages): # XXX write() ? ( node.write(...) )
"""
Push a message list to this context's input queue.
@ -50,19 +51,29 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
for message in messages:
self.input.put(message)
def send(self, value, _control=False):
# XXX deprecated alias
recv = deprecated_alias('recv', write)
def send(self, value, _control=False): # XXX self.send(....)
"""
Sends a message to all of this context's outputs.
:param mixed value: message
:param _control: if true, won't count in statistics.
"""
if not _control:
self.increment('out')
for output in self.outputs:
output.put(value)
def get(self):
if is_error(value):
value.apply(self.handle_error)
else:
for output in self.outputs:
output.put(value)
push = deprecated_alias('push', send)
def get(self): # recv() ? input_data = self.receive()
"""
Get from the queue first, then increment stats, so if Queue raise Timeout or Empty, stat won't be changed.
@ -95,12 +106,6 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
# todo add timer
self.handle_results(input_bag, input_bag.apply(self._stack))
def push(self, bag):
# MAKE THIS PUBLIC API FOR CONTEXT PROCESSORS !!!
# xxx handle error or send in first call to apply(...)?
# xxx return value ?
bag.apply(self.handle_error) if is_error(bag) else self.send(bag)
def handle_results(self, input_bag, results):
# self._exec_time += timer.duration
# Put data onto output channels
@ -108,7 +113,7 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
results = iter_if_not_sequence(results)
except TypeError: # not an iterator
if results:
self.push(_resolve(input_bag, results))
self.send(_resolve(input_bag, results))
else:
# case with no result, an execution went through anyway, use for stats.
# self._exec_count += 1
@ -120,7 +125,7 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
except StopIteration:
break
else:
self.push(_resolve(input_bag, result))
self.send(_resolve(input_bag, result))
def _resolve(input_bag, output):

2
bonobo/execution/plugin.py
View File

@ -31,4 +31,4 @@ class PluginExecutionContext(LoopingExecutionContext):
try:
self.wrapped.run()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
self.handle_error(exc, traceback.format_exc())

17
bonobo/plugins.py
View File

@ -1,3 +1,7 @@
from bonobo.config import Configurable
from bonobo.util.objects import get_attribute_or_create
class Plugin:
"""
A plugin is an extension to the core behavior of bonobo. If you're writing transformations, you should not need
@ -21,3 +25,16 @@ class Plugin:
def finalize(self):
pass
def get_enhancers(obj):
try:
return get_attribute_or_create(obj, '__enhancers__', list())
except AttributeError:
return list()
class NodeEnhancer(Configurable):
def __matmul__(self, other):
get_enhancers(other).append(self)
return other

9
bonobo/structs/bags.py
View File

@ -43,7 +43,7 @@ class Bag:
def args(self):
if self._parent is None:
return self._args
return (*self._parent.args, *self._args, )
return (*self._parent.args, *self._args,)
@property
def kwargs(self):
@ -85,12 +85,15 @@ class Bag:
@classmethod
def inherit(cls, *args, **kwargs):
return cls(*args, _flags=(INHERIT_INPUT, ), **kwargs)
return cls(*args, _flags=(INHERIT_INPUT,), **kwargs)
def __eq__(self, other):
return isinstance(other, Bag) and other.args == self.args and other.kwargs == self.kwargs
def __repr__(self):
return '<{} ({})>'.format(
type(self).__name__, ', '.
join(itertools.chain(
join(itertools.chain(
map(repr, self.args),
('{}={}'.format(k, repr(v)) for k, v in self.kwargs.items()),
))

5
bonobo/util/helpers.py
View File

@ -1,3 +1,7 @@
from bonobo.util.compat import deprecated
@deprecated
def console_run(*chain, output=True, plugins=None, strategy=None):
from bonobo import run
from bonobo.ext.console import ConsoleOutputPlugin
@ -5,6 +9,7 @@ def console_run(*chain, output=True, plugins=None, strategy=None):
return run(*chain, plugins=(plugins or []) + [ConsoleOutputPlugin()] if output else [], strategy=strategy)
@deprecated
def jupyter_run(*chain, plugins=None, strategy=None):
from bonobo import run
from bonobo.ext.jupyter import JupyterOutputPlugin

9
bonobo/util/objects.py
View File

@ -199,3 +199,12 @@ class ValueHolder:
def __invert__(self):
return ~self.value
def get_attribute_or_create(obj, attr, default):
try:
return getattr(obj, attr)
except AttributeError:
setattr(obj, attr, default)
return getattr(obj, attr)