From 474999a87e6b3cf8033cc58b3df70434e6bf1911 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 15:12:48 +0200 Subject: [PATCH 1/3] Trying to fix unending transformations on start() error. --- .gitignore | 1 + bonobo/execution/base.py | 70 ++++++++++++----------------------- bonobo/execution/graph.py | 10 ++--- bonobo/execution/node.py | 15 +++++--- bonobo/strategies/executor.py | 28 ++++++++++---- bonobo/util/errors.py | 30 +++++++++++++++ 6 files changed, 90 insertions(+), 64 deletions(-) create mode 100644 bonobo/util/errors.py diff --git a/.gitignore b/.gitignore index 179ed2e..a88bd7b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ /.idea /.release /bonobo.iml +/bonobo/examples/work_in_progress/ /bonobo/ext/jupyter/js/node_modules/ /build/ /coverage.xml diff --git a/bonobo/execution/base.py b/bonobo/execution/base.py index 9a96cb1..e2dfb55 100644 --- a/bonobo/execution/base.py +++ b/bonobo/execution/base.py @@ -1,9 +1,9 @@ -import sys import traceback from time import sleep from bonobo.config import Container from bonobo.config.processors import resolve_processors +from bonobo.util.errors import print_error from bonobo.util.iterators import ensure_tuple from bonobo.util.objects import Wrapper @@ -43,16 +43,13 @@ class LoopingExecutionContext(Wrapper): False), ('{}.start() can only be called on a new node.').format(type(self).__name__) assert self._context is None self._started = True - try: - if self.parent: - self._context = self.parent.services.args_for(self.wrapped) - elif self.services: - self._context = self.services.args_for(self.wrapped) - else: - self._context = () - except Exception as exc: # pylint: disable=broad-except - self.handle_error(exc, traceback.format_exc()) - raise + + if self.parent: + self._context = self.parent.services.args_for(self.wrapped) + elif self.services: + self._context = self.services.args_for(self.wrapped) + else: + self._context = () for processor in resolve_processors(self.wrapped): try: @@ -80,41 +77,22 @@ class LoopingExecutionContext(Wrapper): if self._stopped: return - assert self._context is not None - self._stopped = True - while len(self._stack): - processor = self._stack.pop() - try: - # todo yield from ? how to ? - next(processor) - except StopIteration as exc: - # This is normal, and wanted. - pass - except Exception as exc: # pylint: disable=broad-except - self.handle_error(exc, traceback.format_exc()) - raise - else: - # No error ? We should have had StopIteration ... - raise RuntimeError('Context processors should not yield more than once.') + if self._context is not None: + while len(self._stack): + processor = self._stack.pop() + try: + # todo yield from ? how to ? + next(processor) + except StopIteration as exc: + # This is normal, and wanted. + pass + except Exception as exc: # pylint: disable=broad-except + self.handle_error(exc, traceback.format_exc()) + raise + else: + # No error ? We should have had StopIteration ... + raise RuntimeError('Context processors should not yield more than once.') def handle_error(self, exc, trace): - """ - Error handler. Whatever happens in a plugin or component, if it looks like an exception, taste like an exception - or somehow make me think it is an exception, I'll handle it. - - :param exc: the culprit - :param trace: Hercule Poirot's logbook. - :return: to hell - """ - - from colorama import Fore, Style - print( - Style.BRIGHT, - Fore.RED, - '\U0001F4A3 {} in {}'.format(type(exc).__name__, self.wrapped), - Style.RESET_ALL, - sep='', - file=sys.stderr, - ) - print(trace) + return print_error(exc, trace, context=self.wrapped) diff --git a/bonobo/execution/graph.py b/bonobo/execution/graph.py index d5e241f..6f95ac3 100644 --- a/bonobo/execution/graph.py +++ b/bonobo/execution/graph.py @@ -25,15 +25,15 @@ class GraphExecutionContext: self.plugins = [PluginExecutionContext(plugin, parent=self) for plugin in plugins or ()] self.services = Container(services) if services else Container() - for i, component_context in enumerate(self): + for i, node_context in enumerate(self): try: - component_context.outputs = [self[j].input for j in self.graph.outputs_of(i)] + node_context.outputs = [self[j].input for j in self.graph.outputs_of(i)] except KeyError: continue - component_context.input.on_begin = partial(component_context.send, BEGIN, _control=True) - component_context.input.on_end = partial(component_context.send, END, _control=True) - component_context.input.on_finalize = partial(component_context.stop) + node_context.input.on_begin = partial(node_context.send, BEGIN, _control=True) + node_context.input.on_end = partial(node_context.send, END, _control=True) + node_context.input.on_finalize = partial(node_context.stop) def __getitem__(self, item): return self.nodes[item] diff --git a/bonobo/execution/node.py b/bonobo/execution/node.py index 8822e73..4b52ec4 100644 --- a/bonobo/execution/node.py +++ b/bonobo/execution/node.py @@ -7,7 +7,8 @@ from bonobo.core.inputs import Input from bonobo.core.statistics import WithStatistics from bonobo.errors import InactiveReadableError from bonobo.execution.base import LoopingExecutionContext -from bonobo.structs.bags import Bag, ErrorBag +from bonobo.structs.bags import Bag +from bonobo.util.errors import is_error from bonobo.util.iterators import iter_if_not_sequence @@ -32,7 +33,13 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext): return (('+' if self.alive else '-') + ' ' + self.__name__ + ' ' + self.get_statistics_as_string()).strip() def __repr__(self): - return '<' + self.__str__() + '>' + stats = self.get_statistics_as_string().strip() + return '<{}({}{}){}>'.format( + type(self).__name__, + '+' if self.alive else '', + self.__name__, + (' ' + stats) if stats else '', + ) def recv(self, *messages): """ @@ -116,10 +123,6 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext): self.push(_resolve(input_bag, result)) -def is_error(bag): - return isinstance(bag, ErrorBag) - - def _resolve(input_bag, output): # NotModified means to send the input unmodified to output. if output is NOT_MODIFIED: diff --git a/bonobo/strategies/executor.py b/bonobo/strategies/executor.py index f3306ea..d43f2cd 100644 --- a/bonobo/strategies/executor.py +++ b/bonobo/strategies/executor.py @@ -1,10 +1,12 @@ import time +import traceback from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor from bonobo.constants import BEGIN, END from bonobo.strategies.base import Strategy from bonobo.structs.bags import Bag +from bonobo.util.errors import print_error class ExecutorStrategy(Strategy): @@ -27,20 +29,32 @@ class ExecutorStrategy(Strategy): futures = [] for plugin_context in context.plugins: - def _runner(plugin_context=plugin_context): - plugin_context.start() - plugin_context.loop() - plugin_context.stop() + try: + plugin_context.start() + plugin_context.loop() + plugin_context.stop() + except Exception as exc: + print_error(exc, traceback.format_exc(), prefix='Error in plugin context', context=plugin_context) futures.append(executor.submit(_runner)) for node_context in context.nodes: def _runner(node_context=node_context): - node_context.start() - node_context.loop() - node_context.stop() + try: + node_context.start() + except Exception as exc: + print_error(exc, traceback.format_exc(), prefix='Could not start node context', + context=node_context) + node_context.input.on_end() + else: + node_context.loop() + + try: + node_context.stop() + except Exception as exc: + print_error(exc, traceback.format_exc(), prefix='Could not stop node context', context=node_context) futures.append(executor.submit(_runner)) diff --git a/bonobo/util/errors.py b/bonobo/util/errors.py new file mode 100644 index 0000000..468ba2f --- /dev/null +++ b/bonobo/util/errors.py @@ -0,0 +1,30 @@ +import sys + +from bonobo.structs.bags import ErrorBag + + +def is_error(bag): + return isinstance(bag, ErrorBag) + + +def print_error(exc, trace, context=None, prefix=''): + """ + Error handler. Whatever happens in a plugin or component, if it looks like an exception, taste like an exception + or somehow make me think it is an exception, I'll handle it. + + :param exc: the culprit + :param trace: Hercule Poirot's logbook. + :return: to hell + """ + + from colorama import Fore, Style + print( + Style.BRIGHT, + Fore.RED, + '\U0001F4A3 {}{}{}'.format((prefix + ': ') if prefix else '', type(exc).__name__, + ' in {!r}'.format(context) if context else ''), + Style.RESET_ALL, + sep='', + file=sys.stderr, + ) + print(trace) From 0feccb1aa96f7baa4546525857d91fb31d00b2c7 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 15:21:26 +0200 Subject: [PATCH 2/3] reformating code, and adding specific rules for examples so it shows up correctly on readthedocs, by default. --- bonobo/_api.py | 11 ++++-- bonobo/basics.py | 1 + bonobo/examples/.style.yapf | 4 +++ bonobo/examples/datasets/fablabs.py | 36 ++++++++++++++----- bonobo/examples/files/json_handlers.py | 2 ++ bonobo/examples/tutorials/tut02_01_read.py | 4 +-- bonobo/examples/tutorials/tut02_02_write.py | 4 +-- .../examples/tutorials/tut02_03_writeasmap.py | 4 +-- bonobo/ext/jupyter/plugin.py | 2 -- bonobo/strategies/executor.py | 6 ++-- bonobo/structs/bags.py | 2 +- bonobo/util/errors.py | 5 +-- setup.py | 32 ++++++++--------- 13 files changed, 70 insertions(+), 43 deletions(-) create mode 100644 bonobo/examples/.style.yapf diff --git a/bonobo/_api.py b/bonobo/_api.py index 7d0f15e..1c05e59 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -88,7 +88,15 @@ def open_fs(fs_url, *args, **kwargs): # bonobo.basics -register_api_group(Limit, PrettyPrint, Tee, count, identity, noop, pprint, ) +register_api_group( + Limit, + PrettyPrint, + Tee, + count, + identity, + noop, + pprint, +) # bonobo.io register_api_group(CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter) @@ -116,4 +124,3 @@ def get_examples_path(*pathsegments): @register_api def open_examples_fs(*pathsegments): return open_fs(get_examples_path(*pathsegments)) - diff --git a/bonobo/basics.py b/bonobo/basics.py index 6fd8c3d..97282ab 100644 --- a/bonobo/basics.py +++ b/bonobo/basics.py @@ -66,6 +66,7 @@ pprint = Tee(_pprint) def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True): from bonobo.constants import NOT_MODIFIED + def _pprint(*args, **kwargs): nonlocal title_keys, sort, print_values diff --git a/bonobo/examples/.style.yapf b/bonobo/examples/.style.yapf new file mode 100644 index 0000000..70f7590 --- /dev/null +++ b/bonobo/examples/.style.yapf @@ -0,0 +1,4 @@ +[style] +based_on_style = pep8 +column_limit = 80 +dedent_closing_brackets = true diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index 17ee841..ae74c0b 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -25,7 +25,9 @@ from bonobo.ext.opendatasoft import OpenDataSoftAPI try: import pycountry except ImportError as exc: - raise ImportError('You must install package "pycountry" to run this example.') from exc + raise ImportError( + 'You must install package "pycountry" to run this example.' + ) from exc API_DATASET = 'fablabs-in-the-world' API_NETLOC = 'datanova.laposte.fr' @@ -57,20 +59,38 @@ def display(row): address = list( filter( None, ( - ' '.join(filter(None, (row.get('postal_code', None), row.get('city', None)))), row.get('county', None), - row.get('country'), + ' '.join( + filter( + None, + (row.get('postal_code', None), row.get('city', None)) + ) + ), row.get('county', None), row.get('country'), ) ) ) - print(' - {}address{}: {address}'.format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address))) - print(' - {}links{}: {links}'.format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links']))) - print(' - {}geometry{}: {geometry}'.format(Fore.BLUE, Style.RESET_ALL, **row)) - print(' - {}source{}: {source}'.format(Fore.BLUE, Style.RESET_ALL, source='datanova/' + API_DATASET)) + print( + ' - {}address{}: {address}'. + format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address)) + ) + print( + ' - {}links{}: {links}'. + format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links'])) + ) + print( + ' - {}geometry{}: {geometry}'. + format(Fore.BLUE, Style.RESET_ALL, **row) + ) + print( + ' - {}source{}: {source}'. + format(Fore.BLUE, Style.RESET_ALL, source='datanova/' + API_DATASET) + ) graph = bonobo.Graph( - OpenDataSoftAPI(dataset=API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris'), + OpenDataSoftAPI( + dataset=API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris' + ), normalize, filter_france, bonobo.Tee(display), diff --git a/bonobo/examples/files/json_handlers.py b/bonobo/examples/files/json_handlers.py index 727a137..0bb0436 100644 --- a/bonobo/examples/files/json_handlers.py +++ b/bonobo/examples/files/json_handlers.py @@ -1,9 +1,11 @@ import bonobo from bonobo.commands.run import get_default_services + def get_fields(row): return row['fields'] + graph = bonobo.Graph( bonobo.JsonReader(path='datasets/theaters.json'), get_fields, diff --git a/bonobo/examples/tutorials/tut02_01_read.py b/bonobo/examples/tutorials/tut02_01_read.py index e934c79..28baa9e 100644 --- a/bonobo/examples/tutorials/tut02_01_read.py +++ b/bonobo/examples/tutorials/tut02_01_read.py @@ -6,6 +6,4 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={ - 'fs': bonobo.open_examples_fs('datasets') - }) + bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) diff --git a/bonobo/examples/tutorials/tut02_02_write.py b/bonobo/examples/tutorials/tut02_02_write.py index d63dd47..a3c4811 100644 --- a/bonobo/examples/tutorials/tut02_02_write.py +++ b/bonobo/examples/tutorials/tut02_02_write.py @@ -12,6 +12,4 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={ - 'fs': bonobo.open_examples_fs('datasets') - }) + bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) diff --git a/bonobo/examples/tutorials/tut02_03_writeasmap.py b/bonobo/examples/tutorials/tut02_03_writeasmap.py index e131acd..bdf3194 100644 --- a/bonobo/examples/tutorials/tut02_03_writeasmap.py +++ b/bonobo/examples/tutorials/tut02_03_writeasmap.py @@ -22,6 +22,4 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={ - 'fs': bonobo.open_examples_fs('datasets') - }) + bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) diff --git a/bonobo/ext/jupyter/plugin.py b/bonobo/ext/jupyter/plugin.py index fe907be..a418c83 100644 --- a/bonobo/ext/jupyter/plugin.py +++ b/bonobo/ext/jupyter/plugin.py @@ -22,5 +22,3 @@ class JupyterOutputPlugin(Plugin): self.widget.value = [repr(node) for node in self.context.parent.nodes] finalize = run - - diff --git a/bonobo/strategies/executor.py b/bonobo/strategies/executor.py index d43f2cd..3f34862 100644 --- a/bonobo/strategies/executor.py +++ b/bonobo/strategies/executor.py @@ -29,6 +29,7 @@ class ExecutorStrategy(Strategy): futures = [] for plugin_context in context.plugins: + def _runner(plugin_context=plugin_context): try: plugin_context.start() @@ -45,8 +46,9 @@ class ExecutorStrategy(Strategy): try: node_context.start() except Exception as exc: - print_error(exc, traceback.format_exc(), prefix='Could not start node context', - context=node_context) + print_error( + exc, traceback.format_exc(), prefix='Could not start node context', context=node_context + ) node_context.input.on_end() else: node_context.loop() diff --git a/bonobo/structs/bags.py b/bonobo/structs/bags.py index 4d86f89..1d4a7e8 100644 --- a/bonobo/structs/bags.py +++ b/bonobo/structs/bags.py @@ -32,7 +32,7 @@ class Bag: foo notbaz """ - + def __init__(self, *args, _flags=None, _parent=None, **kwargs): self._flags = _flags or () self._parent = _parent diff --git a/bonobo/util/errors.py b/bonobo/util/errors.py index 468ba2f..bd1f51f 100644 --- a/bonobo/util/errors.py +++ b/bonobo/util/errors.py @@ -21,8 +21,9 @@ def print_error(exc, trace, context=None, prefix=''): print( Style.BRIGHT, Fore.RED, - '\U0001F4A3 {}{}{}'.format((prefix + ': ') if prefix else '', type(exc).__name__, - ' in {!r}'.format(context) if context else ''), + '\U0001F4A3 {}{}{}'.format( + (prefix + ': ') if prefix else '', type(exc).__name__, ' in {!r}'.format(context) if context else '' + ), Style.RESET_ALL, sep='', file=sys.stderr, diff --git a/setup.py b/setup.py index 83bd632..8b02f65 100644 --- a/setup.py +++ b/setup.py @@ -45,39 +45,37 @@ setup( description='Bonobo', license='Apache License, Version 2.0', install_requires=[ - 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', - 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0' + 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0' ], version=version, long_description=read('README.rst'), classifiers=read('classifiers.txt', tolines), packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, - data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/ext/jupyter/static/extension.js', - 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map' - ])], + data_files=[ + ( + 'share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js', + 'bonobo/ext/jupyter/static/index.js.map' + ] + ) + ], extras_require={ 'dev': [ - 'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', - 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx', + 'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx', 'sphinx_rtd_theme', 'yapf' ], 'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5'] }, entry_points={ 'bonobo.commands': [ - 'init = bonobo.commands.init:register', - 'run = bonobo.commands.run:register', + 'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register' ], 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], - 'edgy.project.features': - ['bonobo = ' - 'bonobo.ext.edgy.project.feature:BonoboFeature'] + 'edgy.project.features': ['bonobo = ' + 'bonobo.ext.edgy.project.feature:BonoboFeature'] }, url='https://www.bonobo-project.org/', - download_url= - 'https://github.com/python-bonobo/bonobo/tarball/{version}'.format( - version=version), ) + download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), +) From bbd258c313fbd219ab6351cc5bd9ca41d08c184b Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 15:22:32 +0200 Subject: [PATCH 3/3] 74 characters instead of 80 for examples, as it seems that desktop version display less characters than tablet on rtd theme. --- bonobo/examples/.style.yapf | 2 +- bonobo/examples/datasets/fablabs.py | 11 +++++++---- bonobo/examples/tutorials/tut02_01_read.py | 4 +++- bonobo/examples/tutorials/tut02_02_write.py | 4 +++- bonobo/examples/tutorials/tut02_03_writeasmap.py | 4 +++- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/bonobo/examples/.style.yapf b/bonobo/examples/.style.yapf index 70f7590..ffebd86 100644 --- a/bonobo/examples/.style.yapf +++ b/bonobo/examples/.style.yapf @@ -1,4 +1,4 @@ [style] based_on_style = pep8 -column_limit = 80 +column_limit = 74 dedent_closing_brackets = true diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index ae74c0b..be95fe1 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -61,8 +61,10 @@ def display(row): None, ( ' '.join( filter( - None, - (row.get('postal_code', None), row.get('city', None)) + None, ( + row.get('postal_code', None), + row.get('city', None) + ) ) ), row.get('county', None), row.get('country'), ) @@ -82,8 +84,9 @@ def display(row): format(Fore.BLUE, Style.RESET_ALL, **row) ) print( - ' - {}source{}: {source}'. - format(Fore.BLUE, Style.RESET_ALL, source='datanova/' + API_DATASET) + ' - {}source{}: {source}'.format( + Fore.BLUE, Style.RESET_ALL, source='datanova/' + API_DATASET + ) ) diff --git a/bonobo/examples/tutorials/tut02_01_read.py b/bonobo/examples/tutorials/tut02_01_read.py index 28baa9e..9806d43 100644 --- a/bonobo/examples/tutorials/tut02_01_read.py +++ b/bonobo/examples/tutorials/tut02_01_read.py @@ -6,4 +6,6 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) + bonobo.run( + graph, services={'fs': bonobo.open_examples_fs('datasets')} + ) diff --git a/bonobo/examples/tutorials/tut02_02_write.py b/bonobo/examples/tutorials/tut02_02_write.py index a3c4811..161c58e 100644 --- a/bonobo/examples/tutorials/tut02_02_write.py +++ b/bonobo/examples/tutorials/tut02_02_write.py @@ -12,4 +12,6 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) + bonobo.run( + graph, services={'fs': bonobo.open_examples_fs('datasets')} + ) diff --git a/bonobo/examples/tutorials/tut02_03_writeasmap.py b/bonobo/examples/tutorials/tut02_03_writeasmap.py index bdf3194..4598b56 100644 --- a/bonobo/examples/tutorials/tut02_03_writeasmap.py +++ b/bonobo/examples/tutorials/tut02_03_writeasmap.py @@ -22,4 +22,6 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) + bonobo.run( + graph, services={'fs': bonobo.open_examples_fs('datasets')} + )