Moves jupyter extension to both bonobo.contrib.jupyter (for the jupyter widget) and to bonobo.plugins (for the executor-side plugin).

.gitignore

@@ -23,8 +23,8 @@
 .python-version
 /.idea
 /.release
+/bonobo/contrib/jupyter/js/node_modules/
 /bonobo/examples/work_in_progress/
-/bonobo/ext/jupyter/js/node_modules/
 /build/
 /coverage.xml
 /dist/

Makefile

@@ -1,4 +1,4 @@
-# Generated by Medikit 0.4.1 on 2017-11-04.
+# Generated by Medikit 0.4.1 on 2017-11-12.
 # All changes will be overriden.
 
 PACKAGE ?= bonobo
@@ -10,6 +10,7 @@ PYTHON_REQUIREMENTS_DEV_FILE ?= requirements-dev.txt
 QUICK ?= 
 PIP ?= $(PYTHON_DIRNAME)/pip
 PIP_INSTALL_OPTIONS ?= 
+VERSION ?= $(shell git describe 2>/dev/null || git rev-parse --short HEAD)
 PYTEST ?= $(PYTHON_DIRNAME)/pytest
 PYTEST_OPTIONS ?= --capture=no --cov=$(PACKAGE) --cov-report html
 SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build
@@ -18,7 +19,6 @@ SPHINX_SOURCEDIR ?= docs
 SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
 YAPF ?= $(PYTHON) -m yapf
 YAPF_OPTIONS ?= -rip
-VERSION ?= $(shell git describe 2>/dev/null || echo dev)
 
 .PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev test update update-requirements
 

Projectfile

@@ -18,9 +18,9 @@ python.setup(
     data_files=[
         (
             'share/jupyter/nbextensions/bonobo-jupyter', [
-                'bonobo/ext/jupyter/static/extension.js',
+                'bonobo/contrib/jupyter/static/extension.js',
-                'bonobo/ext/jupyter/static/index.js',
+                'bonobo/contrib/jupyter/static/index.js',
-                'bonobo/ext/jupyter/static/index.js.map',
+                'bonobo/contrib/jupyter/static/index.js.map',
             ]
         ),
     ],
@@ -41,23 +41,24 @@ python.setup(
 
 python.add_requirements(
     'fs >=2.0,<2.1',
-    'jinja2 >=2.9,<2.10',
+    'graphviz >=0.8,<0.9',
+    'jinja2 >=2.9,<3',
     'mondrian >=0.4,<0.5',
     'packaging >=16,<17',
-    'psutil >=5.4,<6.0',
+    'psutil >=5.4,<6',
-    'requests >=2.0,<3.0',
+    'requests >=2,<3',
     'stevedore >=1.27,<1.28',
     'whistle >=1.0,<1.1',
     dev=[
-        'pytest-sugar >=0.8,<0.9',
+        'pytest-sugar >=0.9,<0.10',
         'pytest-timeout >=1,<2',
     ],
     docker=[
         'bonobo-docker >=0.5.0',
     ],
     jupyter=[
-        'jupyter >=1.0,<1.1',
         'ipywidgets >=6.0.0,<7',
+        'jupyter >=1.0,<1.1',
     ],
     sqlalchemy=[
         'bonobo-sqlalchemy >=0.5.1',
@@ -66,6 +67,6 @@ python.add_requirements(
 
 # Following requirements are not enforced, because some dependencies enforce them so we don't want to break
 # the packaging in case it changes in dep.
-python.add_requirements('colorama >=0.3', )
+python.add_requirements('colorama >=0.3')
 
 # vim: ft=python:

bonobo/_api.py

@@ -74,7 +74,7 @@ def run(graph, *, plugins=None, services=None, strategy=None):
 
         if _is_jupyter_notebook():
             try:
-                from bonobo.ext.jupyter import JupyterOutputPlugin
+                from bonobo.contrib.jupyter import JupyterOutputPlugin
             except ImportError:
                 import logging
                 logging.warning(

bonobo/contrib/jupyter/__init__.py

@@ -0,0 +1,15 @@
+from bonobo.plugins.jupyter import JupyterOutputPlugin
+
+
+def _jupyter_nbextension_paths():
+    return [{
+        'section': 'notebook',
+        'src': 'static',
+        'dest': 'bonobo-jupyter',
+        'require': 'bonobo-jupyter/extension'
+    }]
+
+
+__all__ = [
+    'JupyterOutputPlugin',
+]

bonobo/contrib/jupyter/js/.gitignore

@@ -0,0 +1 @@
+/node_modules

bonobo/contrib/jupyter/js/README.rst

@@ -0,0 +1,19 @@
+Bonobo within Jupyter
+=====================
+
+Install
+-------
+
+.. code-block:: shell-session
+
+    yarn install
+
+
+Watch mode (for development)
+----------------------------
+
+.. code-block:: shell-session
+
+    yarn run webpack --watch
+
+

bonobo/contrib/jupyter/js/dist/index.js

@@ -69,7 +69,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	// When serialiazing entire widget state for embedding, only values different from the
 	// defaults will be specified.
 	
-	var BonoboModel = widgets.DOMWidgetModel.extend({
+	const BonoboModel = widgets.DOMWidgetModel.extend({
 	    defaults: _.extend({}, widgets.DOMWidgetModel.prototype.defaults, {
 	        _model_name: 'BonoboModel',
 	        _view_name: 'BonoboView',
@@ -81,7 +81,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	
 	
 	// Custom View. Renders the widget model.
-	var BonoboView = widgets.DOMWidgetView.extend({
+	const BonoboView = widgets.DOMWidgetView.extend({
 	    render: function () {
 	        this.value_changed();
 	        this.model.on('change:value', this.value_changed, this);
@@ -89,7 +89,9 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	
 	    value_changed: function () {
 	        this.$el.html(
-	            this.model.get('value').join('<br>')
+	            '<div class="rendered_html"><table style="margin: 0; border: 1px solid black;">' + this.model.get('value').map((key, i) => {
+	                return `<tr><td>${key.status}</td><td>${key.name}</td><td>${key.stats}</td><td>${key.flags}</td></tr>`
+	            }).join('\n') + '</table></div>'
 	        );
 	    },
 	});

bonobo/contrib/jupyter/js/src/bonobo.js

@@ -8,7 +8,7 @@ var _ = require('underscore');
 // When serialiazing entire widget state for embedding, only values different from the
 // defaults will be specified.
 
-var BonoboModel = widgets.DOMWidgetModel.extend({
+const BonoboModel = widgets.DOMWidgetModel.extend({
     defaults: _.extend({}, widgets.DOMWidgetModel.prototype.defaults, {
         _model_name: 'BonoboModel',
         _view_name: 'BonoboView',
@@ -20,7 +20,7 @@ var BonoboModel = widgets.DOMWidgetModel.extend({
 
 
 // Custom View. Renders the widget model.
-var BonoboView = widgets.DOMWidgetView.extend({
+const BonoboView = widgets.DOMWidgetView.extend({
     render: function () {
         this.value_changed();
         this.model.on('change:value', this.value_changed, this);
@@ -28,7 +28,9 @@ var BonoboView = widgets.DOMWidgetView.extend({
 
     value_changed: function () {
         this.$el.html(
-            this.model.get('value').join('<br>')
+            '<div class="rendered_html"><table style="margin: 0; border: 1px solid black;">' + this.model.get('value').map((key, i) => {
+                return `<tr><td>${key.status}</td><td>${key.name}</td><td>${key.stats}</td><td>${key.flags}</td></tr>`
+            }).join('\n') + '</table></div>'
         );
     },
 });

bonobo/contrib/jupyter/static/index.js

@@ -72,7 +72,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	// When serialiazing entire widget state for embedding, only values different from the
 	// defaults will be specified.
 	
-	var BonoboModel = widgets.DOMWidgetModel.extend({
+	const BonoboModel = widgets.DOMWidgetModel.extend({
 	    defaults: _.extend({}, widgets.DOMWidgetModel.prototype.defaults, {
 	        _model_name: 'BonoboModel',
 	        _view_name: 'BonoboView',
@@ -84,7 +84,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	
 	
 	// Custom View. Renders the widget model.
-	var BonoboView = widgets.DOMWidgetView.extend({
+	const BonoboView = widgets.DOMWidgetView.extend({
 	    render: function () {
 	        this.value_changed();
 	        this.model.on('change:value', this.value_changed, this);
@@ -92,7 +92,9 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	
 	    value_changed: function () {
 	        this.$el.html(
-	            this.model.get('value').join('<br>')
+	            '<div class="rendered_html"><table style="margin: 0; border: 1px solid black;">' + this.model.get('value').map((key, i) => {
+	                return `<tr><td>${key.status}</td><td>${key.name}</td><td>${key.stats}</td><td>${key.flags}</td></tr>`
+	            }).join('\n') + '</table></div>'
 	        );
 	    },
 	});

bonobo/execution/contexts/node.py

@@ -159,6 +159,14 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
             # self._exec_count += 1
             pass
 
+    def as_dict(self):
+        return {
+            'status': self.status,
+            'name': self.name,
+            'stats': self.get_statistics_as_string(),
+            'flags': self.get_flags_as_string(),
+        }
+
 
 def isflag(param):
     return isinstance(param, Token) and param in (NOT_MODIFIED, )

bonobo/execution/strategies/executor.py

@@ -8,6 +8,7 @@ from bonobo.constants import BEGIN, END
 from bonobo.execution.strategies.base import Strategy
 from bonobo.util import get_name
 
+logger = logging.getLogger(__name__)
 
 class ExecutorStrategy(Strategy):
     """
@@ -30,8 +31,7 @@ class ExecutorStrategy(Strategy):
             try:
                 context.start(self.get_starter(executor, futures))
             except:
-                logging.getLogger(__name__
+                logger.critical('Exception caught while starting execution context.', exc_info=sys.exc_info())
-                                  ).warning('KeyboardInterrupt received. Trying to terminate the nodes gracefully.')
 
             while context.alive:
                 try:

bonobo/ext/__init__.py

@@ -1 +0,0 @@
-""" Extensions, not required. """

bonobo/ext/jupyter/__init__.py

@@ -1,10 +0,0 @@
-from .plugin import JupyterOutputPlugin
-
-
-def _jupyter_nbextension_paths():
-    return [{'section': 'notebook', 'src': 'static', 'dest': 'bonobo-jupyter', 'require': 'bonobo-jupyter/extension'}]
-
-
-__all__ = [
-    'JupyterOutputPlugin',
-]

bonobo/ext/jupyter/js/README.md

@@ -1,19 +0,0 @@
-Bonobo integration in Jupyter
-
-Package Install
----------------
-
-**Prerequisites**
-- [node](http://nodejs.org/)
-
-```bash
-npm install --save bonobo-jupyter
-```
-
-Watch mode (for development)
-----------------------------
-
-```bash
-./node_modules/.bin/webpack --watch
-``
-

bonobo/ext/jupyter/plugin.py

@@ -1,26 +0,0 @@
-import logging
-
-from bonobo.ext.jupyter.widget import BonoboWidget
-from bonobo.plugins import Plugin
-
-try:
-    import IPython.core.display
-except ImportError as e:
-    logging.exception(
-        'You must install Jupyter to use the bonobo Jupyter extension. Easiest way is to install the '
-        'optional "jupyter" dependencies with «pip install bonobo[jupyter]», but you can also install a '
-        'specific version by yourself.'
-    )
-
-
-class JupyterOutputPlugin(Plugin):
-    def initialize(self):
-        self.widget = BonoboWidget()
-        IPython.core.display.display(self.widget)
-
-    def run(self):
-        self.widget.value = [
-            str(self.context.parent[i]) for i in self.context.parent.graph.topologically_sorted_indexes
-        ]
-
-    finalize = run

bonobo/plugins/__init__.py

@@ -3,8 +3,11 @@ class Plugin:
     A plugin is an extension to the core behavior of bonobo. If you're writing transformations, you should not need
     to use this interface.
     
-    For examples, you can read bonobo.ext.console.ConsoleOutputPlugin, or bonobo.ext.jupyter.JupyterOutputPlugin that
+    For examples, you can read bonobo.plugins.console.ConsoleOutputPlugin, or bonobo.plugins.jupyter.JupyterOutputPlugin
-    respectively permits an interactive output on an ANSI console and a rich output in a jupyter notebook.
+    that respectively permits an interactive output on an ANSI console and a rich output in a jupyter notebook. Note
+    that you most probably won't instanciate them by yourself at runtime, as it's the default behaviour of bonobo to use
+    them if your in a compatible context (aka an interactive terminal for the console plugin, or a jupyter notebook for
+    the notebook plugin.)
     
     Warning: THE PLUGIN API IS PRE-ALPHA AND WILL EVOLVE BEFORE 1.0, DO NOT RELY ON IT BEING STABLE!
 

bonobo/plugins/jupyter.py

@@ -0,0 +1,35 @@
+import logging
+
+from bonobo.contrib.jupyter.widget import BonoboWidget
+from bonobo.execution import events
+from bonobo.plugins import Plugin
+
+try:
+    import IPython.core.display
+except ImportError as e:
+    logging.exception(
+        'You must install Jupyter to use the bonobo Jupyter extension. Easiest way is to install the '
+        'optional "jupyter" dependencies with «pip install bonobo[jupyter]», but you can also install a '
+        'specific version by yourself.'
+    )
+
+
+class JupyterOutputPlugin(Plugin):
+    def register(self, dispatcher):
+        dispatcher.add_listener(events.START, self.setup)
+        dispatcher.add_listener(events.TICK, self.tick)
+        dispatcher.add_listener(events.STOPPED, self.tick)
+
+    def unregister(self, dispatcher):
+        dispatcher.remove_listener(events.STOPPED, self.tick)
+        dispatcher.remove_listener(events.TICK, self.tick)
+        dispatcher.remove_listener(events.START, self.setup)
+
+    def setup(self, event):
+        self.widget = BonoboWidget()
+        IPython.core.display.display(self.widget)
+
+    def tick(self, event):
+        self.widget.value = [
+            event.context[i].as_dict() for i in event.context.graph.topologically_sorted_indexes
+        ]

bonobo/structs/graphs.py

@@ -1,6 +1,9 @@
+import html
 import json
 from copy import copy
 
+from graphviz.dot import Digraph
+
 from bonobo.constants import BEGIN
 from bonobo.util import get_name
 
@@ -112,23 +115,31 @@ class Graph:
             self._topologcally_sorted_indexes_cache = tuple(filter(lambda i: type(i) is int, reversed(order)))
             return self._topologcally_sorted_indexes_cache
 
+    @property
+    def graphviz(self):
+        try:
+            return self._graphviz
+        except AttributeError:
+            g = Digraph()
+            g.attr(rankdir='LR')
+            g.node('BEGIN', shape='point')
+            for i in self.outputs_of(BEGIN):
+                g.edge('BEGIN', str(i))
+            for ix in self.topologically_sorted_indexes:
+                g.node(str(ix), label=get_name(self[ix]))
+                for iy in self.outputs_of(ix):
+                    g.edge(str(ix), str(iy))
+            self._graphviz = g
+            return self._graphviz
+
     def _repr_dot_(self):
-        src = [
+        return str(self.graphviz)
-            'digraph {',
-            '  rankdir = LR;',
-            '  "BEGIN" [shape="point"];',
-        ]
 
-        for i in self.outputs_of(BEGIN):
+    def _repr_svg_(self):
-            src.append('  "BEGIN" -> ' + _get_graphviz_node_id(self, i) + ';')
+        return self.graphviz._repr_svg_()
 
-        for ix in self.topologically_sorted_indexes:
+    def _repr_html_(self):
-            for iy in self.outputs_of(ix):
+        return '<div>{}</div><pre>{}</pre>'.format(self.graphviz._repr_svg_(), html.escape(repr(self)))
-                src.append('  {} -> {};'.format(_get_graphviz_node_id(self, ix), _get_graphviz_node_id(self, iy)))
-
-        src.append('}')
-
-        return '\n'.join(src)
 
     def _resolve_index(self, mixed):
         """ Find the index based on various strategies for a node, probably an input or output of chain. Supported inputs are indexes, node values or names.

docs/_templates/sidebarlogo.html

@@ -1,7 +1,9 @@
 <a href="{{ pathto(master_doc) }}" style="border: none">
     <h1 style="text-align: center; margin: 0;">
         <img class="logo" src="{{ pathto('_static/bonobo.png', 1) }}" title="Bonobo" style="width: 48px; height: 48px; vertical-align: bottom"/>
-        Bonobo
+        <span class="brand">
+            Bonobo
+        </span>
     </h1>
 </a>
 

docs/conf.py

@@ -193,5 +193,5 @@ rst_epilog = """
 .. |longversion| replace:: v.{version}
 
 """.format(
-    version = version,
+    version=version,
 )

docs/tutorial/1-init.rst

@@ -242,8 +242,8 @@ The console output contains two things.
       a call, but the execution will move to the next row.
 
 
-Moving forward
+Wrap up
-::::::::::::::
+:::::::
 
 That's all for this first step.
 

requirements-dev.txt

@@ -1,18 +1,18 @@
 -e .[dev]
 alabaster==0.7.10
 babel==2.5.1
-certifi==2017.7.27.1
+certifi==2017.11.5
 chardet==3.0.4
-coverage==4.4.1
+coverage==4.4.2
 docutils==0.14
 idna==2.6
 imagesize==0.7.1
-jinja2==2.9.6
+jinja2==2.10
 markupsafe==1.0
 py==1.4.34
 pygments==2.2.0
 pytest-cov==2.5.1
-pytest-sugar==0.8.0
+pytest-sugar==0.9.0
 pytest-timeout==1.2.0
 pytest==3.2.3
 pytz==2017.3

requirements-docker.txt

@@ -1,16 +1,16 @@
 -e .[docker]
 appdirs==1.4.3
 bonobo-docker==0.5.0
-certifi==2017.7.27.1
+certifi==2017.11.5
 chardet==3.0.4
 colorama==0.3.9
 docker-pycreds==0.2.1
 docker==2.3.0
-fs==2.0.12
+fs==2.0.16
 idna==2.6
 packaging==16.8
 pbr==3.1.1
-psutil==5.4.0
+psutil==5.4.1
 pyparsing==2.2.0
 pytz==2017.3
 requests==2.18.4

requirements-jupyter.txt

@@ -3,32 +3,32 @@ appnope==0.1.0
 bleach==2.1.1
 decorator==4.1.2
 entrypoints==0.2.3
-html5lib==0.999999999
+html5lib==1.0b10
 ipykernel==4.6.1
 ipython-genutils==0.2.0
 ipython==6.2.1
 ipywidgets==6.0.1
 jedi==0.11.0
-jinja2==2.9.6
+jinja2==2.10
 jsonschema==2.6.0
 jupyter-client==5.1.0
 jupyter-console==5.2.0
 jupyter-core==4.4.0
 jupyter==1.0.0
 markupsafe==1.0
-mistune==0.8
+mistune==0.8.1
 nbconvert==5.3.1
 nbformat==4.4.0
 notebook==5.2.1
 pandocfilters==1.4.2
 parso==0.1.0
-pexpect==4.2.1
+pexpect==4.3.0
 pickleshare==0.7.4
 prompt-toolkit==1.0.15
 ptyprocess==0.5.2
 pygments==2.2.0
 python-dateutil==2.6.1
-pyzmq==16.0.3
+pyzmq==17.0.0b3
 qtconsole==4.3.1
 simplegeneric==0.8.1
 six==1.11.0

requirements-sqlalchemy.txt

@@ -1,14 +1,14 @@
 -e .[sqlalchemy]
 appdirs==1.4.3
 bonobo-sqlalchemy==0.5.1
-certifi==2017.7.27.1
+certifi==2017.11.5
 chardet==3.0.4
 colorama==0.3.9
-fs==2.0.12
+fs==2.0.16
 idna==2.6
 packaging==16.8
 pbr==3.1.1
-psutil==5.4.0
+psutil==5.4.1
 pyparsing==2.2.0
 pytz==2017.3
 requests==2.18.4

requirements.txt

@@ -1,16 +1,17 @@
 -e .
 appdirs==1.4.3
-certifi==2017.7.27.1
+certifi==2017.11.5
 chardet==3.0.4
 colorama==0.3.9
-fs==2.0.12
+fs==2.0.16
+graphviz==0.8.1
 idna==2.6
-jinja2==2.9.6
+jinja2==2.10
 markupsafe==1.0
 mondrian==0.4.0
 packaging==16.8
 pbr==3.1.1
-psutil==5.4.0
+psutil==5.4.1
 pyparsing==2.2.0
 pytz==2017.3
 requests==2.18.4

setup.py

@@ -43,6 +43,14 @@ else:
 setup(
     author='Romain Dorgueil',
     author_email='romain@dorgueil.net',
+    data_files=[
+        (
+            'share/jupyter/nbextensions/bonobo-jupyter', [
+                'bonobo/contrib/jupyter/static/extension.js', 'bonobo/contrib/jupyter/static/index.js',
+                'bonobo/contrib/jupyter/static/index.js.map'
+            ]
+        )
+    ],
     description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for '
                  'python 3.5+.'),
     license='Apache License, Version 2.0',
@@ -53,14 +61,14 @@ setup(
     packages=find_packages(exclude=['ez_setup', 'example', 'test']),
     include_package_data=True,
     install_requires=[
-        'colorama (>= 0.3)', 'fs (>= 2.0, < 2.1)', 'jinja2 (>= 2.9, < 2.10)', 'mondrian (>= 0.4, < 0.5)',
+        'colorama (>= 0.3)', 'fs (>= 2.0, < 2.1)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (>= 2.9, < 3)',
-        'packaging (>= 16, < 17)', 'psutil (>= 5.4, < 6.0)', 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.27, < 1.28)',
+        'mondrian (>= 0.4, < 0.5)', 'packaging (>= 16, < 17)', 'psutil (>= 5.4, < 6)', 'requests (>= 2, < 3)',
-        'whistle (>= 1.0, < 1.1)'
+        'stevedore (>= 1.27, < 1.28)', 'whistle (>= 1.0, < 1.1)'
     ],
     extras_require={
         'dev': [
             'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)',
-            'pytest-sugar (>= 0.8, < 0.9)', 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)', 'yapf'
+            'pytest-sugar (>= 0.9, < 0.10)', 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)', 'yapf'
         ],
         'docker': ['bonobo-docker (>= 0.5.0)'],
         'jupyter': ['ipywidgets (>= 6.0.0, < 7)', 'jupyter (>= 1.0, < 1.1)'],

tests/commands/test_convert.py

@@ -6,8 +6,9 @@ from bonobo.util.environ import change_working_directory
 from bonobo.util.testing import all_runners
 
 
-@pytest.mark.skipif(sys.version_info < (3, 6),
+@pytest.mark.skipif(
-                    reason="python 3.5 does not preserve kwargs order and this cant pass for now")
+    sys.version_info < (3, 6), reason="python 3.5 does not preserve kwargs order and this cant pass for now"
+)
 @all_runners
 def test_convert(runner, tmpdir):
     csv_content = 'id;name\n1;Romain'

tests/ext/test_ods.py

@@ -1,6 +1,6 @@
 from unittest.mock import patch
 
-from bonobo.ext.opendatasoft import OpenDataSoftAPI
+from bonobo.contrib.opendatasoft import OpenDataSoftAPI
 from bonobo.util.objects import ValueHolder