Merge pull request #219 from hartym/develop

Update stdlib, new plugin architecture applied to existing plugins.

.gitignore

@@ -23,8 +23,8 @@
 .python-version
 /.idea
 /.release
+/bonobo/contrib/jupyter/js/node_modules/
 /bonobo/examples/work_in_progress/
-/bonobo/ext/jupyter/js/node_modules/
 /build/
 /coverage.xml
 /dist/

Makefile

@@ -1,4 +1,4 @@
-# Generated by Medikit 0.4.1 on 2017-11-04.
+# Generated by Medikit 0.4.1 on 2017-11-12.
 # All changes will be overriden.
 
 PACKAGE ?= bonobo
@@ -10,6 +10,7 @@ PYTHON_REQUIREMENTS_DEV_FILE ?= requirements-dev.txt
 QUICK ?= 
 PIP ?= $(PYTHON_DIRNAME)/pip
 PIP_INSTALL_OPTIONS ?= 
+VERSION ?= $(shell git describe 2>/dev/null || git rev-parse --short HEAD)
 PYTEST ?= $(PYTHON_DIRNAME)/pytest
 PYTEST_OPTIONS ?= --capture=no --cov=$(PACKAGE) --cov-report html
 SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build
@@ -18,7 +19,6 @@ SPHINX_SOURCEDIR ?= docs
 SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
 YAPF ?= $(PYTHON) -m yapf
 YAPF_OPTIONS ?= -rip
-VERSION ?= $(shell git describe 2>/dev/null || echo dev)
 
 .PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev test update update-requirements
 

Projectfile

@@ -18,9 +18,9 @@ python.setup(
     data_files=[
         (
             'share/jupyter/nbextensions/bonobo-jupyter', [
-                'bonobo/ext/jupyter/static/extension.js',
+                'bonobo/contrib/jupyter/static/extension.js',
-                'bonobo/ext/jupyter/static/index.js',
+                'bonobo/contrib/jupyter/static/index.js',
-                'bonobo/ext/jupyter/static/index.js.map',
+                'bonobo/contrib/jupyter/static/index.js.map',
             ]
         ),
     ],
@@ -41,23 +41,24 @@ python.setup(
 
 python.add_requirements(
     'fs >=2.0,<2.1',
-    'jinja2 >=2.9,<2.10',
+    'graphviz >=0.8,<0.9',
+    'jinja2 >=2.9,<3',
     'mondrian >=0.4,<0.5',
     'packaging >=16,<17',
-    'psutil >=5.4,<6.0',
+    'psutil >=5.4,<6',
-    'requests >=2.0,<3.0',
+    'requests >=2,<3',
     'stevedore >=1.27,<1.28',
     'whistle >=1.0,<1.1',
     dev=[
-        'pytest-sugar >=0.8,<0.9',
+        'pytest-sugar >=0.9,<0.10',
         'pytest-timeout >=1,<2',
     ],
     docker=[
         'bonobo-docker >=0.5.0',
     ],
     jupyter=[
-        'jupyter >=1.0,<1.1',
         'ipywidgets >=6.0.0,<7',
+        'jupyter >=1.0,<1.1',
     ],
     sqlalchemy=[
         'bonobo-sqlalchemy >=0.5.1',
@@ -66,6 +67,6 @@ python.add_requirements(
 
 # Following requirements are not enforced, because some dependencies enforce them so we don't want to break
 # the packaging in case it changes in dep.
-python.add_requirements('colorama >=0.3', )
+python.add_requirements('colorama >=0.3')
 
 # vim: ft=python:

benchmarks/parameters.py

@@ -0,0 +1,56 @@
+"""
+Compare passing a dict to passing a dict as kwargs to a stupid transformation
+
+Last results (1 mill calls):
+
+j1 1.5026444319955772
+k1 1.8377482700016117
+j2 1.1962292949901894
+k2 1.5545833489886718
+j3 1.0014333260041894
+k3 1.353256585993222
+
+"""
+import json
+import timeit
+
+
+def j1(d):
+    return {'prepend': 'foo', **d, 'append': 'bar'}
+
+
+def k1(**d):
+    return {'prepend': 'foo', **d, 'append': 'bar'}
+
+
+def j2(d):
+    return {**d}
+
+
+def k2(**d):
+    return {**d}
+
+
+def j3(d):
+    return None
+
+
+def k3(**d):
+    return None
+
+
+if __name__ == '__main__':
+    import timeit
+
+    with open('person.json') as f:
+        json_data = json.load(f)
+
+    for i in 1, 2, 3:
+        print(
+            'j{}'.format(i),
+            timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i))
+        )
+        print(
+            'k{}'.format(i),
+            timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i))
+        )

benchmarks/person.json

@@ -0,0 +1,46 @@
+{
+  "@context": "http://schema.org",
+  "@type": "MusicEvent",
+  "location": {
+    "@type": "MusicVenue",
+    "name": "Chicago Symphony Center",
+    "address": "220 S. Michigan Ave, Chicago, Illinois, USA"
+  },
+  "name": "Shostakovich Leningrad",
+  "offers": {
+    "@type": "Offer",
+    "url": "/examples/ticket/12341234",
+    "price": "40",
+    "priceCurrency": "USD",
+    "availability": "http://schema.org/InStock"
+  },
+  "performer": [
+    {
+      "@type": "MusicGroup",
+      "name": "Chicago Symphony Orchestra",
+      "sameAs": [
+        "http://cso.org/",
+        "http://en.wikipedia.org/wiki/Chicago_Symphony_Orchestra"
+      ]
+    },
+    {
+      "@type": "Person",
+      "image": "/examples/jvanzweden_s.jpg",
+      "name": "Jaap van Zweden",
+      "sameAs": "http://www.jaapvanzweden.com/"
+    }
+  ],
+  "startDate": "2014-05-23T20:00",
+  "workPerformed": [
+    {
+      "@type": "CreativeWork",
+      "name": "Britten Four Sea Interludes and Passacaglia from Peter Grimes",
+      "sameAs": "http://en.wikipedia.org/wiki/Peter_Grimes"
+    },
+    {
+      "@type": "CreativeWork",
+      "name": "Shostakovich Symphony No. 7 (Leningrad)",
+      "sameAs": "http://en.wikipedia.org/wiki/Symphony_No._7_(Shostakovich)"
+    }
+  ]
+}

bonobo/_api.py

@@ -1,6 +1,26 @@
 from bonobo.execution.strategies import create_strategy
-from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
+from bonobo.nodes import (
-    PickleReader, PickleWriter, PrettyPrinter, RateLimited, Tee, arg0_to_kwargs, count, identity, kwargs_to_arg0, noop
+    CsvReader,
+    CsvWriter,
+    FileReader,
+    FileWriter,
+    Filter,
+    FixedWindow,
+    JsonReader,
+    JsonWriter,
+    Limit,
+    PickleReader,
+    PickleWriter,
+    PrettyPrinter,
+    RateLimited,
+    Tee,
+    Update,
+    arg0_to_kwargs,
+    count,
+    identity,
+    kwargs_to_arg0,
+    noop,
+)
 from bonobo.nodes import LdjsonReader, LdjsonWriter
 from bonobo.structs import Bag, ErrorBag, Graph, Token
 from bonobo.util import get_name
@@ -25,8 +45,10 @@ def register_graph_api(x, __all__=__all__):
     required_parameters = {'plugins', 'services', 'strategy'}
     assert parameters[0] == 'graph', 'First parameter of a graph api function must be "graph".'
     assert required_parameters.intersection(
-        parameters) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join(
+        parameters
-        sorted(required_parameters))
+    ) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join(
+        sorted(required_parameters)
+    )
 
     return register_api(x, __all__=__all__)
 
@@ -74,7 +96,7 @@ def run(graph, *, plugins=None, services=None, strategy=None):
 
         if _is_jupyter_notebook():
             try:
-                from bonobo.ext.jupyter import JupyterOutputPlugin
+                from bonobo.contrib.jupyter import JupyterOutputPlugin
             except ImportError:
                 import logging
                 logging.warning(
@@ -149,6 +171,7 @@ register_api_group(
     FileReader,
     FileWriter,
     Filter,
+    FixedWindow,
     JsonReader,
     JsonWriter,
     LdjsonReader,
@@ -159,6 +182,7 @@ register_api_group(
     PrettyPrinter,
     RateLimited,
     Tee,
+    Update,
     arg0_to_kwargs,
     count,
     identity,

bonobo/contrib/django/__init__.py

@@ -0,0 +1,7 @@
+from .utils import create_or_update
+from .commands import ETLCommand
+
+__all__ = [
+    'ETLCommand',
+    'create_or_update',
+]

bonobo/contrib/django/commands.py

@@ -1,54 +1,16 @@
 from logging import getLogger
 
 import bonobo
-import bonobo.util
 from bonobo.plugins.console import ConsoleOutputPlugin
 from bonobo.util.term import CLEAR_EOL
 from colorama import Fore, Back, Style
-from django.core.management.base import BaseCommand, OutputWrapper
+from django.core.management import BaseCommand
+from django.core.management.base import OutputWrapper
+
+from .utils import create_or_update
 
 
 class ETLCommand(BaseCommand):
-    GraphType = bonobo.Graph
-
-    def create_parser(self, prog_name, subcommand):
-        return bonobo.get_argument_parser(
-            super().create_parser(prog_name, subcommand)
-        )
-
-    def create_or_update(self, model, *, defaults=None, save=True, **kwargs):
-        """
-        Create or update a django model instance.
-
-        :param model:
-        :param defaults:
-        :param kwargs:
-        :return: object, created, updated
-
-        """
-        obj, created = model._default_manager.get_or_create(defaults=defaults, **kwargs)
-
-        updated = False
-        if not created:
-            for k, v in defaults.items():
-                if getattr(obj, k) != v:
-                    setattr(obj, k, v)
-                    updated = True
-
-            if updated and save:
-                obj.save()
-
-        return obj, created, updated
-
-    def get_graph(self, *args, **options):
-        def not_implemented():
-            raise NotImplementedError('You must implement {}.get_graph() method.'.format(self))
-
-        return self.GraphType(not_implemented)
-
-    def get_services(self):
-        return {}
-
     @property
     def logger(self):
         try:
@@ -57,6 +19,20 @@ class ETLCommand(BaseCommand):
             self._logger = getLogger(type(self).__module__)
             return self._logger
 
+    create_or_update = staticmethod(create_or_update)
+
+    def create_parser(self, prog_name, subcommand):
+        return bonobo.get_argument_parser(super().create_parser(prog_name, subcommand))
+
+    def get_graph(self, *args, **options):
+        def not_implemented():
+            raise NotImplementedError('You must implement {}.get_graph() method.'.format(self))
+
+        return bonobo.Graph(not_implemented)
+
+    def get_services(self):
+        return {}
+
     def info(self, *args, **kwargs):
         self.logger.info(*args, **kwargs)
 

bonobo/contrib/django/utils.py

@@ -0,0 +1,23 @@
+def create_or_update(model, *, defaults=None, save=True, **kwargs):
+    """
+    Create or update a django model instance.
+
+    :param model:
+    :param defaults:
+    :param kwargs:
+    :return: object, created, updated
+
+    """
+    obj, created = model._default_manager.get_or_create(defaults=defaults, **kwargs)
+
+    updated = False
+    if not created:
+        for k, v in defaults.items():
+            if getattr(obj, k) != v:
+                setattr(obj, k, v)
+                updated = True
+
+        if updated and save:
+            obj.save()
+
+    return obj, created, updated

bonobo/contrib/jupyter/__init__.py

@@ -1,4 +1,4 @@
-from .plugin import JupyterOutputPlugin
+from bonobo.plugins.jupyter import JupyterOutputPlugin
 
 
 def _jupyter_nbextension_paths():

bonobo/contrib/jupyter/js/.gitignore

@@ -0,0 +1 @@
+/node_modules

bonobo/contrib/jupyter/js/README.rst

@@ -0,0 +1,19 @@
+Bonobo within Jupyter
+=====================
+
+Install
+-------
+
+.. code-block:: shell-session
+
+    yarn install
+
+
+Watch mode (for development)
+----------------------------
+
+.. code-block:: shell-session
+
+    yarn run webpack --watch
+
+

bonobo/contrib/jupyter/js/dist/index.js

@@ -69,7 +69,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	// When serialiazing entire widget state for embedding, only values different from the
 	// defaults will be specified.
 	
-	var BonoboModel = widgets.DOMWidgetModel.extend({
+	const BonoboModel = widgets.DOMWidgetModel.extend({
 	    defaults: _.extend({}, widgets.DOMWidgetModel.prototype.defaults, {
 	        _model_name: 'BonoboModel',
 	        _view_name: 'BonoboView',
@@ -81,7 +81,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	
 	
 	// Custom View. Renders the widget model.
-	var BonoboView = widgets.DOMWidgetView.extend({
+	const BonoboView = widgets.DOMWidgetView.extend({
 	    render: function () {
 	        this.value_changed();
 	        this.model.on('change:value', this.value_changed, this);
@@ -89,7 +89,9 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	
 	    value_changed: function () {
 	        this.$el.html(
-	            this.model.get('value').join('<br>')
+	            '<div class="rendered_html"><table style="margin: 0; border: 1px solid black;">' + this.model.get('value').map((key, i) => {
+	                return `<tr><td>${key.status}</td><td>${key.name}</td><td>${key.stats}</td><td>${key.flags}</td></tr>`
+	            }).join('\n') + '</table></div>'
 	        );
 	    },
 	});

bonobo/contrib/jupyter/js/src/bonobo.js

@@ -8,7 +8,7 @@ var _ = require('underscore');
 // When serialiazing entire widget state for embedding, only values different from the
 // defaults will be specified.
 
-var BonoboModel = widgets.DOMWidgetModel.extend({
+const BonoboModel = widgets.DOMWidgetModel.extend({
     defaults: _.extend({}, widgets.DOMWidgetModel.prototype.defaults, {
         _model_name: 'BonoboModel',
         _view_name: 'BonoboView',
@@ -20,7 +20,7 @@ var BonoboModel = widgets.DOMWidgetModel.extend({
 
 
 // Custom View. Renders the widget model.
-var BonoboView = widgets.DOMWidgetView.extend({
+const BonoboView = widgets.DOMWidgetView.extend({
     render: function () {
         this.value_changed();
         this.model.on('change:value', this.value_changed, this);
@@ -28,7 +28,9 @@ var BonoboView = widgets.DOMWidgetView.extend({
 
     value_changed: function () {
         this.$el.html(
-            this.model.get('value').join('<br>')
+            '<div class="rendered_html"><table style="margin: 0; border: 1px solid black;">' + this.model.get('value').map((key, i) => {
+                return `<tr><td>${key.status}</td><td>${key.name}</td><td>${key.stats}</td><td>${key.flags}</td></tr>`
+            }).join('\n') + '</table></div>'
         );
     },
 });

bonobo/contrib/jupyter/static/index.js

@@ -72,7 +72,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	// When serialiazing entire widget state for embedding, only values different from the
 	// defaults will be specified.
 	
-	var BonoboModel = widgets.DOMWidgetModel.extend({
+	const BonoboModel = widgets.DOMWidgetModel.extend({
 	    defaults: _.extend({}, widgets.DOMWidgetModel.prototype.defaults, {
 	        _model_name: 'BonoboModel',
 	        _view_name: 'BonoboView',
@@ -84,7 +84,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	
 	
 	// Custom View. Renders the widget model.
-	var BonoboView = widgets.DOMWidgetView.extend({
+	const BonoboView = widgets.DOMWidgetView.extend({
 	    render: function () {
 	        this.value_changed();
 	        this.model.on('change:value', this.value_changed, this);
@@ -92,7 +92,9 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	
 	    value_changed: function () {
 	        this.$el.html(
-	            this.model.get('value').join('<br>')
+	            '<div class="rendered_html"><table style="margin: 0; border: 1px solid black;">' + this.model.get('value').map((key, i) => {
+	                return `<tr><td>${key.status}</td><td>${key.name}</td><td>${key.stats}</td><td>${key.flags}</td></tr>`
+	            }).join('\n') + '</table></div>'
 	        );
 	    },
 	});

bonobo/examples/datasets/coffeeshops.py

@@ -15,7 +15,7 @@ Extracts a list of parisian bars where you can buy a coffee for a reasonable pri
 
 import bonobo
 from bonobo.commands import get_default_services
-from bonobo.ext.opendatasoft import OpenDataSoftAPI
+from bonobo.contrib.opendatasoft import OpenDataSoftAPI
 
 filename = 'coffeeshops.txt'
 

bonobo/examples/datasets/fablabs.py

@@ -20,7 +20,7 @@ from colorama import Fore, Style
 
 import bonobo
 from bonobo.commands import get_default_services
-from bonobo.ext.opendatasoft import OpenDataSoftAPI
+from bonobo.contrib.opendatasoft import OpenDataSoftAPI
 
 try:
     import pycountry

bonobo/execution/contexts/node.py

@@ -159,6 +159,14 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
             # self._exec_count += 1
             pass
 
+    def as_dict(self):
+        return {
+            'status': self.status,
+            'name': self.name,
+            'stats': self.get_statistics_as_string(),
+            'flags': self.get_flags_as_string(),
+        }
+
 
 def isflag(param):
     return isinstance(param, Token) and param in (NOT_MODIFIED, )

bonobo/execution/strategies/executor.py

@@ -8,6 +8,8 @@ from bonobo.constants import BEGIN, END
 from bonobo.execution.strategies.base import Strategy
 from bonobo.util import get_name
 
+logger = logging.getLogger(__name__)
+
 
 class ExecutorStrategy(Strategy):
     """
@@ -30,8 +32,7 @@ class ExecutorStrategy(Strategy):
             try:
                 context.start(self.get_starter(executor, futures))
             except:
-                logging.getLogger(__name__
+                logger.critical('Exception caught while starting execution context.', exc_info=sys.exc_info())
-                                  ).warning('KeyboardInterrupt received. Trying to terminate the nodes gracefully.')
 
             while context.alive:
                 try:

bonobo/ext/__init__.py

@@ -1 +0,0 @@
-""" Extensions, not required. """

bonobo/ext/jupyter/js/README.md

@@ -1,19 +0,0 @@
-Bonobo integration in Jupyter
-
-Package Install
----------------
-
-**Prerequisites**
-- [node](http://nodejs.org/)
-
-```bash
-npm install --save bonobo-jupyter
-```
-
-Watch mode (for development)
-----------------------------
-
-```bash
-./node_modules/.bin/webpack --watch
-``
-

bonobo/ext/jupyter/plugin.py

@@ -1,26 +0,0 @@
-import logging
-
-from bonobo.ext.jupyter.widget import BonoboWidget
-from bonobo.plugins import Plugin
-
-try:
-    import IPython.core.display
-except ImportError as e:
-    logging.exception(
-        'You must install Jupyter to use the bonobo Jupyter extension. Easiest way is to install the '
-        'optional "jupyter" dependencies with «pip install bonobo[jupyter]», but you can also install a '
-        'specific version by yourself.'
-    )
-
-
-class JupyterOutputPlugin(Plugin):
-    def initialize(self):
-        self.widget = BonoboWidget()
-        IPython.core.display.display(self.widget)
-
-    def run(self):
-        self.widget.value = [
-            str(self.context.parent[i]) for i in self.context.parent.graph.topologically_sorted_indexes
-        ]
-
-    finalize = run

bonobo/nodes/basics.py

@@ -4,16 +4,17 @@ import itertools
 from bonobo import settings
 from bonobo.config import Configurable, Option
 from bonobo.config.processors import ContextProcessor
+from bonobo.constants import NOT_MODIFIED
 from bonobo.structs.bags import Bag
 from bonobo.util.objects import ValueHolder
 from bonobo.util.term import CLEAR_EOL
 
-from bonobo.constants import NOT_MODIFIED
-
 __all__ = [
+    'FixedWindow',
     'Limit',
     'PrettyPrinter',
     'Tee',
+    'Update',
     'arg0_to_kwargs',
     'count',
     'identity',
@@ -128,3 +129,49 @@ def kwargs_to_arg0(**row):
     :return: bonobo.Bag
     """
     return Bag(row)
+
+
+def Update(*consts, **kwconsts):
+    """
+    Transformation factory to update a stream with constant values, by appending to args and updating kwargs.
+
+    :param consts: what to append to the input stream args
+    :param kwconsts: what to use to update input stream kwargs
+    :return: function
+
+    """
+
+    def update(*args, **kwargs):
+        nonlocal consts, kwconsts
+        return (*args, *consts, {**kwargs, **kwconsts})
+
+    update.__name__ = 'Update({})'.format(Bag.format_args(*consts, **kwconsts))
+
+    return update
+
+
+class FixedWindow(Configurable):
+    """
+    Transformation factory to create fixed windows of inputs, as lists.
+
+    For example, if the input is successively 1, 2, 3, 4, etc. and you pass it through a ``FixedWindow(2)``, you'll get
+    lists of elements 2 by 2: [1, 2], [3, 4], ...
+
+    """
+
+    length = Option(int, positional=True)  # type: int
+
+    @ContextProcessor
+    def buffer(self, context):
+        buffer = yield ValueHolder([])
+        if len(buffer):
+            context.send(Bag(buffer.get()))
+
+    def call(self, buffer, x):
+        buffer.append(x)
+        if len(buffer) >= self.length:
+            yield buffer.get()
+            buffer.set([])
+
+
+

bonobo/nodes/io/json.py

@@ -35,13 +35,14 @@ class JsonWriter(FileWriter, JsonHandler):
         yield
         file.write(self.suffix)
 
-    def write(self, fs, file, lineno, **row):
+    def write(self, fs, file, lineno, arg0=None, **kwargs):
         """
         Write a json row on the next line of file pointed by ctx.file.
 
         :param ctx:
         :param row:
         """
+        row = _getrow(arg0, kwargs)
         self._write_line(file, (self.eol if lineno.value else '') + json.dumps(row))
         lineno += 1
         return NOT_MODIFIED
@@ -59,7 +60,19 @@ class LdjsonReader(FileReader):
 class LdjsonWriter(FileWriter):
     """Write a stream of JSON objects, one object per line."""
 
-    def write(self, fs, file, lineno, **row):
+    def write(self, fs, file, lineno, arg0=None, **kwargs):
-        lineno += 1  # class-level variable
+        row = _getrow(arg0, kwargs)
         file.write(json.dumps(row) + '\n')
+        lineno += 1  # class-level variable
         return NOT_MODIFIED
+
+
+def _getrow(arg0, kwargs):
+    if len(kwargs):
+        assert arg0 is None, 'Got both positional and keyword arguments, I recommend using keyword arguments.'
+        return kwargs
+
+    if arg0 is not None:
+        return arg0
+
+    return kwargs

bonobo/plugins/__init__.py

@@ -3,8 +3,11 @@ class Plugin:
     A plugin is an extension to the core behavior of bonobo. If you're writing transformations, you should not need
     to use this interface.
     
-    For examples, you can read bonobo.ext.console.ConsoleOutputPlugin, or bonobo.ext.jupyter.JupyterOutputPlugin that
+    For examples, you can read bonobo.plugins.console.ConsoleOutputPlugin, or bonobo.plugins.jupyter.JupyterOutputPlugin
-    respectively permits an interactive output on an ANSI console and a rich output in a jupyter notebook.
+    that respectively permits an interactive output on an ANSI console and a rich output in a jupyter notebook. Note
+    that you most probably won't instanciate them by yourself at runtime, as it's the default behaviour of bonobo to use
+    them if your in a compatible context (aka an interactive terminal for the console plugin, or a jupyter notebook for
+    the notebook plugin.)
     
     Warning: THE PLUGIN API IS PRE-ALPHA AND WILL EVOLVE BEFORE 1.0, DO NOT RELY ON IT BEING STABLE!
 

bonobo/plugins/jupyter.py

@@ -0,0 +1,33 @@
+import logging
+
+from bonobo.contrib.jupyter.widget import BonoboWidget
+from bonobo.execution import events
+from bonobo.plugins import Plugin
+
+try:
+    import IPython.core.display
+except ImportError as e:
+    logging.exception(
+        'You must install Jupyter to use the bonobo Jupyter extension. Easiest way is to install the '
+        'optional "jupyter" dependencies with «pip install bonobo[jupyter]», but you can also install a '
+        'specific version by yourself.'
+    )
+
+
+class JupyterOutputPlugin(Plugin):
+    def register(self, dispatcher):
+        dispatcher.add_listener(events.START, self.setup)
+        dispatcher.add_listener(events.TICK, self.tick)
+        dispatcher.add_listener(events.STOPPED, self.tick)
+
+    def unregister(self, dispatcher):
+        dispatcher.remove_listener(events.STOPPED, self.tick)
+        dispatcher.remove_listener(events.TICK, self.tick)
+        dispatcher.remove_listener(events.START, self.setup)
+
+    def setup(self, event):
+        self.widget = BonoboWidget()
+        IPython.core.display.display(self.widget)
+
+    def tick(self, event):
+        self.widget.value = [event.context[i].as_dict() for i in event.context.graph.topologically_sorted_indexes]

bonobo/structs/bags.py

@@ -1,7 +1,7 @@
 import itertools
 
-from bonobo.structs.tokens import Token
 from bonobo.constants import INHERIT_INPUT, LOOPBACK
+from bonobo.structs.tokens import Token
 
 __all__ = [
     'Bag',
@@ -36,6 +36,10 @@ class Bag:
 
     default_flags = ()
 
+    @staticmethod
+    def format_args(*args, **kwargs):
+        return ', '.join(itertools.chain(map(repr, args), ('{}={!r}'.format(k, v) for k, v in kwargs.items())))
+
     def __new__(cls, *args, _flags=None, _parent=None, **kwargs):
         # Handle the special case where we call Bag's constructor with only one bag or token as argument.
         if len(args) == 1 and len(kwargs) == 0:
@@ -86,6 +90,9 @@ class Bag:
             self._args = args
             self._kwargs = kwargs
 
+    def __repr__(self):
+        return 'Bag({})'.format(Bag.format_args(*self.args, **self.kwargs))
+
     @property
     def args(self):
         if self._parent is None:
@@ -141,7 +148,7 @@ class Bag:
 
     @classmethod
     def inherit(cls, *args, **kwargs):
-        return cls(*args, _flags=(INHERIT_INPUT, ), **kwargs)
+        return cls(*args, _flags=(INHERIT_INPUT,), **kwargs)
 
     def __eq__(self, other):
         # XXX there are overlapping cases, but this is very handy for now. Let's think about it later.
@@ -169,19 +176,9 @@ class Bag:
 
         return len(self.args) == 1 and not self.kwargs and self.args[0] == other
 
-    def __repr__(self):
-        return '<{} ({})>'.format(
-            type(self).__name__, ', '.join(
-                itertools.chain(
-                    map(repr, self.args),
-                    ('{}={}'.format(k, repr(v)) for k, v in self.kwargs.items()),
-                )
-            )
-        )
-
 
 class LoopbackBag(Bag):
-    default_flags = (LOOPBACK, )
+    default_flags = (LOOPBACK,)
 
 
 class ErrorBag(Bag):

bonobo/structs/graphs.py

@@ -1,6 +1,9 @@
+import html
 import json
 from copy import copy
 
+from graphviz.dot import Digraph
+
 from bonobo.constants import BEGIN
 from bonobo.util import get_name
 
@@ -112,23 +115,31 @@ class Graph:
             self._topologcally_sorted_indexes_cache = tuple(filter(lambda i: type(i) is int, reversed(order)))
             return self._topologcally_sorted_indexes_cache
 
+    @property
+    def graphviz(self):
+        try:
+            return self._graphviz
+        except AttributeError:
+            g = Digraph()
+            g.attr(rankdir='LR')
+            g.node('BEGIN', shape='point')
+            for i in self.outputs_of(BEGIN):
+                g.edge('BEGIN', str(i))
+            for ix in self.topologically_sorted_indexes:
+                g.node(str(ix), label=get_name(self[ix]))
+                for iy in self.outputs_of(ix):
+                    g.edge(str(ix), str(iy))
+            self._graphviz = g
+            return self._graphviz
+
     def _repr_dot_(self):
-        src = [
+        return str(self.graphviz)
-            'digraph {',
-            '  rankdir = LR;',
-            '  "BEGIN" [shape="point"];',
-        ]
 
-        for i in self.outputs_of(BEGIN):
+    def _repr_svg_(self):
-            src.append('  "BEGIN" -> ' + _get_graphviz_node_id(self, i) + ';')
+        return self.graphviz._repr_svg_()
 
-        for ix in self.topologically_sorted_indexes:
+    def _repr_html_(self):
-            for iy in self.outputs_of(ix):
+        return '<div>{}</div><pre>{}</pre>'.format(self.graphviz._repr_svg_(), html.escape(repr(self)))
-                src.append('  {} -> {};'.format(_get_graphviz_node_id(self, ix), _get_graphviz_node_id(self, iy)))
-
-        src.append('}')
-
-        return '\n'.join(src)
 
     def _resolve_index(self, mixed):
         """ Find the index based on various strategies for a node, probably an input or output of chain. Supported inputs are indexes, node values or names.

bonobo/util/objects.py

@@ -225,6 +225,9 @@ class ValueHolder:
     def __setitem__(self, key, value):
         self._value[key] = value
 
+    def __getattr__(self, item):
+        return getattr(self._value, item)
+
 
 def get_attribute_or_create(obj, attr, default):
     try:

docs/_templates/sidebarlogo.html

@@ -1,7 +1,9 @@
 <a href="{{ pathto(master_doc) }}" style="border: none">
     <h1 style="text-align: center; margin: 0;">
         <img class="logo" src="{{ pathto('_static/bonobo.png', 1) }}" title="Bonobo" style="width: 48px; height: 48px; vertical-align: bottom"/>
-        Bonobo
+        <span class="brand">
+            Bonobo
+        </span>
     </h1>
 </a>
 

docs/conf.py

@@ -193,5 +193,5 @@ rst_epilog = """
 .. |longversion| replace:: v.{version}
 
 """.format(
-    version = version,
+    version=version,
 )

docs/tutorial/1-init.rst

@@ -242,8 +242,8 @@ The console output contains two things.
       a call, but the execution will move to the next row.
 
 
-Moving forward
+Wrap up
-::::::::::::::
+:::::::
 
 That's all for this first step.
 

requirements-dev.txt

@@ -1,18 +1,18 @@
 -e .[dev]
 alabaster==0.7.10
 babel==2.5.1
-certifi==2017.7.27.1
+certifi==2017.11.5
 chardet==3.0.4
-coverage==4.4.1
+coverage==4.4.2
 docutils==0.14
 idna==2.6
 imagesize==0.7.1
-jinja2==2.9.6
+jinja2==2.10
 markupsafe==1.0
 py==1.4.34
 pygments==2.2.0
 pytest-cov==2.5.1
-pytest-sugar==0.8.0
+pytest-sugar==0.9.0
 pytest-timeout==1.2.0
 pytest==3.2.3
 pytz==2017.3

requirements-docker.txt

@@ -1,16 +1,16 @@
 -e .[docker]
 appdirs==1.4.3
 bonobo-docker==0.5.0
-certifi==2017.7.27.1
+certifi==2017.11.5
 chardet==3.0.4
 colorama==0.3.9
 docker-pycreds==0.2.1
 docker==2.3.0
-fs==2.0.12
+fs==2.0.16
 idna==2.6
 packaging==16.8
 pbr==3.1.1
-psutil==5.4.0
+psutil==5.4.1
 pyparsing==2.2.0
 pytz==2017.3
 requests==2.18.4

requirements-jupyter.txt

@@ -3,32 +3,32 @@ appnope==0.1.0
 bleach==2.1.1
 decorator==4.1.2
 entrypoints==0.2.3
-html5lib==0.999999999
+html5lib==1.0b10
 ipykernel==4.6.1
 ipython-genutils==0.2.0
 ipython==6.2.1
 ipywidgets==6.0.1
 jedi==0.11.0
-jinja2==2.9.6
+jinja2==2.10
 jsonschema==2.6.0
 jupyter-client==5.1.0
 jupyter-console==5.2.0
 jupyter-core==4.4.0
 jupyter==1.0.0
 markupsafe==1.0
-mistune==0.8
+mistune==0.8.1
 nbconvert==5.3.1
 nbformat==4.4.0
 notebook==5.2.1
 pandocfilters==1.4.2
 parso==0.1.0
-pexpect==4.2.1
+pexpect==4.3.0
 pickleshare==0.7.4
 prompt-toolkit==1.0.15
 ptyprocess==0.5.2
 pygments==2.2.0
 python-dateutil==2.6.1
-pyzmq==16.0.3
+pyzmq==17.0.0b3
 qtconsole==4.3.1
 simplegeneric==0.8.1
 six==1.11.0

requirements-sqlalchemy.txt

@@ -1,14 +1,14 @@
 -e .[sqlalchemy]
 appdirs==1.4.3
 bonobo-sqlalchemy==0.5.1
-certifi==2017.7.27.1
+certifi==2017.11.5
 chardet==3.0.4
 colorama==0.3.9
-fs==2.0.12
+fs==2.0.16
 idna==2.6
 packaging==16.8
 pbr==3.1.1
-psutil==5.4.0
+psutil==5.4.1
 pyparsing==2.2.0
 pytz==2017.3
 requests==2.18.4

requirements.txt

@@ -1,16 +1,17 @@
 -e .
 appdirs==1.4.3
-certifi==2017.7.27.1
+certifi==2017.11.5
 chardet==3.0.4
 colorama==0.3.9
-fs==2.0.12
+fs==2.0.16
+graphviz==0.8.1
 idna==2.6
-jinja2==2.9.6
+jinja2==2.10
 markupsafe==1.0
 mondrian==0.4.0
 packaging==16.8
 pbr==3.1.1
-psutil==5.4.0
+psutil==5.4.1
 pyparsing==2.2.0
 pytz==2017.3
 requests==2.18.4

setup.py

@@ -43,6 +43,14 @@ else:
 setup(
     author='Romain Dorgueil',
     author_email='romain@dorgueil.net',
+    data_files=[
+        (
+            'share/jupyter/nbextensions/bonobo-jupyter', [
+                'bonobo/contrib/jupyter/static/extension.js', 'bonobo/contrib/jupyter/static/index.js',
+                'bonobo/contrib/jupyter/static/index.js.map'
+            ]
+        )
+    ],
     description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for '
                  'python 3.5+.'),
     license='Apache License, Version 2.0',
@@ -53,14 +61,14 @@ setup(
     packages=find_packages(exclude=['ez_setup', 'example', 'test']),
     include_package_data=True,
     install_requires=[
-        'colorama (>= 0.3)', 'fs (>= 2.0, < 2.1)', 'jinja2 (>= 2.9, < 2.10)', 'mondrian (>= 0.4, < 0.5)',
+        'colorama (>= 0.3)', 'fs (>= 2.0, < 2.1)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (>= 2.9, < 3)',
-        'packaging (>= 16, < 17)', 'psutil (>= 5.4, < 6.0)', 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.27, < 1.28)',
+        'mondrian (>= 0.4, < 0.5)', 'packaging (>= 16, < 17)', 'psutil (>= 5.4, < 6)', 'requests (>= 2, < 3)',
-        'whistle (>= 1.0, < 1.1)'
+        'stevedore (>= 1.27, < 1.28)', 'whistle (>= 1.0, < 1.1)'
     ],
     extras_require={
         'dev': [
             'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)',
-            'pytest-sugar (>= 0.8, < 0.9)', 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)', 'yapf'
+            'pytest-sugar (>= 0.9, < 0.10)', 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)', 'yapf'
         ],
         'docker': ['bonobo-docker (>= 0.5.0)'],
         'jupyter': ['ipywidgets (>= 6.0.0, < 7)', 'jupyter (>= 1.0, < 1.1)'],

tests/commands/test_convert.py

@@ -6,8 +6,9 @@ from bonobo.util.environ import change_working_directory
 from bonobo.util.testing import all_runners
 
 
-@pytest.mark.skipif(sys.version_info < (3, 6),
+@pytest.mark.skipif(
-                    reason="python 3.5 does not preserve kwargs order and this cant pass for now")
+    sys.version_info < (3, 6), reason="python 3.5 does not preserve kwargs order and this cant pass for now"
+)
 @all_runners
 def test_convert(runner, tmpdir):
     csv_content = 'id;name\n1;Romain'

tests/ext/test_ods.py

@@ -1,6 +1,6 @@
 from unittest.mock import patch
 
-from bonobo.ext.opendatasoft import OpenDataSoftAPI
+from bonobo.contrib.opendatasoft import OpenDataSoftAPI
 from bonobo.util.objects import ValueHolder
 
 

tests/nodes/test_basics.py

@@ -1,3 +1,4 @@
+from operator import methodcaller
 from unittest.mock import MagicMock
 
 import pytest
@@ -5,6 +6,7 @@ import pytest
 import bonobo
 from bonobo.config.processors import ContextCurrifier
 from bonobo.constants import NOT_MODIFIED
+from bonobo.util.testing import BufferingNodeExecutionContext
 
 
 def test_count():
@@ -72,3 +74,38 @@ def test_tee():
 
 def test_noop():
     assert bonobo.noop(1, 2, 3, 4, foo='bar') == NOT_MODIFIED
+
+
+def test_update():
+    with BufferingNodeExecutionContext(bonobo.Update('a', k=True)) as context:
+        context.write_sync('a', ('a', {'b': 1}), ('b', {'k': False}))
+    assert context.get_buffer() == [
+        bonobo.Bag('a', 'a', k=True),
+        bonobo.Bag('a', 'a', b=1, k=True),
+        bonobo.Bag('b', 'a', k=True),
+    ]
+    assert context.name == "Update('a', k=True)"
+
+
+def test_fixedwindow():
+    with BufferingNodeExecutionContext(bonobo.FixedWindow(2)) as context:
+        context.write_sync(*range(10))
+    assert context.get_buffer() == [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]]
+
+    with BufferingNodeExecutionContext(bonobo.FixedWindow(2)) as context:
+        context.write_sync(*range(9))
+    assert context.get_buffer() == [[0, 1], [2, 3], [4, 5], [6, 7], [8]]
+
+    with BufferingNodeExecutionContext(bonobo.FixedWindow(1)) as context:
+        context.write_sync(*range(3))
+    assert context.get_buffer() == [[0], [1], [2]]
+
+
+def test_methodcaller():
+    with BufferingNodeExecutionContext(methodcaller('swapcase')) as context:
+        context.write_sync('aaa', 'bBb', 'CcC')
+    assert context.get_buffer() == ['AAA', 'BbB', 'cCc']
+
+    with BufferingNodeExecutionContext(methodcaller('zfill', 5)) as context:
+        context.write_sync('a', 'bb', 'ccc')
+    assert context.get_buffer() == ['0000a', '000bb', '00ccc']

tests/structs/test_bags.py

@@ -159,7 +159,7 @@ def test_eq_operator_dict():
 
 def test_repr():
     bag = Bag('a', a=1)
-    assert repr(bag) == "<Bag ('a', a=1)>"
+    assert repr(bag) == "Bag('a', a=1)"
 
 
 def test_iterator():