Merge branch 'master' into transformation_factory

.gitignore

@@ -2,6 +2,7 @@
 *,cover
 *.egg
 *.egg-info/
+*.iml
 *.log
 *.manifest
 *.mo
@@ -20,25 +21,17 @@
 .installed.cfg
 .ipynb_checkpoints
 .python-version
-.tox/
-.webassets-cache
 /.idea
 /.release
-/bonobo.iml
 /bonobo/examples/work_in_progress/
 /bonobo/ext/jupyter/js/node_modules/
 /build/
 /coverage.xml
-/develop-eggs/
 /dist/
 /docs/_build/
-/downloads/
 /eggs/
 /examples/private
-/htmlcov/
 /sdist/
 /tags
-celerybeat-schedule
-parts/
 pip-delete-this-directory.txt
 pip-log.txt

.travis.yml

@@ -9,8 +9,6 @@ install:
   - make install-dev
   - pip install coveralls
 script:
-  - make clean docs test
+  - make clean test
-  - pip install pycountry
-  - bin/run_all_examples.sh
 after_success:
   - coveralls

Makefile

@@ -1,7 +1,7 @@
 # This file has been auto-generated.
 # All changes will be lost, see Projectfile.
 #
-# Updated at 2017-05-03 18:02:59.359160
+# Updated at 2017-07-04 10:50:55.775681
 
 PACKAGE ?= bonobo
 PYTHON ?= $(shell which python)
@@ -18,10 +18,11 @@ SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build
 SPHINX_OPTIONS ?= 
 SPHINX_SOURCEDIR ?= docs
 SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
-YAPF ?= $(PYTHON_DIRNAME)/yapf
+YAPF ?= $(PYTHON) -m yapf
 YAPF_OPTIONS ?= -rip
+VERSION ?= $(shell git describe 2>/dev/null || echo dev)
 
-.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev lint test
+.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev test
 
 # Installs the local project dependencies.
 install:
@@ -39,9 +40,6 @@ install-dev:
 clean:
 	rm -rf build dist *.egg-info
 
-lint: install-dev
-	$(PYTHON_DIRNAME)/pylint --py3k $(PACKAGE) -f html > pylint.html
-
 test: install-dev
 	$(PYTEST) $(PYTEST_OPTIONS) tests
 
@@ -50,3 +48,4 @@ $(SPHINX_SOURCEDIR): install-dev
 
 format: install-dev
 	$(YAPF) $(YAPF_OPTIONS) .
+	$(YAPF) $(YAPF_OPTIONS) Projectfile

Projectfile

@@ -1,60 +1,30 @@
 # bonobo (see github.com/python-edgy/project)
 
-name = 'bonobo'
+from edgy.project import require
-description = 'Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.'
-license = 'Apache License, Version 2.0'
 
-url = 'https://www.bonobo-project.org/'
+pytest = require('pytest')
-download_url = 'https://github.com/python-bonobo/bonobo/tarball/{version}'
+python = require('python')
+sphinx = require('sphinx')
+yapf = require('yapf')
 
-author = 'Romain Dorgueil'
+python.setup(
-author_email = 'romain@dorgueil.net'
+    name='bonobo',
-
+    description='Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.',
-enable_features = {
+    license='Apache License, Version 2.0',
-    'make',
+    url='https://www.bonobo-project.org/',
-    'sphinx',
+    download_url='https://github.com/python-bonobo/bonobo/tarball/{version}',
-    'pytest',
+    author='Romain Dorgueil',
-    'git',
+    author_email='romain@dorgueil.net',
-    'pylint',
+    data_files=[
-    'python',
+        (
-    'yapf',
+            'share/jupyter/nbextensions/bonobo-jupyter', [
-}
-
-# stricts deendencies in requirements.txt
-install_requires = [
-    'colorama >=0.3,<1.0',
-    'fs >=2.0,<3.0',
-    'psutil >=5.2,<6.0',
-    'requests >=2.0,<3.0',
-    'stevedore >=1.21,<2.0',
-]
-
-extras_require = {
-    'jupyter': [
-        'jupyter >=1.0,<1.1',
-        'ipywidgets >=6.0.0.beta5'
-    ],
-    'dev': [
-        'coverage >=4,<5',
-        'pylint >=1,<2',
-        'pytest >=3,<4',
-        'pytest-cov >=2,<3',
-        'pytest-timeout >=1,<2',
-        'sphinx',
-        'sphinx_rtd_theme',
-        'yapf',
-    ],
-}
-
-data_files = [
-    ('share/jupyter/nbextensions/bonobo-jupyter', [
                 'bonobo/ext/jupyter/static/extension.js',
                 'bonobo/ext/jupyter/static/index.js',
                 'bonobo/ext/jupyter/static/index.js.map',
-    ]),
+            ]
-]
+        ),
-
+    ],
-entry_points = {
+    entry_points={
         'console_scripts': [
             'bonobo = bonobo.commands:entrypoint',
         ],
@@ -63,11 +33,26 @@ entry_points = {
             'run = bonobo.commands.run:register',
             'version = bonobo.commands.version:register',
         ],
-    'edgy.project.features': [
+    }
-        'bonobo = bonobo.ext.edgy.project.feature:BonoboFeature'
+)
-    ]
-}
 
-@listen('edgy.project.feature.make.on_generate', priority=10)
+python.add_requirements(
-def on_make_generate_docker_targets(event):
+    'colorama >=0.3,<1.0',
-    event.makefile['SPHINX_SOURCEDIR'] = 'docs'
+    'fs >=2.0,<3.0',
+    'packaging >=16,<17',
+    'psutil >=5.2,<6.0',
+    'requests >=2.0,<3.0',
+    'stevedore >=1.21,<2.0',
+    dev=[
+        'cookiecutter >=1.5,<1.6',
+        'pytest-sugar >=0.8,<0.9',
+        'pytest-timeout >=1,<2',
+    ],
+    docker=[
+        'bonobo-docker',
+    ],
+    jupyter=[
+        'jupyter >=1.0,<1.1',
+        'ipywidgets >=6.0.0,<7',
+    ]
+)

README.rst

@@ -7,28 +7,29 @@ Data-processing for humans.
 .. image:: https://img.shields.io/pypi/v/bonobo.svg
     :target: https://pypi.python.org/pypi/bonobo
     :alt: PyPI
+
 .. image:: https://img.shields.io/pypi/pyversions/bonobo.svg
     :target: https://pypi.python.org/pypi/bonobo
     :alt: Versions
 
-.. image:: https://readthedocs.org/projects/bonobo/badge/?version=0.3
+.. image:: https://readthedocs.org/projects/bonobo/badge/?version=latest
     :target: http://docs.bonobo-project.org/
     :alt: Documentation
 
-.. image:: https://travis-ci.org/python-bonobo/bonobo.svg?branch=0.3
+.. image:: https://travis-ci.org/python-bonobo/bonobo.svg?branch=master
     :target: https://travis-ci.org/python-bonobo/bonobo
     :alt: Continuous Integration (Linux)
 
-.. image:: https://ci.appveyor.com/api/projects/status/github/python-bonobo/bonobo?retina=true&branch=0.3&svg=true
+.. image:: https://ci.appveyor.com/api/projects/status/github/python-bonobo/bonobo?retina=true&branch=master&svg=true
-    :target: https://ci.appveyor.com/project/hartym/bonobo?branch=0.3
+    :target: https://ci.appveyor.com/project/hartym/bonobo?branch=master
     :alt: Continuous Integration (Windows)
 
 .. image:: https://codeclimate.com/github/python-bonobo/bonobo/badges/gpa.svg
    :target: https://codeclimate.com/github/python-bonobo/bonobo
    :alt: Code Climate
 
-.. image:: https://img.shields.io/coveralls/python-bonobo/bonobo/0.3.svg
+.. image:: https://img.shields.io/coveralls/python-bonobo/bonobo/master.svg
-    :target: https://coveralls.io/github/python-bonobo/bonobo?branch=0.3
+    :target: https://coveralls.io/github/python-bonobo/bonobo?branch=master
     :alt: Coverage
 
 Bonobo is an extract-transform-load framework for python 3.5+ (see comparisons with other data tools).
@@ -50,12 +51,12 @@ so as though it may not yet be complete or fully stable (please, allow us to rea
 
 ----
 
+Homepage: https://www.bonobo-project.org/ (`Roadmap <https://www.bonobo-project.org/roadmap>`_)
+
 Documentation: http://docs.bonobo-project.org/
 
 Issues: https://github.com/python-bonobo/bonobo/issues
 
-Roadmap: https://www.bonobo-project.org/roadmap
-
 Slack: https://bonobo-slack.herokuapp.com/
 
 Release announcements: http://eepurl.com/csHFKL

bonobo/_api.py

@@ -1,8 +1,6 @@
-import warnings
-
 from bonobo.structs import Bag, Graph, Token
 from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
-    PrettyPrint, Tee, count, identity, noop, pprint
+    PrettyPrinter, PickleWriter, PickleReader, Tee, count, identity, noop, pprint
 from bonobo.strategies import create_strategy
 from bonobo.util.objects import get_name
 
@@ -44,12 +42,16 @@ def run(graph, strategy=None, plugins=None, services=None):
 
     plugins = plugins or []
 
-    if _is_interactive_console():  # pragma: no cover
+    from bonobo import settings
+    settings.check()
+
+    if not settings.QUIET:  # pragma: no cover
+        if _is_interactive_console():
             from bonobo.ext.console import ConsoleOutputPlugin
             if ConsoleOutputPlugin not in plugins:
                 plugins.append(ConsoleOutputPlugin)
 
-    if _is_jupyter_notebook():  # pragma: no cover
+        if _is_jupyter_notebook():
             from bonobo.ext.jupyter import JupyterOutputPlugin
             if JupyterOutputPlugin not in plugins:
                 plugins.append(JupyterOutputPlugin)
@@ -66,7 +68,7 @@ register_api(create_strategy)
 
 # Shortcut to filesystem2's open_fs, that we make available there for convenience.
 @register_api
-def open_fs(fs_url, *args, **kwargs):
+def open_fs(fs_url=None, *args, **kwargs):
     """
     Wraps :func:`fs.open_fs` function with a few candies.
     
@@ -80,7 +82,13 @@ def open_fs(fs_url, *args, **kwargs):
     :returns: :class:`~fs.base.FS` object
     """
     from fs import open_fs as _open_fs
-    return _open_fs(str(fs_url), *args, **kwargs)
+    from os.path import expanduser
+    from os import getcwd
+
+    if fs_url is None:
+        fs_url = getcwd()
+
+    return _open_fs(expanduser(str(fs_url)), *args, **kwargs)
 
 
 # bonobo.nodes
@@ -93,7 +101,9 @@ register_api_group(
     JsonReader,
     JsonWriter,
     Limit,
-    PrettyPrint,
+    PrettyPrinter,
+    PickleReader,
+    PickleWriter,
     Tee,
     count,
     identity,

bonobo/_version.py

@@ -1 +1 @@
-__version__ = '0.3.0a1'
+__version__ = '0.4.3'

bonobo/commands/__init__.py

@@ -1,11 +1,13 @@
 import argparse
 
-import logging
+from bonobo import logging, settings
-from stevedore import ExtensionManager
+
+logger = logging.get_logger()
 
 
 def entrypoint(args=None):
     parser = argparse.ArgumentParser()
+    parser.add_argument('--debug', '-D', action='store_true')
 
     subparsers = parser.add_subparsers(dest='command')
     subparsers.required = True
@@ -17,12 +19,17 @@ def entrypoint(args=None):
             parser = subparsers.add_parser(ext.name)
             commands[ext.name] = ext.plugin(parser)
         except Exception:
-            logging.exception('Error while loading command {}.'.format(ext.name))
+            logger.exception('Error while loading command {}.'.format(ext.name))
 
-    mgr = ExtensionManager(
+    from stevedore import ExtensionManager
-        namespace='bonobo.commands',
+    mgr = ExtensionManager(namespace='bonobo.commands')
-    )
     mgr.map(register_extension)
 
     args = parser.parse_args(args).__dict__
+    if args.pop('debug', False):
+        settings.DEBUG = True
+        settings.LOGGING_LEVEL = logging.DEBUG
+        logging.set_level(settings.LOGGING_LEVEL)
+
+    logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args))
     commands[args.pop('command')](**args)

bonobo/commands/init.py

@@ -1,16 +1,20 @@
-import os
+def execute(name, branch):
-
-
-def execute():
     try:
-        from edgy.project.__main__ import handle_init
+        from cookiecutter.main import cookiecutter
     except ImportError as exc:
         raise ImportError(
-            'You must install "edgy.project" to use this command.\n\n $ pip install edgy.project\n'
+            'You must install "cookiecutter" to use this command.\n\n $ pip install cookiecutter\n'
         ) from exc
 
-    return handle_init(os.path.join(os.getcwd(), 'Projectfile'))
+    return cookiecutter(
+        'https://github.com/python-bonobo/cookiecutter-bonobo.git',
+        extra_context={'name': name},
+        no_input=True,
+        checkout=branch
+    )
 
 
 def register(parser):
+    parser.add_argument('name')
+    parser.add_argument('--branch', '-b', default='master')
     return execute

bonobo/commands/run.py

@@ -1,12 +1,11 @@
-import argparse
-
 import os
 
-import bonobo
-
 DEFAULT_SERVICES_FILENAME = '_services.py'
 DEFAULT_SERVICES_ATTR = 'get_services'
 
+DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', )
+DEFAULT_GRAPH_ATTR = 'get_graph'
+
 
 def get_default_services(filename, services=None):
     dirname = os.path.dirname(filename)
@@ -18,10 +17,8 @@ def get_default_services(filename, services=None):
             '__name__': '__bonobo__',
             '__file__': services_filename,
         }
-        try:
         exec(code, context)
-        except Exception:
+
-            raise
         return {
             **context[DEFAULT_SERVICES_ATTR](),
             **(services or {}),
@@ -29,26 +26,47 @@ def get_default_services(filename, services=None):
     return services or {}
 
 
-def execute(file, quiet=False):
+def execute(filename, module, install=False, quiet=False, verbose=False):
-    with file:
+    import runpy
-        code = compile(file.read(), file.name, 'exec')
+    from bonobo import Graph, run, settings
 
-    # TODO: A few special variables should be set before running the file:
+    if quiet:
-    #
+        settings.QUIET = True
-    # See:
-    #  - https://docs.python.org/3/reference/import.html#import-mod-attrs
-    #  - https://docs.python.org/3/library/runpy.html#runpy.run_module
-    context = {
-        '__name__': '__bonobo__',
-        '__file__': file.name,
-    }
 
-    try:
+    if verbose:
-        exec(code, context)
+        settings.DEBUG = True
-    except Exception as exc:
-        raise
 
-    graphs = dict((k, v) for k, v in context.items() if isinstance(v, bonobo.Graph))
+    if filename:
+        if os.path.isdir(filename):
+            if install:
+                import importlib
+                import pip
+                requirements = os.path.join(filename, 'requirements.txt')
+                pip.main(['install', '-r', requirements])
+                # Some shenanigans to be sure everything is importable after this, especially .egg-link files which
+                # are referenced in *.pth files and apparently loaded by site.py at some magic bootstrap moment of the
+                # python interpreter.
+                pip.utils.pkg_resources = importlib.reload(pip.utils.pkg_resources)
+                import site
+                importlib.reload(site)
+
+            pathname = filename
+            for filename in DEFAULT_GRAPH_FILENAMES:
+                filename = os.path.join(pathname, filename)
+                if os.path.exists(filename):
+                    break
+            if not os.path.exists(filename):
+                raise IOError('Could not find entrypoint (candidates: {}).'.format(', '.join(DEFAULT_GRAPH_FILENAMES)))
+        elif install:
+            raise RuntimeError('Cannot --install on a file (only available for dirs containing requirements.txt).')
+        context = runpy.run_path(filename, run_name='__bonobo__')
+    elif module:
+        context = runpy.run_module(module, run_name='__bonobo__')
+        filename = context['__file__']
+    else:
+        raise RuntimeError('UNEXPECTED: argparse should not allow this.')
+
+    graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph))
 
     assert len(graphs) == 1, (
         'Having zero or more than one graph definition in one file is unsupported for now, '
@@ -59,16 +77,26 @@ def execute(file, quiet=False):
 
     # todo if console and not quiet, then add the console plugin
     # todo when better console plugin, add it if console and just disable display
-    return bonobo.run(
+    return run(
         graph,
         plugins=[],
         services=get_default_services(
-            file.name, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
+            filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
         )
     )
 
 
+def register_generic_run_arguments(parser, required=True):
+    source_group = parser.add_mutually_exclusive_group(required=required)
+    source_group.add_argument('filename', nargs='?', type=str)
+    source_group.add_argument('--module', '-m', type=str)
+    return parser
+
+
 def register(parser):
-    parser.add_argument('file', type=argparse.FileType())
+    parser = register_generic_run_arguments(parser)
-    parser.add_argument('--quiet', action='store_true')
+    verbosity_group = parser.add_mutually_exclusive_group()
+    verbosity_group.add_argument('--quiet', '-q', action='store_true')
+    verbosity_group.add_argument('--verbose', '-v', action='store_true')
+    parser.add_argument('--install', '-I', action='store_true')
     return execute

bonobo/commands/version.py

@@ -1,9 +1,40 @@
-import bonobo
+def format_version(mod, *, name=None, quiet=False):
+    from bonobo.util.pkgs import bonobo_packages
+    args = {
+        'name': name or mod.__name__,
+        'version': mod.__version__,
+        'location': bonobo_packages[name or mod.__name__].location
+    }
+
+    if not quiet:
+        return '{name} v.{version} (in {location})'.format(**args)
+    if quiet < 2:
+        return '{name} {version}'.format(**args)
+    if quiet < 3:
+        return '{version}'.format(**args)
+
+    raise RuntimeError('Hard to be so quiet...')
 
 
-def execute():
+def execute(all=False, quiet=False):
-    print('{} v.{}'.format(bonobo.__name__, bonobo.__version__))
+    import bonobo
+    from bonobo.util.pkgs import bonobo_packages
+
+    print(format_version(bonobo, quiet=quiet))
+    if all:
+        for name in sorted(bonobo_packages):
+            if name != 'bonobo':
+                try:
+                    mod = __import__(name.replace('-', '_'))
+                    try:
+                        print(format_version(mod, name=name, quiet=quiet))
+                    except Exception as exc:
+                        print('{} ({})'.format(name, exc))
+                except ImportError as exc:
+                    print('{} is not importable ({}).'.format(name, exc))
 
 
 def register(parser):
+    parser.add_argument('--all', '-a', action='store_true')
+    parser.add_argument('--quiet', '-q', action='count')
     return execute

bonobo/config/__init__.py

@@ -1,13 +1,16 @@
 from bonobo.config.configurables import Configurable
-from bonobo.config.options import Option, Method
+from bonobo.config.options import Method, Option
 from bonobo.config.processors import ContextProcessor
-from bonobo.config.services import Container, Service
+from bonobo.config.services import Container, Exclusive, Service, requires
 
+# bonobo.config public programming interface
 __all__ = [
     'Configurable',
     'Container',
     'ContextProcessor',
-    'Option',
+    'Exclusive',
     'Method',
+    'Option',
     'Service',
+    'requires',
 ]

bonobo/config/configurables.py

@@ -1,6 +1,6 @@
 from bonobo.config.options import Method, Option
 from bonobo.config.processors import ContextProcessor
-from bonobo.errors import ConfigurationError
+from bonobo.errors import ConfigurationError, AbstractError
 
 __all__ = [
     'Configurable',
@@ -26,16 +26,19 @@ class ConfigurableMeta(type):
                     if isinstance(value, ContextProcessor):
                         cls.__processors__.append(value)
                     else:
+                        if not value.name:
+                            value.name = name
+
                         if isinstance(value, Method):
                             if cls.__wrappable__:
                                 raise ConfigurationError(
                                     'Cannot define more than one "Method" option in a configurable. That may change in the future.'
                                 )
                             cls.__wrappable__ = name
-                        if not value.name:
+
-                            value.name = name
                         if not name in cls.__options__:
                             cls.__options__[name] = value
+
                         if value.positional:
                             cls.__positional_options__.append(name)
 
@@ -53,11 +56,9 @@ class Configurable(metaclass=ConfigurableMeta):
 
     def __new__(cls, *args, **kwargs):
         if cls.__wrappable__ and len(args) == 1 and hasattr(args[0], '__call__'):
-            wrapped, args = args[0], args[1:]
+            return type(args[0].__name__, (cls, ), {cls.__wrappable__: args[0]})
-            return type(wrapped.__name__, (cls, ), {cls.__wrappable__: wrapped})
 
-        # XXX is that correct ??? how does it pass args/kwargs to __init__ ???
+        return super(Configurable, cls).__new__(cls)
-        return super().__new__(cls)
 
     def __init__(self, *args, **kwargs):
         super().__init__()

bonobo/config/options.py

@@ -67,12 +67,12 @@ class Option:
     def __set__(self, inst, value):
         inst.__options_values__[self.name] = self.clean(value)
 
-    def get_default(self):
-        return self.default() if callable(self.default) else self.default
-
     def clean(self, value):
         return self.type(value) if self.type else value
 
+    def get_default(self):
+        return self.default() if callable(self.default) else self.default
+
 
 class Method(Option):
     """
@@ -106,7 +106,7 @@ class Method(Option):
     """
 
     def __init__(self):
-        super().__init__(None, required=False, positional=True)
+        super().__init__(None, required=False)
 
     def __get__(self, inst, typ):
         if not self.name in inst.__options_values__:
@@ -114,6 +114,8 @@ class Method(Option):
         return inst.__options_values__[self.name]
 
     def __set__(self, inst, value):
+        if isinstance(value, str):
+            raise ValueError('should be callable')
         inst.__options_values__[self.name] = self.type(value) if self.type else value
 
     def clean(self, value):

bonobo/config/processors.py

@@ -1,4 +1,3 @@
-import types
 from collections import Iterable
 from contextlib import contextmanager
 
@@ -10,6 +9,34 @@ _CONTEXT_PROCESSORS_ATTR = '__processors__'
 
 
 class ContextProcessor(Option):
+    """
+    A ContextProcessor is a kind of transformation decorator that can setup and teardown a transformation and runtime
+    related dependencies, at the execution level.
+
+    It works like a yielding context manager, and is the recommended way to setup and teardown objects you'll need
+    in the context of one execution. It's the way to overcome the stateless nature of transformations.
+
+    The yielded values will be passed as positional arguments to the next context processors (order do matter), and
+    finally to the __call__ method of the transformation.
+
+    Warning: this may change for a similar but simpler implementation, don't relly too much on it (yet).
+
+    Example:
+
+        >>> from bonobo.config import Configurable
+        >>> from bonobo.util.objects import ValueHolder
+
+        >>> class Counter(Configurable):
+        ...     @ContextProcessor
+        ...     def counter(self, context):
+        ...         yield ValueHolder(0)
+        ...
+        ...     def __call__(self, counter, *args, **kwargs):
+        ...         counter += 1
+        ...         yield counter.get()
+
+    """
+
     @property
     def __name__(self):
         return self.func.__name__
@@ -104,14 +131,7 @@ def resolve_processors(mixed):
     try:
         yield from mixed.__processors__
     except AttributeError:
-        # old code, deprecated usage
+        yield from ()
-        if isinstance(mixed, types.FunctionType):
-            yield from getattr(mixed, _CONTEXT_PROCESSORS_ATTR, ())
-
-        for cls in reversed((mixed if isinstance(mixed, type) else type(mixed)).__mro__):
-            yield from cls.__dict__.get(_CONTEXT_PROCESSORS_ATTR, ())
-
-    return ()
 
 
 get_context_processors = deprecated_alias('get_context_processors', resolve_processors)

bonobo/config/services.py

@@ -1,7 +1,10 @@
 import re
+import threading
 import types
+from contextlib import ContextDecorator
 
 from bonobo.config.options import Option
+from bonobo.errors import MissingServiceImplementationError
 
 _service_name_re = re.compile(r"^[^\d\W]\w*(:?\.[^\d\W]\w*)*$", re.UNICODE)
 
@@ -53,7 +56,11 @@ class Service(Option):
         inst.__options_values__[self.name] = validate_service_name(value)
 
     def resolve(self, inst, services):
-        return services.get(getattr(inst, self.name))
+        try:
+            name = getattr(inst, self.name)
+        except AttributeError:
+            name = self.name
+        return services.get(name)
 
 
 class Container(dict):
@@ -78,8 +85,64 @@ class Container(dict):
         if not name in self:
             if default:
                 return default
-            raise KeyError('Cannot resolve service {!r} using provided service collection.'.format(name))
+            raise MissingServiceImplementationError(
+                'Cannot resolve service {!r} using provided service collection.'.format(name)
+            )
         value = super().get(name)
+        # XXX this is not documented and can lead to errors.
         if isinstance(value, types.LambdaType):
             value = value(self)
         return value
+
+
+class Exclusive(ContextDecorator):
+    """
+    Decorator and context manager used to require exclusive usage of an object, most probably a service. It's usefull
+    for example if call order matters on a service implementation (think of an http api that requires a nonce or version
+    parameter ...).
+
+    Usage:
+
+        >>> def handler(some_service):
+        ...     with Exclusive(some_service):
+        ...         some_service.call_1()
+        ...         some_service.call_2()
+        ...         some_service.call_3()
+
+    This will ensure that nobody else is using the same service while in the "with" block, using a lock primitive to
+    ensure that.
+
+    """
+    _locks = {}
+
+    def __init__(self, wrapped):
+        self._wrapped = wrapped
+
+    def get_lock(self):
+        _id = id(self._wrapped)
+        if not _id in Exclusive._locks:
+            Exclusive._locks[_id] = threading.RLock()
+        return Exclusive._locks[_id]
+
+    def __enter__(self):
+        self.get_lock().acquire()
+        return self._wrapped
+
+    def __exit__(self, *exc):
+        self.get_lock().release()
+
+
+def requires(*service_names):
+    def decorate(mixed):
+        try:
+            options = mixed.__options__
+        except AttributeError:
+            mixed.__options__ = options = {}
+
+        for service_name in service_names:
+            service = Service(service_name)
+            service.name = service_name
+            options[service_name] = service
+        return mixed
+
+    return decorate

bonobo/errors.py

@@ -56,3 +56,7 @@ class ProhibitedOperationError(RuntimeError):
 
 class ConfigurationError(Exception):
     pass
+
+
+class MissingServiceImplementationError(KeyError):
+    pass

bonobo/examples/datasets/coffeeshops.json

@@ -1,182 +1,182 @@
-{"O q de poule": "53 rue du ruisseau, 75018 Paris, France",
+{"Le Reynou": "2 bis quai de la m\u00e9gisserie, 75001 Paris, France",
+"les montparnos": "65 boulevard Pasteur, 75015 Paris, France",
+"Le Saint Jean": "23 rue des abbesses, 75018 Paris, France",
+"Le Felteu": "1 rue Pecquay, 75004 Paris, France",
+"O q de poule": "53 rue du ruisseau, 75018 Paris, France",
 "Le chantereine": "51 Rue Victoire, 75009 Paris, France",
+"Le M\u00fcller": "11 rue Feutrier, 75018 Paris, France",
 "La Caravane": "Rue de la Fontaine au Roi, 75011 Paris, France",
 "Le Pas Sage": "1 Passage du Grand Cerf, 75002 Paris, France",
 "La Renaissance": "112 Rue Championnet, 75018 Paris, France",
 "Ext\u00e9rieur Quai": "5, rue d'Alsace, 75010 Paris, France",
-"Le Reynou": "2 bis quai de la m\u00e9gisserie, 75001 Paris, France",
+"Le Sully": "6 Bd henri IV, 75004 Paris, France",
-"les montparnos": "65 boulevard Pasteur, 75015 Paris, France",
+"Le drapeau de la fidelit\u00e9": "21 rue Copreaux, 75015 Paris, France",
+"Le caf\u00e9 des amis": "125 rue Blomet, 75015 Paris, France",
+"Le Kleemend's": "34 avenue Pierre Mend\u00e8s-France, 75013 Paris, France",
+"Assaporare Dix sur Dix": "75, avenue Ledru-Rollin, 75012 Paris, France",
+"Caf\u00e9 Pierre": "202 rue du faubourg st antoine, 75012 Paris, France",
+"Le Caf\u00e9 Livres": "10 rue Saint Martin, 75004 Paris, France",
+"Le Chaumontois": "12 rue Armand Carrel, 75018 Paris, France",
+"Le Square": "31 rue Saint-Dominique, 75007 Paris, France",
+"Les Arcades": "61 rue de Ponthieu, 75008 Paris, France",
+"Le Bosquet": "46 avenue Bosquet, 75007 Paris, France",
 "Le bistrot de Ma\u00eblle et Augustin": "42 rue coquill\u00e8re, 75001 Paris, France",
 "D\u00e9d\u00e9 la frite": "52 rue Notre-Dame des Victoires, 75002 Paris, France",
-"Assaporare Dix sur Dix": "75, avenue Ledru-Rollin, 75012 Paris, France",
 "Cardinal Saint-Germain": "11 boulevard Saint-Germain, 75005 Paris, France",
 "Caf\u00e9 antoine": "17 rue Jean de la Fontaine, 75016 Paris, France",
 "Au cerceau d'or": "129 boulevard sebastopol, 75002 Paris, France",
-"Le Chaumontois": "12 rue Armand Carrel, 75018 Paris, France",
 "Aux cadrans": "21 ter boulevard Diderot, 75012 Paris, France",
-"Le Saint Jean": "23 rue des abbesses, 75018 Paris, France",
-"Le Square": "31 rue Saint-Dominique, 75007 Paris, France",
-"Les Arcades": "61 rue de Ponthieu, 75008 Paris, France",
 "Caf\u00e9 Lea": "5 rue Claude Bernard, 75005 Paris, France",
 "Le Bellerive": "71 quai de Seine, 75019 Paris, France",
 "La Bauloise": "36 rue du hameau, 75015 Paris, France",
 "Le Dellac": "14 rue Rougemont, 75009 Paris, France",
-"Le Bosquet": "46 avenue Bosquet, 75007 Paris, France",
-"Le Sully": "6 Bd henri IV, 75004 Paris, France",
-"Le Felteu": "1 rue Pecquay, 75004 Paris, France",
-"Le drapeau de la fidelit\u00e9": "21 rue Copreaux, 75015 Paris, France",
-"Le caf\u00e9 des amis": "125 rue Blomet, 75015 Paris, France",
-"Le Kleemend's": "34 avenue Pierre Mend\u00e8s-France, 75013 Paris, France",
-"Caf\u00e9 Pierre": "202 rue du faubourg st antoine, 75012 Paris, France",
-"Le M\u00fcller": "11 rue Feutrier, 75018 Paris, France",
-"Le Caf\u00e9 Livres": "10 rue Saint Martin, 75004 Paris, France",
-"La Cordonnerie": "142 Rue Saint-Denis 75002 Paris, 75002 Paris, France",
-"Invitez vous chez nous": "7 rue Ep\u00e9e de Bois, 75005 Paris, France",
-"Au bon coin": "49 rue des Cloys, 75018 Paris, France",
-"La Br\u00fblerie des Ternes": "111 rue mouffetard, 75005 Paris, France",
-"Le Petit Choiseul": "23 rue saint augustin, 75002 Paris, France",
-"O'Breizh": "27 rue de Penthi\u00e8vre, 75008 Paris, France",
-"Le Supercoin": "3, rue Baudelique, 75018 Paris, France",
-"Populettes": "86 bis rue Riquet, 75018 Paris, France",
-"Le Couvent": "69 rue Broca, 75013 Paris, France",
-"Caf\u00e9 Zen": "46 rue Victoire, 75009 Paris, France",
-"Le Chat bossu": "126, rue du Faubourg Saint Antoine, 75012 Paris, France",
-"Le petit club": "55 rue de la tombe Issoire, 75014 Paris, France",
-"Le Relais Haussmann": "146, boulevard Haussmann, 75008 Paris, France",
-"Denfert caf\u00e9": "58 boulvevard Saint Jacques, 75014 Paris, France",
-"Bagels & Coffee Corner": "Place de Clichy, 75017 Paris, France",
-"Le Plein soleil": "90 avenue Parmentier, 75011 Paris, France",
-"La Perle": "78 rue vieille du temple, 75003 Paris, France",
-"Le Caf\u00e9 frapp\u00e9": "95 rue Montmartre, 75002 Paris, France",
-"L'\u00c9cir": "59 Boulevard Saint-Jacques, 75014 Paris, France",
-"Le Descartes": "1 rue Thouin, 75005 Paris, France",
 "Br\u00fblerie San Jos\u00e9": "30 rue des Petits-Champs, 75002 Paris, France",
 "Caf\u00e9 de la Mairie (du VIII)": "rue de Lisbonne, 75008 Paris, France",
-"Au panini de la place": "47 rue Belgrand, 75020 Paris, France",
+"Le General Beuret": "9 Place du General Beuret, 75015 Paris, France",
-"Extra old caf\u00e9": "307 fg saint Antoine, 75011 Paris, France",
+"Le Cap Bourbon": "1 rue Louis le Grand, 75002 Paris, France",
 "En attendant l'or": "3 rue Faidherbe, 75011 Paris, France",
-"Le Pure caf\u00e9": "14 rue Jean Mac\u00e9, 75011 Paris, France",
+"Caf\u00e9 Martin": "2 place Martin Nadaud, 75001 Paris, France",
-"Le Village": "182 rue de Courcelles, 75017 Paris, France",
+"Etienne": "14 rue Turbigo, Paris, 75001 Paris, France",
-"Le Malar": "88 rue Saint-Dominique, 75007 Paris, France",
+"L'ing\u00e9nu": "184 bd Voltaire, 75011 Paris, France",
-"Pause Caf\u00e9": "41 rue de Charonne, 75011 Paris, France",
+"Le Biz": "18 rue Favart, 75002 Paris, France",
-"Chez Fafa": "44 rue Vinaigriers, 75010 Paris, France",
+"L'Olive": "8 rue L'Olive, 75018 Paris, France",
-"La Recoleta au Manoir": "229 avenue Gambetta, 75020 Paris, France",
+"Le pari's caf\u00e9": "104 rue caulaincourt, 75018 Paris, France",
-"Le Pareloup": "80 Rue Saint-Charles, 75015 Paris, France",
+"Le Poulailler": "60 rue saint-sabin, 75011 Paris, France",
 "La Marine": "55 bis quai de valmy, 75010 Paris, France",
 "American Kitchen": "49 rue bichat, 75010 Paris, France",
+"Chai 33": "33 Cour Saint Emilion, 75012 Paris, France",
 "Face Bar": "82 rue des archives, 75003 Paris, France",
 "Le Bloc": "21 avenue Brochant, 75017 Paris, France",
 "La Bricole": "52 rue Liebniz, 75018 Paris, France",
 "le ronsard": "place maubert, 75005 Paris, France",
 "l'Usine": "1 rue d'Avron, 75020 Paris, France",
-"La Brasserie Gait\u00e9": "3 rue de la Gait\u00e9, 75014 Paris, France",
+"La Cordonnerie": "142 Rue Saint-Denis 75002 Paris, 75002 Paris, France",
-"Le General Beuret": "9 Place du General Beuret, 75015 Paris, France",
+"Invitez vous chez nous": "7 rue Ep\u00e9e de Bois, 75005 Paris, France",
-"Le Cap Bourbon": "1 rue Louis le Grand, 75002 Paris, France",
+"Le sully": "13 rue du Faubourg Saint Denis, 75010 Paris, France",
 "Le Ragueneau": "202 rue Saint-Honor\u00e9, 75001 Paris, France",
 "Le Germinal": "95 avenue Emile Zola, 75015 Paris, France",
-"Caf\u00e9 Martin": "2 place Martin Nadaud, 75001 Paris, France",
-"Etienne": "14 rue Turbigo, Paris, 75001 Paris, France",
-"L'ing\u00e9nu": "184 bd Voltaire, 75011 Paris, France",
 "Le refuge": "72 rue lamarck, 75018 Paris, France",
-"Le Biz": "18 rue Favart, 75002 Paris, France",
-"L'Olive": "8 rue L'Olive, 75018 Paris, France",
-"Le sully": "13 rue du Faubourg Saint Denis, 75010 Paris, France",
 "Drole d'endroit pour une rencontre": "58 rue de Montorgueil, 75002 Paris, France",
+"Le Petit Choiseul": "23 rue saint augustin, 75002 Paris, France",
+"O'Breizh": "27 rue de Penthi\u00e8vre, 75008 Paris, France",
+"Le Supercoin": "3, rue Baudelique, 75018 Paris, France",
+"Populettes": "86 bis rue Riquet, 75018 Paris, France",
+"La Recoleta au Manoir": "229 avenue Gambetta, 75020 Paris, France",
 "L'Assassin": "99 rue Jean-Pierre Timbaud, 75011 Paris, France",
-"Le pari's caf\u00e9": "104 rue caulaincourt, 75018 Paris, France",
+"Le Pareloup": "80 Rue Saint-Charles, 75015 Paris, France",
-"Le Poulailler": "60 rue saint-sabin, 75011 Paris, France",
+"Caf\u00e9 Zen": "46 rue Victoire, 75009 Paris, France",
-"Chai 33": "33 Cour Saint Emilion, 75012 Paris, France",
+"La Brasserie Gait\u00e9": "3 rue de la Gait\u00e9, 75014 Paris, France",
+"Au bon coin": "49 rue des Cloys, 75018 Paris, France",
+"La Br\u00fblerie des Ternes": "111 rue mouffetard, 75005 Paris, France",
+"Le Chat bossu": "126, rue du Faubourg Saint Antoine, 75012 Paris, France",
+"Denfert caf\u00e9": "58 boulvevard Saint Jacques, 75014 Paris, France",
+"Le Couvent": "69 rue Broca, 75013 Paris, France",
+"Bagels & Coffee Corner": "Place de Clichy, 75017 Paris, France",
+"La Perle": "78 rue vieille du temple, 75003 Paris, France",
+"Le Caf\u00e9 frapp\u00e9": "95 rue Montmartre, 75002 Paris, France",
+"L'\u00c9cir": "59 Boulevard Saint-Jacques, 75014 Paris, France",
+"Le Descartes": "1 rue Thouin, 75005 Paris, France",
+"Le petit club": "55 rue de la tombe Issoire, 75014 Paris, France",
+"Le Relais Haussmann": "146, boulevard Haussmann, 75008 Paris, France",
+"Au panini de la place": "47 rue Belgrand, 75020 Paris, France",
+"Extra old caf\u00e9": "307 fg saint Antoine, 75011 Paris, France",
+"Le Plein soleil": "90 avenue Parmentier, 75011 Paris, France",
+"Le Pure caf\u00e9": "14 rue Jean Mac\u00e9, 75011 Paris, France",
+"Le Village": "182 rue de Courcelles, 75017 Paris, France",
+"Le Malar": "88 rue Saint-Dominique, 75007 Paris, France",
+"Pause Caf\u00e9": "41 rue de Charonne, 75011 Paris, France",
+"Chez Fafa": "44 rue Vinaigriers, 75010 Paris, France",
 "Caf\u00e9 dans l'aerogare Air France Invalides": "2 rue Robert Esnault Pelterie, 75007 Paris, France",
+"Le relais de la victoire": "73 rue de la Victoire, 75009 Paris, France",
+"Caprice caf\u00e9": "12 avenue Jean Moulin, 75014 Paris, France",
+"Caves populaires": "22 rue des Dames, 75017 Paris, France",
+"Cafe de grenelle": "188 rue de Grenelle, 75007 Paris, France",
+"Chez Prune": "36 rue Beaurepaire, 75010 Paris, France",
+"L'anjou": "1 rue de Montholon, 75009 Paris, France",
+"Le Brio": "216, rue Marcadet, 75018 Paris, France",
+"Tamm Bara": "7 rue Clisson, 75013 Paris, France",
+"La chaumi\u00e8re gourmande": "Route de la Muette \u00e0 Neuilly",
+"Club hippique du Jardin d\u2019Acclimatation": "75016 Paris, France",
 "Les P\u00e8res Populaires": "46 rue de Buzenval, 75020 Paris, France",
 "Epicerie Musicale": "55bis quai de Valmy, 75010 Paris, France",
-"Le relais de la victoire": "73 rue de la Victoire, 75009 Paris, France",
 "Le Centenaire": "104 rue amelot, 75011 Paris, France",
-"Caf\u00e9 Pistache": "9 rue des petits champs, 75001 Paris, France",
+"Le Zazabar": "116 Rue de M\u00e9nilmontant, 75020 Paris, France",
-"La Cagnotte": "13 Rue Jean-Baptiste Dumay, 75020 Paris, France",
-"Le bal du pirate": "60 rue des bergers, 75015 Paris, France",
-"bistrot les timbr\u00e9s": "14 rue d'alleray, 75015 Paris, France",
-"Le Killy Jen": "28 bis boulevard Diderot, 75012 Paris, France",
-"le 1 cinq": "172 rue de vaugirard, 75015 Paris, France",
-"Les Artisans": "106 rue Lecourbe, 75015 Paris, France",
-"Peperoni": "83 avenue de Wagram, 75001 Paris, France",
 "Ragueneau": "202 rue Saint Honor\u00e9, 75001 Paris, France",
-"l'orillon bar": "35 rue de l'orillon, 75011 Paris, France",
-"zic zinc": "95 rue claude decaen, 75012 Paris, France",
 "L'In\u00e9vitable": "22 rue Linn\u00e9, 75005 Paris, France",
-"Le Brio": "216, rue Marcadet, 75018 Paris, France",
 "Le Dunois": "77 rue Dunois, 75013 Paris, France",
 "La Montagne Sans Genevi\u00e8ve": "13 Rue du Pot de Fer, 75005 Paris, France",
 "Le Caminito": "48 rue du Dessous des Berges, 75013 Paris, France",
 "Le petit Bretonneau": "Le petit Bretonneau - \u00e0 l'int\u00e9rieur de l'H\u00f4pital, 75018 Paris, France",
-"La chaumi\u00e8re gourmande": "Route de la Muette \u00e0 Neuilly",
+"Le bal du pirate": "60 rue des bergers, 75015 Paris, France",
-"Club hippique du Jardin d\u2019Acclimatation": "75016 Paris, France",
-"Caprice caf\u00e9": "12 avenue Jean Moulin, 75014 Paris, France",
-"Le Zazabar": "116 Rue de M\u00e9nilmontant, 75020 Paris, France",
-"Caf\u00e9 beauveau": "9 rue de Miromesnil, 75008 Paris, France",
-"Caves populaires": "22 rue des Dames, 75017 Paris, France",
-"Cafe de grenelle": "188 rue de Grenelle, 75007 Paris, France",
-"Au Vin Des Rues": "21 rue Boulard, 75014 Paris, France",
 "L'antre d'eux": "16 rue DE MEZIERES, 75006 Paris, France",
-"Chez Prune": "36 rue Beaurepaire, 75010 Paris, France",
+"l'orillon bar": "35 rue de l'orillon, 75011 Paris, France",
-"L'anjou": "1 rue de Montholon, 75009 Paris, France",
+"zic zinc": "95 rue claude decaen, 75012 Paris, France",
-"Tamm Bara": "7 rue Clisson, 75013 Paris, France",
+"Caf\u00e9 Pistache": "9 rue des petits champs, 75001 Paris, France",
-"La cantoche de Paname": "40 Boulevard Beaumarchais, 75011 Paris, France",
+"La Cagnotte": "13 Rue Jean-Baptiste Dumay, 75020 Paris, France",
-"Le Saint Ren\u00e9": "148 Boulevard de Charonne, 75020 Paris, France",
+"bistrot les timbr\u00e9s": "14 rue d'alleray, 75015 Paris, France",
-"Caf\u00e9 Clochette": "16 avenue Richerand, 75010 Paris, France",
+"Le Killy Jen": "28 bis boulevard Diderot, 75012 Paris, France",
-"L'europ\u00e9en": "21 Bis Boulevard Diderot, 75012 Paris, France",
+"Caf\u00e9 beauveau": "9 rue de Miromesnil, 75008 Paris, France",
+"le 1 cinq": "172 rue de vaugirard, 75015 Paris, France",
+"Au Vin Des Rues": "21 rue Boulard, 75014 Paris, France",
+"Les Artisans": "106 rue Lecourbe, 75015 Paris, France",
+"Peperoni": "83 avenue de Wagram, 75001 Paris, France",
 "Le BB (Bouchon des Batignolles)": "2 rue Lemercier, 75017 Paris, France",
 "La Libert\u00e9": "196 rue du faubourg saint-antoine, 75012 Paris, France",
 "Chez Rutabaga": "16 rue des Petits Champs, 75002 Paris, France",
-"Les caves populaires": "22 rue des Dames, 75017 Paris, France",
+"La cantoche de Paname": "40 Boulevard Beaumarchais, 75011 Paris, France",
-"Le Plomb du cantal": "3 rue Ga\u00eet\u00e9, 75014 Paris, France",
+"Le Saint Ren\u00e9": "148 Boulevard de Charonne, 75020 Paris, France",
-"Bistrot Saint-Antoine": "58 rue du Fbg Saint-Antoine, 75012 Paris, France",
-"Trois pi\u00e8ces cuisine": "101 rue des dames, 75017 Paris, France",
 "La Brocante": "10 rue Rossini, 75009 Paris, France",
-"Le Zinc": "61 avenue de la Motte Picquet, 75015 Paris, France",
+"Caf\u00e9 Clochette": "16 avenue Richerand, 75010 Paris, France",
-"Chez Oscar": "11/13 boulevard Beaumarchais, 75004 Paris, France",
+"L'europ\u00e9en": "21 Bis Boulevard Diderot, 75012 Paris, France",
-"Le Piquet": "48 avenue de la Motte Picquet, 75015 Paris, France",
-"L'avant comptoir": "3 carrefour de l'Od\u00e9on, 75006 Paris, France",
-"le chateau d'eau": "67 rue du Ch\u00e2teau d'eau, 75010 Paris, France",
-"maison du vin": "52 rue des plantes, 75014 Paris, France",
-"Les Vendangeurs": "6/8 rue Stanislas, 75006 Paris, France",
 "NoMa": "39 rue Notre Dame de Nazareth, 75003 Paris, France",
-"Chez Luna": "108 rue de M\u00e9nilmontant, 75020 Paris, France",
-"Le Tournebride": "104 rue Mouffetard, 75005 Paris, France",
-"le lutece": "380 rue de vaugirard, 75015 Paris, France",
-"Le bar Fleuri": "1 rue du Plateau, 75019 Paris, France",
-"Le Fronton": "63 rue de Ponthieu, 75008 Paris, France",
 "O'Paris": "1 Rue des Envierges, 75020 Paris, France",
-"Rivolux": "16 rue de Rivoli, 75004 Paris, France",
-"Brasiloja": "16 rue Ganneron, 75018 Paris, France",
 "Botak cafe": "1 rue Paul albert, 75018 Paris, France",
 "La cantine de Zo\u00e9": "136 rue du Faubourg poissonni\u00e8re, 75010 Paris, France",
-"Institut des Cultures d'Islam": "19-23 rue L\u00e9on, 75018 Paris, France",
+"Les caves populaires": "22 rue des Dames, 75017 Paris, France",
-"Chez Miamophile": "6 rue M\u00e9lingue, 75019 Paris, France",
+"Le Plomb du cantal": "3 rue Ga\u00eet\u00e9, 75014 Paris, France",
-"Canopy Caf\u00e9 associatif": "19 rue Pajol, 75018 Paris, France",
+"Trois pi\u00e8ces cuisine": "101 rue des dames, 75017 Paris, France",
-"Caf\u00e9 rallye tournelles": "11 Quai de la Tournelle, 75005 Paris, France",
+"Le Zinc": "61 avenue de la Motte Picquet, 75015 Paris, France",
-"Petits Freres des Pauvres": "47 rue de Batignolles, 75017 Paris, France",
+"L'avant comptoir": "3 carrefour de l'Od\u00e9on, 75006 Paris, France",
+"Les Vendangeurs": "6/8 rue Stanislas, 75006 Paris, France",
+"Chez Luna": "108 rue de M\u00e9nilmontant, 75020 Paris, France",
+"Le bar Fleuri": "1 rue du Plateau, 75019 Paris, France",
+"Bistrot Saint-Antoine": "58 rue du Fbg Saint-Antoine, 75012 Paris, France",
+"Chez Oscar": "11/13 boulevard Beaumarchais, 75004 Paris, France",
+"Le Piquet": "48 avenue de la Motte Picquet, 75015 Paris, France",
+"le chateau d'eau": "67 rue du Ch\u00e2teau d'eau, 75010 Paris, France",
+"maison du vin": "52 rue des plantes, 75014 Paris, France",
+"Le Tournebride": "104 rue Mouffetard, 75005 Paris, France",
+"Le Fronton": "63 rue de Ponthieu, 75008 Paris, France",
+"le lutece": "380 rue de vaugirard, 75015 Paris, France",
+"Rivolux": "16 rue de Rivoli, 75004 Paris, France",
+"Brasiloja": "16 rue Ganneron, 75018 Paris, France",
 "Le caf\u00e9 Monde et M\u00e9dias": "Place de la R\u00e9publique, 75003 Paris, France",
 "L'entrep\u00f4t": "157 rue Bercy 75012 Paris, 75012 Paris, France",
 "Coffee Chope": "344Vrue Vaugirard, 75015 Paris, France",
-"Le Comptoir": "354 bis rue Vaugirard, 75015 Paris, France",
-"L'empreinte": "54, avenue Daumesnil, 75012 Paris, France",
-"Caf\u00e9 Victor": "10 boulevard Victor, 75015 Paris, France",
-"Caf\u00e9 Varenne": "36 rue de Varenne, 75007 Paris, France",
-"Le Brigadier": "12 rue Blanche, 75009 Paris, France",
-"Melting Pot": "3 rue de Lagny, 75020 Paris, France",
-"L'Entracte": "place de l'opera, 75002 Paris, France",
-"le Zango": "58 rue Daguerre, 75014 Paris, France",
-"Panem": "18 rue de Crussol, 75011 Paris, France",
-"Waikiki": "10 rue d\"Ulm, 75005 Paris, France",
 "l'El\u00e9phant du nil": "125 Rue Saint-Antoine, 75004 Paris, France",
 "Le Parc Vaugirard": "358 rue de Vaugirard, 75015 Paris, France",
 "Pari's Caf\u00e9": "174 avenue de Clichy, 75017 Paris, France",
+"Le Comptoir": "354 bis rue Vaugirard, 75015 Paris, France",
+"Caf\u00e9 Varenne": "36 rue de Varenne, 75007 Paris, France",
+"Melting Pot": "3 rue de Lagny, 75020 Paris, France",
+"le Zango": "58 rue Daguerre, 75014 Paris, France",
+"Chez Miamophile": "6 rue M\u00e9lingue, 75019 Paris, France",
+"Institut des Cultures d'Islam": "19-23 rue L\u00e9on, 75018 Paris, France",
+"Canopy Caf\u00e9 associatif": "19 rue Pajol, 75018 Paris, France",
+"Caf\u00e9 rallye tournelles": "11 Quai de la Tournelle, 75005 Paris, France",
+"Petits Freres des Pauvres": "47 rue de Batignolles, 75017 Paris, France",
 "Brasserie le Morvan": "61 rue du ch\u00e2teau d'eau, 75010 Paris, France",
+"L'Angle": "28 rue de Ponthieu, 75008 Paris, France",
+"Caf\u00e9 Dupont": "198 rue de la Convention, 75015 Paris, France",
+"L'Entracte": "place de l'opera, 75002 Paris, France",
+"Panem": "18 rue de Crussol, 75011 Paris, France",
 "Au pays de Vannes": "34 bis rue de Wattignies, 75012 Paris, France",
 "Le Lucernaire": "53 rue Notre-Dame des Champs, 75006 Paris, France",
-"L'Angle": "28 rue de Ponthieu, 75008 Paris, France",
 "Le Caf\u00e9 d'avant": "35 rue Claude Bernard, 75005 Paris, France",
-"Caf\u00e9 Dupont": "198 rue de la Convention, 75015 Paris, France",
 "L'\u00e2ge d'or": "26 rue du Docteur Magnan, 75013 Paris, France",
 "Le S\u00e9vign\u00e9": "15 rue du Parc Royal, 75003 Paris, France",
-"L'horizon": "93, rue de la Roquette, 75011 Paris, France"}
+"L'horizon": "93, rue de la Roquette, 75011 Paris, France",
+"L'empreinte": "54, avenue Daumesnil, 75012 Paris, France",
+"Caf\u00e9 Victor": "10 boulevard Victor, 75015 Paris, France",
+"Le Brigadier": "12 rue Blanche, 75009 Paris, France",
+"Waikiki": "10 rue d\"Ulm, 75005 Paris, France"}

bonobo/examples/datasets/coffeeshops.txt

@@ -1,182 +1,182 @@
+Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
+les montparnos, 65 boulevard Pasteur, 75015 Paris, France
+Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
+Le Felteu, 1 rue Pecquay, 75004 Paris, France
 O q de poule, 53 rue du ruisseau, 75018 Paris, France
 Le chantereine, 51 Rue Victoire, 75009 Paris, France
+Le Müller, 11 rue Feutrier, 75018 Paris, France
 La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France
 Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France
 La Renaissance, 112 Rue Championnet, 75018 Paris, France
 Extérieur Quai, 5, rue d'Alsace, 75010 Paris, France
-Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
+Le Sully, 6 Bd henri IV, 75004 Paris, France
-les montparnos, 65 boulevard Pasteur, 75015 Paris, France
+Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
+Le café des amis, 125 rue Blomet, 75015 Paris, France
+Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
+Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
+Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
+Le Café Livres, 10 rue Saint Martin, 75004 Paris, France
+Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France
+Le Square, 31 rue Saint-Dominique, 75007 Paris, France
+Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
+Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
 Le bistrot de Maëlle et Augustin, 42 rue coquillère, 75001 Paris, France
 Dédé la frite, 52 rue Notre-Dame des Victoires, 75002 Paris, France
-Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
 Cardinal Saint-Germain, 11 boulevard Saint-Germain, 75005 Paris, France
 Café antoine, 17 rue Jean de la Fontaine, 75016 Paris, France
 Au cerceau d'or, 129 boulevard sebastopol, 75002 Paris, France
-Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France
 Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France
-Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
-Le Square, 31 rue Saint-Dominique, 75007 Paris, France
-Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
 Café Lea, 5 rue Claude Bernard, 75005 Paris, France
 Le Bellerive, 71 quai de Seine, 75019 Paris, France
 La Bauloise, 36 rue du hameau, 75015 Paris, France
 Le Dellac, 14 rue Rougemont, 75009 Paris, France
-Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
-Le Sully, 6 Bd henri IV, 75004 Paris, France
-Le Felteu, 1 rue Pecquay, 75004 Paris, France
-Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
-Le café des amis, 125 rue Blomet, 75015 Paris, France
-Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
-Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
-Le Müller, 11 rue Feutrier, 75018 Paris, France
-Le Café Livres, 10 rue Saint Martin, 75004 Paris, France
-La Cordonnerie, 142 Rue Saint-Denis 75002 Paris, 75002 Paris, France
-Invitez vous chez nous, 7 rue Epée de Bois, 75005 Paris, France
-Au bon coin, 49 rue des Cloys, 75018 Paris, France
-La Brûlerie des Ternes, 111 rue mouffetard, 75005 Paris, France
-Le Petit Choiseul, 23 rue saint augustin, 75002 Paris, France
-O'Breizh, 27 rue de Penthièvre, 75008 Paris, France
-Le Supercoin, 3, rue Baudelique, 75018 Paris, France
-Populettes, 86 bis rue Riquet, 75018 Paris, France
-Le Couvent, 69 rue Broca, 75013 Paris, France
-Café Zen, 46 rue Victoire, 75009 Paris, France
-Le Chat bossu, 126, rue du Faubourg Saint Antoine, 75012 Paris, France
-Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France
-Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France
-Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France
-Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
-Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France
-La Perle, 78 rue vieille du temple, 75003 Paris, France
-Le Café frappé, 95 rue Montmartre, 75002 Paris, France
-L'Écir, 59 Boulevard Saint-Jacques, 75014 Paris, France
-Le Descartes, 1 rue Thouin, 75005 Paris, France
 Brûlerie San José, 30 rue des Petits-Champs, 75002 Paris, France
 Café de la Mairie (du VIII), rue de Lisbonne, 75008 Paris, France
-Au panini de la place, 47 rue Belgrand, 75020 Paris, France
+Le General Beuret, 9 Place du General Beuret, 75015 Paris, France
-Extra old café, 307 fg saint Antoine, 75011 Paris, France
+Le Cap Bourbon, 1 rue Louis le Grand, 75002 Paris, France
 En attendant l'or, 3 rue Faidherbe, 75011 Paris, France
-Le Pure café, 14 rue Jean Macé, 75011 Paris, France
+Café Martin, 2 place Martin Nadaud, 75001 Paris, France
-Le Village, 182 rue de Courcelles, 75017 Paris, France
+Etienne, 14 rue Turbigo, Paris, 75001 Paris, France
-Le Malar, 88 rue Saint-Dominique, 75007 Paris, France
+L'ingénu, 184 bd Voltaire, 75011 Paris, France
-Pause Café, 41 rue de Charonne, 75011 Paris, France
+Le Biz, 18 rue Favart, 75002 Paris, France
-Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France
+L'Olive, 8 rue L'Olive, 75018 Paris, France
-La Recoleta au Manoir, 229 avenue Gambetta, 75020 Paris, France
+Le pari's café, 104 rue caulaincourt, 75018 Paris, France
-Le Pareloup, 80 Rue Saint-Charles, 75015 Paris, France
+Le Poulailler, 60 rue saint-sabin, 75011 Paris, France
 La Marine, 55 bis quai de valmy, 75010 Paris, France
 American Kitchen, 49 rue bichat, 75010 Paris, France
+Chai 33, 33 Cour Saint Emilion, 75012 Paris, France
 Face Bar, 82 rue des archives, 75003 Paris, France
 Le Bloc, 21 avenue Brochant, 75017 Paris, France
 La Bricole, 52 rue Liebniz, 75018 Paris, France
 le ronsard, place maubert, 75005 Paris, France
 l'Usine, 1 rue d'Avron, 75020 Paris, France
-La Brasserie Gaité, 3 rue de la Gaité, 75014 Paris, France
+La Cordonnerie, 142 Rue Saint-Denis 75002 Paris, 75002 Paris, France
-Le General Beuret, 9 Place du General Beuret, 75015 Paris, France
+Invitez vous chez nous, 7 rue Epée de Bois, 75005 Paris, France
-Le Cap Bourbon, 1 rue Louis le Grand, 75002 Paris, France
+Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France
 Le Ragueneau, 202 rue Saint-Honoré, 75001 Paris, France
 Le Germinal, 95 avenue Emile Zola, 75015 Paris, France
-Café Martin, 2 place Martin Nadaud, 75001 Paris, France
-Etienne, 14 rue Turbigo, Paris, 75001 Paris, France
-L'ingénu, 184 bd Voltaire, 75011 Paris, France
 Le refuge, 72 rue lamarck, 75018 Paris, France
-Le Biz, 18 rue Favart, 75002 Paris, France
-L'Olive, 8 rue L'Olive, 75018 Paris, France
-Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France
 Drole d'endroit pour une rencontre, 58 rue de Montorgueil, 75002 Paris, France
+Le Petit Choiseul, 23 rue saint augustin, 75002 Paris, France
+O'Breizh, 27 rue de Penthièvre, 75008 Paris, France
+Le Supercoin, 3, rue Baudelique, 75018 Paris, France
+Populettes, 86 bis rue Riquet, 75018 Paris, France
+La Recoleta au Manoir, 229 avenue Gambetta, 75020 Paris, France
 L'Assassin, 99 rue Jean-Pierre Timbaud, 75011 Paris, France
-Le pari's café, 104 rue caulaincourt, 75018 Paris, France
+Le Pareloup, 80 Rue Saint-Charles, 75015 Paris, France
-Le Poulailler, 60 rue saint-sabin, 75011 Paris, France
+Café Zen, 46 rue Victoire, 75009 Paris, France
-Chai 33, 33 Cour Saint Emilion, 75012 Paris, France
+La Brasserie Gaité, 3 rue de la Gaité, 75014 Paris, France
+Au bon coin, 49 rue des Cloys, 75018 Paris, France
+La Brûlerie des Ternes, 111 rue mouffetard, 75005 Paris, France
+Le Chat bossu, 126, rue du Faubourg Saint Antoine, 75012 Paris, France
+Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France
+Le Couvent, 69 rue Broca, 75013 Paris, France
+Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
+La Perle, 78 rue vieille du temple, 75003 Paris, France
+Le Café frappé, 95 rue Montmartre, 75002 Paris, France
+L'Écir, 59 Boulevard Saint-Jacques, 75014 Paris, France
+Le Descartes, 1 rue Thouin, 75005 Paris, France
+Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France
+Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France
+Au panini de la place, 47 rue Belgrand, 75020 Paris, France
+Extra old café, 307 fg saint Antoine, 75011 Paris, France
+Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France
+Le Pure café, 14 rue Jean Macé, 75011 Paris, France
+Le Village, 182 rue de Courcelles, 75017 Paris, France
+Le Malar, 88 rue Saint-Dominique, 75007 Paris, France
+Pause Café, 41 rue de Charonne, 75011 Paris, France
+Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France
 Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France
+Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
+Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
+Caves populaires, 22 rue des Dames, 75017 Paris, France
+Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
+Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
+L'anjou, 1 rue de Montholon, 75009 Paris, France
+Le Brio, 216, rue Marcadet, 75018 Paris, France
+Tamm Bara, 7 rue Clisson, 75013 Paris, France
+La chaumière gourmande, Route de la Muette à Neuilly
+Club hippique du Jardin d’Acclimatation, 75016 Paris, France
 Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France
 Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France
-Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
 Le Centenaire, 104 rue amelot, 75011 Paris, France
-Café Pistache, 9 rue des petits champs, 75001 Paris, France
+Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
-La Cagnotte, 13 Rue Jean-Baptiste Dumay, 75020 Paris, France
-Le bal du pirate, 60 rue des bergers, 75015 Paris, France
-bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
-Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
-le 1 cinq, 172 rue de vaugirard, 75015 Paris, France
-Les Artisans, 106 rue Lecourbe, 75015 Paris, France
-Peperoni, 83 avenue de Wagram, 75001 Paris, France
 Ragueneau, 202 rue Saint Honoré, 75001 Paris, France
-l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
-zic zinc, 95 rue claude decaen, 75012 Paris, France
 L'Inévitable, 22 rue Linné, 75005 Paris, France
-Le Brio, 216, rue Marcadet, 75018 Paris, France
 Le Dunois, 77 rue Dunois, 75013 Paris, France
 La Montagne Sans Geneviève, 13 Rue du Pot de Fer, 75005 Paris, France
 Le Caminito, 48 rue du Dessous des Berges, 75013 Paris, France
 Le petit Bretonneau, Le petit Bretonneau - à l'intérieur de l'Hôpital, 75018 Paris, France
-La chaumière gourmande, Route de la Muette à Neuilly
+Le bal du pirate, 60 rue des bergers, 75015 Paris, France
-Club hippique du Jardin d’Acclimatation, 75016 Paris, France
-Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
-Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
-Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
-Caves populaires, 22 rue des Dames, 75017 Paris, France
-Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
-Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
 L'antre d'eux, 16 rue DE MEZIERES, 75006 Paris, France
-Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
+l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
-L'anjou, 1 rue de Montholon, 75009 Paris, France
+zic zinc, 95 rue claude decaen, 75012 Paris, France
-Tamm Bara, 7 rue Clisson, 75013 Paris, France
+Café Pistache, 9 rue des petits champs, 75001 Paris, France
-La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
+La Cagnotte, 13 Rue Jean-Baptiste Dumay, 75020 Paris, France
-Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
+bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
-Café Clochette, 16 avenue Richerand, 75010 Paris, France
+Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
-L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France
+Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
+le 1 cinq, 172 rue de vaugirard, 75015 Paris, France
+Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
+Les Artisans, 106 rue Lecourbe, 75015 Paris, France
+Peperoni, 83 avenue de Wagram, 75001 Paris, France
 Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France
 La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France
 Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France
-Les caves populaires, 22 rue des Dames, 75017 Paris, France
+La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
-Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
+Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
-Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
-Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
 La Brocante, 10 rue Rossini, 75009 Paris, France
-Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
+Café Clochette, 16 avenue Richerand, 75010 Paris, France
-Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
+L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France
-Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
-L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
-le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
-maison du vin, 52 rue des plantes, 75014 Paris, France
-Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
 NoMa, 39 rue Notre Dame de Nazareth, 75003 Paris, France
-Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
-Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
-le lutece, 380 rue de vaugirard, 75015 Paris, France
-Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
-Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
 O'Paris, 1 Rue des Envierges, 75020 Paris, France
-Rivolux, 16 rue de Rivoli, 75004 Paris, France
-Brasiloja, 16 rue Ganneron, 75018 Paris, France
 Botak cafe, 1 rue Paul albert, 75018 Paris, France
 La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France
-Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
+Les caves populaires, 22 rue des Dames, 75017 Paris, France
-Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
+Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
-Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
+Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
-Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France
+Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
-Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
+L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
+Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
+Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
+Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
+Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
+Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
+Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
+le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
+maison du vin, 52 rue des plantes, 75014 Paris, France
+Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
+Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
+le lutece, 380 rue de vaugirard, 75015 Paris, France
+Rivolux, 16 rue de Rivoli, 75004 Paris, France
+Brasiloja, 16 rue Ganneron, 75018 Paris, France
 Le café Monde et Médias, Place de la République, 75003 Paris, France
 L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France
 Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France
-Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
-L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
-Café Victor, 10 boulevard Victor, 75015 Paris, France
-Café Varenne, 36 rue de Varenne, 75007 Paris, France
-Le Brigadier, 12 rue Blanche, 75009 Paris, France
-Melting Pot, 3 rue de Lagny, 75020 Paris, France
-L'Entracte, place de l'opera, 75002 Paris, France
-le Zango, 58 rue Daguerre, 75014 Paris, France
-Panem, 18 rue de Crussol, 75011 Paris, France
-Waikiki, 10 rue d"Ulm, 75005 Paris, France
 l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France
 Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France
 Pari's Café, 174 avenue de Clichy, 75017 Paris, France
+Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
+Café Varenne, 36 rue de Varenne, 75007 Paris, France
+Melting Pot, 3 rue de Lagny, 75020 Paris, France
+le Zango, 58 rue Daguerre, 75014 Paris, France
+Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
+Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
+Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
+Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France
+Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
 Brasserie le Morvan, 61 rue du château d'eau, 75010 Paris, France
+L'Angle, 28 rue de Ponthieu, 75008 Paris, France
+Café Dupont, 198 rue de la Convention, 75015 Paris, France
+L'Entracte, place de l'opera, 75002 Paris, France
+Panem, 18 rue de Crussol, 75011 Paris, France
 Au pays de Vannes, 34 bis rue de Wattignies, 75012 Paris, France
 Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France
-L'Angle, 28 rue de Ponthieu, 75008 Paris, France
 Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France
-Café Dupont, 198 rue de la Convention, 75015 Paris, France
 L'âge d'or, 26 rue du Docteur Magnan, 75013 Paris, France
 Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France
 L'horizon, 93, rue de la Roquette, 75011 Paris, France
+L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
+Café Victor, 10 boulevard Victor, 75015 Paris, France
+Le Brigadier, 12 rue Blanche, 75009 Paris, France
+Waikiki, 10 rue d"Ulm, 75005 Paris, France

bonobo/examples/files/csv_handlers.py

@@ -2,8 +2,8 @@ import bonobo
 from bonobo.commands.run import get_default_services
 
 graph = bonobo.Graph(
-    bonobo.CsvReader('datasets/coffeeshops.txt'),
+    bonobo.CsvReader('datasets/coffeeshops.txt', headers=('item', )),
-    print,
+    bonobo.PrettyPrinter(),
 )
 
 if __name__ == '__main__':

bonobo/examples/files/json_handlers.py

@@ -1,15 +1,16 @@
 import bonobo
+from bonobo import Bag
 from bonobo.commands.run import get_default_services
 
 
-def get_fields(row):
+def get_fields(**row):
-    return row['fields']
+    return Bag(**row['fields'])
 
 
 graph = bonobo.Graph(
     bonobo.JsonReader('datasets/theaters.json'),
     get_fields,
-    bonobo.PrettyPrint(title_keys=('eq_nom_equipement', )),
+    bonobo.PrettyPrinter(),
 )
 
 if __name__ == '__main__':

bonobo/examples/files/pickle_handlers.py

@@ -0,0 +1,59 @@
+'''
+This example shows how a different file system service can be injected
+into a transformation (as compressing pickled objects often makes sense
+anyways).  The pickle itself contains a list of lists as follows:
+
+```
+[
+    ['category', 'sms'],
+    ['ham', 'Go until jurong point, crazy..'],
+    ['ham', 'Ok lar... Joking wif u oni...'],
+    ['spam', 'Free entry in 2 a wkly comp to win...'],
+    ['ham', 'U dun say so early hor... U c already then say...'],
+    ['ham', 'Nah I don't think he goes to usf, he lives around here though'],
+    ['spam', 'FreeMsg Hey there darling it's been 3 week's now...'],
+    ...
+]
+```
+
+where the first column categorizes and sms as "ham" or "spam".  The second
+column contains the sms itself.
+
+Data set taken from:
+https://www.kaggle.com/uciml/sms-spam-collection-dataset/downloads/sms-spam-collection-dataset.zip
+
+The transformation (1) reads the pickled data, (2) marks and shortens
+messages categorized as spam, and (3) prints the output.
+
+'''
+
+import bonobo
+from bonobo.commands.run import get_default_services
+from fs.tarfs import TarFS
+
+
+def cleanse_sms(**row):
+    if row['category'] == 'spam':
+        row['sms_clean'] = '**MARKED AS SPAM** ' + row['sms'][0:50] + (
+            '...' if len(row['sms']) > 50 else ''
+        )
+    else:
+        row['sms_clean'] = row['sms']
+
+    return row['sms_clean']
+
+
+graph = bonobo.Graph(
+    # spam.pkl is within the gzipped tarball
+    bonobo.PickleReader('spam.pkl'),
+    cleanse_sms,
+    bonobo.PrettyPrinter(),
+)
+
+
+def get_services():
+    return {'fs': TarFS(bonobo.get_examples_path('datasets/spam.tgz'))}
+
+
+if __name__ == '__main__':
+    bonobo.run(graph, services=get_default_services(__file__))

bonobo/examples/nodes/slow.py

@@ -0,0 +1,16 @@
+import bonobo
+import time
+
+from bonobo.constants import NOT_MODIFIED
+
+
+def pause(*args, **kwargs):
+    time.sleep(0.1)
+    return NOT_MODIFIED
+
+
+graph = bonobo.Graph(
+    lambda: tuple(range(20)),
+    pause,
+    print,
+)

bonobo/examples/tutorials/tut01e01.py

@@ -0,0 +1,23 @@
+import bonobo
+
+
+def extract():
+    yield 'foo'
+    yield 'bar'
+    yield 'baz'
+
+
+def transform(x):
+    return x.upper()
+
+
+def load(x):
+    print(x)
+
+
+graph = bonobo.Graph(extract, transform, load)
+
+graph.__doc__ = 'hello'
+
+if __name__ == '__main__':
+    bonobo.run(graph)

bonobo/examples/tutorials/tut01e02.py

@@ -0,0 +1,14 @@
+import bonobo
+
+graph = bonobo.Graph(
+    [
+        'foo',
+        'bar',
+        'baz',
+    ],
+    str.upper,
+    print,
+)
+
+if __name__ == '__main__':
+    bonobo.run(graph)

bonobo/examples/tutorials/tut02_01_read.py

@@ -1,11 +0,0 @@
-import bonobo
-
-graph = bonobo.Graph(
-    bonobo.FileReader('coffeeshops.txt'),
-    print,
-)
-
-if __name__ == '__main__':
-    bonobo.run(
-        graph, services={'fs': bonobo.open_examples_fs('datasets')}
-    )

bonobo/examples/tutorials/tut02_02_write.py

@@ -1,17 +0,0 @@
-import bonobo
-
-
-def split_one(line):
-    return line.split(', ', 1)
-
-
-graph = bonobo.Graph(
-    bonobo.FileReader('coffeeshops.txt'),
-    split_one,
-    bonobo.JsonWriter('coffeeshops.json'),
-)
-
-if __name__ == '__main__':
-    bonobo.run(
-        graph, services={'fs': bonobo.open_examples_fs('datasets')}
-    )

bonobo/examples/tutorials/tut02e01_read.py

@@ -0,0 +1,14 @@
+import bonobo
+
+graph = bonobo.Graph(
+    bonobo.FileReader('coffeeshops.txt'),
+    print,
+)
+
+
+def get_services():
+    return {'fs': bonobo.open_examples_fs('datasets')}
+
+
+if __name__ == '__main__':
+    bonobo.run(graph, services=get_services())

bonobo/examples/tutorials/tut02e02_write.py

@@ -0,0 +1,25 @@
+import bonobo
+
+
+def split_one(line):
+    return line.split(', ', 1)
+
+
+graph = bonobo.Graph(
+    bonobo.FileReader('coffeeshops.txt'),
+    split_one,
+    bonobo.JsonWriter(
+        'coffeeshops.json', fs='fs.output', ioformat='arg0'
+    ),
+)
+
+
+def get_services():
+    return {
+        'fs': bonobo.open_examples_fs('datasets'),
+        'fs.output': bonobo.open_fs(),
+    }
+
+
+if __name__ == '__main__':
+    bonobo.run(graph, services=get_services())

bonobo/examples/tutorials/tut02e03_writeasmap.py

@@ -1,4 +1,6 @@
-import bonobo, json
+import json
+
+import bonobo
 
 
 def split_one_to_map(line):
@@ -18,10 +20,16 @@ class MyJsonWriter(bonobo.JsonWriter):
 graph = bonobo.Graph(
     bonobo.FileReader('coffeeshops.txt'),
     split_one_to_map,
-    MyJsonWriter('coffeeshops.json'),
+    MyJsonWriter('coffeeshops.json', fs='fs.output', ioformat='arg0'),
 )
 
+
+def get_services():
+    return {
+        'fs': bonobo.open_examples_fs('datasets'),
+        'fs.output': bonobo.open_fs(),
+    }
+
+
 if __name__ == '__main__':
-    bonobo.run(
+    bonobo.run(graph, services=get_services())
-        graph, services={'fs': bonobo.open_examples_fs('datasets')}
-    )

bonobo/examples/types/__main__.py

@@ -0,0 +1,3 @@
+from bonobo.util.python import require
+
+graph = require('strings').graph

bonobo/execution/base.py

@@ -9,6 +9,14 @@ from bonobo.util.errors import print_error
 from bonobo.util.objects import Wrapper, get_name
 
 
+@contextmanager
+def recoverable(error_handler):
+    try:
+        yield
+    except Exception as exc:  # pylint: disable=broad-except
+        error_handler(exc, traceback.format_exc())
+
+
 @contextmanager
 def unrecoverable(error_handler):
     try:
@@ -50,14 +58,20 @@ class LoopingExecutionContext(Wrapper):
         # XXX enhancers
         self._enhancers = get_enhancers(self.wrapped)
 
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
+        self.stop()
+
     def start(self):
         if self.started:
             raise RuntimeError('Cannot start a node twice ({}).'.format(get_name(self)))
 
         self._started = True
-        self._stack = ContextCurrifier(self.wrapped, *self._get_initial_context())
 
-        with unrecoverable(self.handle_error):
+        self._stack = ContextCurrifier(self.wrapped, *self._get_initial_context())
         self._stack.setup(self)
 
         for enhancer in self._enhancers:
@@ -82,7 +96,7 @@ class LoopingExecutionContext(Wrapper):
             return
 
         try:
-            with unrecoverable(self.handle_error):
+            if self._stack:
                 self._stack.teardown()
         finally:
             self._stopped = True

bonobo/execution/plugin.py

@@ -1,6 +1,4 @@
-import traceback
+from bonobo.execution.base import LoopingExecutionContext, recoverable
-
-from bonobo.execution.base import LoopingExecutionContext
 
 
 class PluginExecutionContext(LoopingExecutionContext):
@@ -14,21 +12,14 @@ class PluginExecutionContext(LoopingExecutionContext):
     def start(self):
         super().start()
 
-        try:
+        with recoverable(self.handle_error):
             self.wrapped.initialize()
-        except Exception as exc:  # pylint: disable=broad-except
-            self.handle_error(exc, traceback.format_exc())
 
     def shutdown(self):
-        try:
+        with recoverable(self.handle_error):
             self.wrapped.finalize()
-        except Exception as exc:  # pylint: disable=broad-except
-            self.handle_error(exc, traceback.format_exc())
-        finally:
         self.alive = False
 
     def step(self):
-        try:
+        with recoverable(self.handle_error):
             self.wrapped.run()
-        except Exception as exc:  # pylint: disable=broad-except
-            self.handle_error(exc, traceback.format_exc())

bonobo/ext/console.py

@@ -1,5 +1,6 @@
-import functools
+import io
 import sys
+from contextlib import redirect_stdout
 
 from colorama import Style, Fore
 
@@ -8,6 +9,21 @@ from bonobo.plugins import Plugin
 from bonobo.util.term import CLEAR_EOL, MOVE_CURSOR_UP
 
 
+class IOBuffer():
+    def __init__(self):
+        self.current = io.StringIO()
+        self.write = self.current.write
+
+    def switch(self):
+        previous = self.current
+        self.current = io.StringIO()
+        self.write = self.current.write
+        try:
+            return previous.getvalue()
+        finally:
+            previous.close()
+
+
 class ConsoleOutputPlugin(Plugin):
     """
     Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor
@@ -21,30 +37,32 @@ class ConsoleOutputPlugin(Plugin):
 
     def initialize(self):
         self.prefix = ''
+        self.counter = 0
+        self._append_cache = ''
+        self.isatty = sys.stdout.isatty()
 
-    def _write(self, graph_context, rewind):
+        self._stdout = sys.stdout
-        if settings.PROFILE:
+        self.stdout = IOBuffer()
-            append = (
+        self.redirect_stdout = redirect_stdout(self.stdout)
-                ('Memory', '{0:.2f} Mb'.format(memory_usage())),
+        self.redirect_stdout.__enter__()
-                # ('Total time', '{0} s'.format(execution_time(harness))),
-            )
-        else:
-            append = ()
-        self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind)
 
     def run(self):
-        if sys.stdout.isatty():
+        if self.isatty:
             self._write(self.context.parent, rewind=True)
         else:
             pass  # not a tty
 
     def finalize(self):
         self._write(self.context.parent, rewind=False)
+        self.redirect_stdout.__exit__(None, None, None)
 
-    @staticmethod
+    def write(self, context, prefix='', rewind=True, append=None):
-    def write(context, prefix='', rewind=True, append=None):
         t_cnt = len(context)
 
+        buffered = self.stdout.switch()
+        for line in buffered.split('\n')[:-1]:
+            print(line + CLEAR_EOL, file=sys.stderr)
+
         for i in context.graph.topologically_sorted_indexes:
             node = context[i]
             name_suffix = '({})'.format(i) if settings.DEBUG else ''
@@ -62,7 +80,7 @@ class ConsoleOutputPlugin(Plugin):
                         Style.RESET_ALL, ' ',
                     )
                 )
-            print(prefix + _line + '\033[0K')
+            print(prefix + _line + '\033[0K', file=sys.stderr)
 
         if append:
             # todo handle multiline
@@ -72,16 +90,30 @@ class ConsoleOutputPlugin(Plugin):
                         ' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v)
                                           for k, v in append), CLEAR_EOL
                     )
-                )
+                ),
+                file=sys.stderr
             )
             t_cnt += 1
 
         if rewind:
-            print(CLEAR_EOL)
+            print(CLEAR_EOL, file=sys.stderr)
-            print(MOVE_CURSOR_UP(t_cnt + 2))
+            print(MOVE_CURSOR_UP(t_cnt + 2), file=sys.stderr)
+
+    def _write(self, graph_context, rewind):
+        if settings.PROFILE:
+            if self.counter % 10 and self._append_cache:
+                append = self._append_cache
+            else:
+                self._append_cache = append = (
+                    ('Memory', '{0:.2f} Mb'.format(memory_usage())),
+                    # ('Total time', '{0} s'.format(execution_time(harness))),
+                )
+        else:
+            append = ()
+        self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind)
+        self.counter += 1
 
 
-@functools.lru_cache(1)
 def memory_usage():
     import os, psutil
     process = psutil.Process(os.getpid())

bonobo/ext/edgy/project/feature.py

@@ -1,22 +0,0 @@
-try:
-    import edgy.project
-except ImportError as e:
-    import logging
-
-    logging.exception('You must install edgy.project to use this.')
-
-import os
-
-from edgy.project.events import subscribe
-from edgy.project.feature import Feature, SUPPORT_PRIORITY
-
-
-class BonoboFeature(Feature):
-    requires = {'python'}
-
-    @subscribe('edgy.project.on_start', priority=SUPPORT_PRIORITY)
-    def on_start(self, event):
-        package_path = event.setup['name'].replace('.', os.sep)
-
-        for file in ('example_graph'):
-            self.render_file(os.path.join(package_path, file + '.py'), os.path.join('tornado', file + '.py.j2'))

bonobo/ext/jupyter/js/dist/index.js

@@ -42,7 +42,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 /************************************************************************/
 /******/ ([
 /* 0 */
-/***/ function(module, exports, __webpack_require__) {
+/***/ (function(module, exports, __webpack_require__) {
 
 	// Entry point for the unpkg bundle containing custom model definitions.
 	//
@@ -55,14 +55,13 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	module.exports['version'] = __webpack_require__(4).version;
 
 
-/***/ },
+/***/ }),
 /* 1 */
-/***/ function(module, exports, __webpack_require__) {
+/***/ (function(module, exports, __webpack_require__) {
 
 	var widgets = __webpack_require__(2);
 	var _ = __webpack_require__(3);
 	
-	
 	// Custom Model. Custom widgets models must at least provide default values
 	// for model attributes, including `_model_name`, `_view_name`, `_model_module`
 	// and `_view_module` when different from the base class.
@@ -102,15 +101,15 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	};
 
 
-/***/ },
+/***/ }),
 /* 2 */
-/***/ function(module, exports) {
+/***/ (function(module, exports) {
 
 	module.exports = __WEBPACK_EXTERNAL_MODULE_2__;
 
-/***/ },
+/***/ }),
 /* 3 */
-/***/ function(module, exports, __webpack_require__) {
+/***/ (function(module, exports, __webpack_require__) {
 
 	var __WEBPACK_AMD_DEFINE_ARRAY__, __WEBPACK_AMD_DEFINE_RESULT__;//     Underscore.js 1.8.3
 	//     http://underscorejs.org
@@ -1662,9 +1661,9 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	}.call(this));
 
 
-/***/ },
+/***/ }),
 /* 4 */
-/***/ function(module, exports) {
+/***/ (function(module, exports) {
 
 	module.exports = {
 		"name": "bonobo-jupyter",
@@ -1696,6 +1695,6 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 		}
 	};
 
-/***/ }
+/***/ })
 /******/ ])});;
 //# sourceMappingURL=index.js.map

bonobo/ext/jupyter/js/src/bonobo.js

@@ -1,7 +1,6 @@
 var widgets = require('jupyter-js-widgets');
 var _ = require('underscore');
 
-
 // Custom Model. Custom widgets models must at least provide default values
 // for model attributes, including `_model_name`, `_view_name`, `_model_module`
 // and `_view_module` when different from the base class.

bonobo/ext/jupyter/plugin.py

@@ -19,6 +19,8 @@ class JupyterOutputPlugin(Plugin):
         IPython.core.display.display(self.widget)
 
     def run(self):
-        self.widget.value = [repr(node) for node in self.context.parent.nodes]
+        self.widget.value = [
+            str(self.context.parent[i]) for i in self.context.parent.graph.topologically_sorted_indexes
+        ]
 
     finalize = run

bonobo/ext/jupyter/static/extension.js

@@ -42,7 +42,7 @@ define(function() { return /******/ (function(modules) { // webpackBootstrap
 /************************************************************************/
 /******/ ([
 /* 0 */
-/***/ function(module, exports) {
+/***/ (function(module, exports) {
 
 	// This file contains the javascript that is run when the notebook is loaded.
 	// It contains some requirejs configuration and the `load_ipython_extension`
@@ -66,5 +66,5 @@ define(function() { return /******/ (function(modules) { // webpackBootstrap
 	};
 
 
-/***/ }
+/***/ })
 /******/ ])});;

bonobo/ext/jupyter/static/index.js

@@ -42,7 +42,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 /************************************************************************/
 /******/ ([
 /* 0 */
-/***/ function(module, exports, __webpack_require__) {
+/***/ (function(module, exports, __webpack_require__) {
 
 	// Entry point for the notebook bundle containing custom model definitions.
 	//
@@ -58,14 +58,13 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	module.exports['version'] = __webpack_require__(4).version;
 
 
-/***/ },
+/***/ }),
 /* 1 */
-/***/ function(module, exports, __webpack_require__) {
+/***/ (function(module, exports, __webpack_require__) {
 
 	var widgets = __webpack_require__(2);
 	var _ = __webpack_require__(3);
 	
-	
 	// Custom Model. Custom widgets models must at least provide default values
 	// for model attributes, including `_model_name`, `_view_name`, `_model_module`
 	// and `_view_module` when different from the base class.
@@ -105,15 +104,15 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	};
 
 
-/***/ },
+/***/ }),
 /* 2 */
-/***/ function(module, exports) {
+/***/ (function(module, exports) {
 
 	module.exports = __WEBPACK_EXTERNAL_MODULE_2__;
 
-/***/ },
+/***/ }),
 /* 3 */
-/***/ function(module, exports, __webpack_require__) {
+/***/ (function(module, exports, __webpack_require__) {
 
 	var __WEBPACK_AMD_DEFINE_ARRAY__, __WEBPACK_AMD_DEFINE_RESULT__;//     Underscore.js 1.8.3
 	//     http://underscorejs.org
@@ -1665,9 +1664,9 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 	}.call(this));
 
 
-/***/ },
+/***/ }),
 /* 4 */
-/***/ function(module, exports) {
+/***/ (function(module, exports) {
 
 	module.exports = {
 		"name": "bonobo-jupyter",
@@ -1699,6 +1698,6 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
 		}
 	};
 
-/***/ }
+/***/ })
 /******/ ])});;
 //# sourceMappingURL=index.js.map

bonobo/ext/jupyter/widget.py

@@ -2,7 +2,7 @@ import ipywidgets as widgets
 from traitlets import List, Unicode
 
 
-@widgets.register('bonobo-widget.Bonobo')
+@widgets.register('bonobo-widget.bonobo')
 class BonoboWidget(widgets.DOMWidget):
     _view_name = Unicode('BonoboView').tag(sync=True)
     _model_name = Unicode('BonoboModel').tag(sync=True)

bonobo/logging.py

@@ -0,0 +1,78 @@
+import logging
+import sys
+import textwrap
+from logging import CRITICAL, DEBUG, ERROR, INFO, WARNING
+
+from colorama import Fore, Style
+
+from bonobo import settings
+from bonobo.util.term import CLEAR_EOL
+
+
+def get_format():
+    yield '{b}[%(fg)s%(levelname)s{b}][{w}'
+    yield '{b}][{w}'.join(('%(spent)04d', '%(name)s'))
+    yield '{b}]'
+    yield ' %(fg)s%(message)s{r}'
+    yield CLEAR_EOL
+
+
+colors = {
+    'b': Fore.BLACK,
+    'w': Fore.LIGHTBLACK_EX,
+    'r': Style.RESET_ALL,
+}
+format = (''.join(get_format())).format(**colors)
+
+
+class Filter(logging.Filter):
+    def filter(self, record):
+        record.spent = record.relativeCreated // 1000
+        if record.levelname == 'DEBG':
+            record.fg = Fore.LIGHTBLACK_EX
+        elif record.levelname == 'INFO':
+            record.fg = Fore.LIGHTWHITE_EX
+        elif record.levelname == 'WARN':
+            record.fg = Fore.LIGHTYELLOW_EX
+        elif record.levelname == 'ERR ':
+            record.fg = Fore.LIGHTRED_EX
+        elif record.levelname == 'CRIT':
+            record.fg = Fore.RED
+        else:
+            record.fg = Fore.LIGHTWHITE_EX
+        return True
+
+
+class Formatter(logging.Formatter):
+    def formatException(self, ei):
+        tb = super().formatException(ei)
+        return textwrap.indent(tb, Fore.BLACK + ' | ' + Fore.WHITE)
+
+
+def setup(level):
+    logging.addLevelName(DEBUG, 'DEBG')
+    logging.addLevelName(INFO, 'INFO')
+    logging.addLevelName(WARNING, 'WARN')
+    logging.addLevelName(ERROR, 'ERR ')
+    logging.addLevelName(CRITICAL, 'CRIT')
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(Formatter(format))
+    handler.addFilter(Filter())
+    root = logging.getLogger()
+    root.addHandler(handler)
+    root.setLevel(level)
+
+
+def set_level(level):
+    logging.getLogger().setLevel(level)
+
+
+def get_logger(name='bonobo'):
+    return logging.getLogger(name)
+
+
+# Compatibility with python logging
+getLogger = get_logger
+
+# Setup formating and level.
+setup(level=settings.LOGGING_LEVEL)

bonobo/nodes/basics.py

@@ -1,9 +1,12 @@
 import functools
 from pprint import pprint as _pprint
 
+import itertools
 from colorama import Fore, Style
 
-from bonobo.config import Configurable, ContextProcessor, Option
+from bonobo import settings
+from bonobo.config import Configurable, Option
+from bonobo.config.processors import ContextProcessor
 from bonobo.structs.bags import Bag
 from bonobo.util.objects import ValueHolder
 from bonobo.util.term import CLEAR_EOL
@@ -15,7 +18,7 @@ __all__ = [
     'Tee',
     'count',
     'pprint',
-    'PrettyPrint',
+    'PrettyPrinter',
     'noop',
 ]
 
@@ -68,7 +71,24 @@ def _count_counter(self, context):
     context.send(Bag(counter._value))
 
 
-pprint = Tee(_pprint)
+class PrettyPrinter(Configurable):
+    def call(self, *args, **kwargs):
+        formater = self._format_quiet if settings.QUIET else self._format_console
+
+        for i, (item, value) in enumerate(itertools.chain(enumerate(args), kwargs.items())):
+            print(formater(i, item, value))
+
+    def _format_quiet(self, i, item, value):
+        return ' '.join(((' ' if i else '-'), str(item), ':', str(value).strip()))
+
+    def _format_console(self, i, item, value):
+        return ' '.join(
+            ((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL)
+        )
+
+
+pprint = PrettyPrinter()
+pprint.__name__ = 'pprint'
 
 
 def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):

bonobo/nodes/io/__init__.py

@@ -3,6 +3,7 @@
 from .file import FileReader, FileWriter
 from .json import JsonReader, JsonWriter
 from .csv import CsvReader, CsvWriter
+from .pickle import PickleReader, PickleWriter
 
 __all__ = [
     'CsvReader',
@@ -11,4 +12,6 @@ __all__ = [
     'FileWriter',
     'JsonReader',
     'JsonWriter',
+    'PickleReader',
+    'PickleWriter',
 ]

bonobo/nodes/io/base.py

@@ -0,0 +1,82 @@
+from bonobo import settings
+from bonobo.config import Configurable, ContextProcessor, Option, Service
+from bonobo.structs.bags import Bag
+
+
+class IOFormatEnabled(Configurable):
+    ioformat = Option(default=settings.IOFORMAT.get)
+
+    def get_input(self, *args, **kwargs):
+        if self.ioformat == settings.IOFORMAT_ARG0:
+            if len(args) != 1 or len(kwargs):
+                raise ValueError(
+                    'Wrong input formating: IOFORMAT=ARG0 implies one arg and no kwargs, got args={!r} and kwargs={!r}.'.
+                    format(args, kwargs)
+                )
+            return args[0]
+
+        if self.ioformat == settings.IOFORMAT_KWARGS:
+            if len(args) or not len(kwargs):
+                raise ValueError(
+                    'Wrong input formating: IOFORMAT=KWARGS ioformat implies no arg, got args={!r} and kwargs={!r}.'.
+                    format(args, kwargs)
+                )
+            return kwargs
+
+        raise NotImplementedError('Unsupported format.')
+
+    def get_output(self, row):
+        if self.ioformat == settings.IOFORMAT_ARG0:
+            return row
+
+        if self.ioformat == settings.IOFORMAT_KWARGS:
+            return Bag(**row)
+
+        raise NotImplementedError('Unsupported format.')
+
+
+class FileHandler(Configurable):
+    """Abstract component factory for file-related components.
+
+    Args:
+        path (str): which path to use within the provided filesystem.
+        eol (str): which character to use to separate lines.
+        mode (str): which mode to use when opening the file.
+        fs (str): service name to use for filesystem.
+    """
+
+    path = Option(str, required=True, positional=True)  # type: str
+    eol = Option(str, default='\n')  # type: str
+    mode = Option(str)  # type: str
+    encoding = Option(str, default='utf-8')  # type: str
+    fs = Service('fs')  # type: str
+
+    @ContextProcessor
+    def file(self, context, fs):
+        with self.open(fs) as file:
+            yield file
+
+    def open(self, fs):
+        return fs.open(self.path, self.mode, encoding=self.encoding)
+
+
+class Reader:
+    """Abstract component factory for readers.
+    """
+
+    def __call__(self, *args, **kwargs):
+        yield from self.read(*args, **kwargs)
+
+    def read(self, *args, **kwargs):
+        raise NotImplementedError('Abstract.')
+
+
+class Writer:
+    """Abstract component factory for writers.
+    """
+
+    def __call__(self, *args, **kwargs):
+        return self.write(*args, **kwargs)
+
+    def write(self, *args, **kwargs):
+        raise NotImplementedError('Abstract.')

bonobo/nodes/io/csv.py

@@ -3,8 +3,9 @@ import csv
 from bonobo.config import Option
 from bonobo.config.processors import ContextProcessor
 from bonobo.constants import NOT_MODIFIED
+from bonobo.nodes.io.file import FileReader, FileWriter
+from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
 from bonobo.util.objects import ValueHolder
-from .file import FileHandler, FileReader, FileWriter
 
 
 class CsvHandler(FileHandler):
@@ -28,7 +29,7 @@ class CsvHandler(FileHandler):
     headers = Option(tuple)
 
 
-class CsvReader(CsvHandler, FileReader):
+class CsvReader(IOFormatEnabled, FileReader, CsvHandler):
     """
     Reads a CSV and yield the values as dicts.
 
@@ -49,6 +50,7 @@ class CsvReader(CsvHandler, FileReader):
 
         if not headers.get():
             headers.set(next(reader))
+        _headers = headers.get()
 
         field_count = len(headers)
 
@@ -60,17 +62,18 @@ class CsvReader(CsvHandler, FileReader):
             if len(row) != field_count:
                 raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count, ))
 
-            yield dict(zip(headers.value, row))
+            yield self.get_output(dict(zip(_headers, row)))
 
 
-class CsvWriter(CsvHandler, FileWriter):
+class CsvWriter(IOFormatEnabled, FileWriter, CsvHandler):
     @ContextProcessor
     def writer(self, context, fs, file, lineno):
         writer = csv.writer(file, delimiter=self.delimiter, quotechar=self.quotechar, lineterminator=self.eol)
         headers = ValueHolder(list(self.headers) if self.headers else None)
         yield writer, headers
 
-    def write(self, fs, file, lineno, writer, headers, row):
+    def write(self, fs, file, lineno, writer, headers, *args, **kwargs):
+        row = self.get_input(*args, **kwargs)
         if not lineno:
             headers.set(headers.value or row.keys())
             writer.writerow(headers.get())

bonobo/nodes/io/file.py

@@ -1,63 +1,11 @@
-from bonobo.config import Option, Service
+from bonobo.config import Option
-from bonobo.config.configurables import Configurable
 from bonobo.config.processors import ContextProcessor
 from bonobo.constants import NOT_MODIFIED
+from bonobo.nodes.io.base import FileHandler, Reader, Writer
 from bonobo.util.objects import ValueHolder
 
-__all__ = [
-    'FileReader',
-    'FileWriter',
-]
 
-
+class FileReader(Reader, FileHandler):
-class FileHandler(Configurable):
-    """Abstract component factory for file-related components.
-    
-    Args:
-        path (str): which path to use within the provided filesystem.
-        eol (str): which character to use to separate lines.
-        mode (str): which mode to use when opening the file.
-        fs (str): service name to use for filesystem.
-    """
-
-    path = Option(str, required=True, positional=True)  # type: str
-    eol = Option(str, default='\n')  # type: str
-    mode = Option(str)  # type: str
-
-    fs = Service('fs')  # type: str
-
-    @ContextProcessor
-    def file(self, context, fs):
-        with self.open(fs) as file:
-            yield file
-
-    def open(self, fs):
-        return fs.open(self.path, self.mode)
-
-
-class Reader(FileHandler):
-    """Abstract component factory for readers.
-    """
-
-    def __call__(self, *args):
-        yield from self.read(*args)
-
-    def read(self, *args):
-        raise NotImplementedError('Abstract.')
-
-
-class Writer(FileHandler):
-    """Abstract component factory for writers.
-    """
-
-    def __call__(self, *args):
-        return self.write(*args)
-
-    def write(self, *args):
-        raise NotImplementedError('Abstract.')
-
-
-class FileReader(Reader):
     """Component factory for file-like readers.
 
     On its own, it can be used to read a file and yield one row per line, trimming the "eol" character at the end if
@@ -75,7 +23,7 @@ class FileReader(Reader):
             yield line.rstrip(self.eol)
 
 
-class FileWriter(Writer):
+class FileWriter(Writer, FileHandler):
     """Component factory for file or file-like writers.
 
     On its own, it can be used to write in a file one line per row that comes into this component. Extending it is
@@ -89,11 +37,11 @@ class FileWriter(Writer):
         lineno = ValueHolder(0)
         yield lineno
 
-    def write(self, fs, file, lineno, row):
+    def write(self, fs, file, lineno, line):
         """
         Write a row on the next line of opened file in context.
         """
-        self._write_line(file, (self.eol if lineno.value else '') + row)
+        self._write_line(file, (self.eol if lineno.value else '') + line)
         lineno += 1
         return NOT_MODIFIED
 

bonobo/nodes/io/json.py

@@ -3,45 +3,39 @@ from itertools import starmap
 
 from bonobo.structs.bags import Bag
 from bonobo.config.processors import ContextProcessor
-from .file import FileWriter, FileReader
+from bonobo.constants import NOT_MODIFIED
-
+from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
-__all__ = [
+from bonobo.nodes.io.file import FileReader, FileWriter
-    'JsonWriter',
-]
 
 
-class JsonHandler():
+class JsonHandler(FileHandler):
     eol = ',\n'
     prefix, suffix = '[', ']'
 
 
-class JsonReader(JsonHandler, FileReader):
+class JsonReader(IOFormatEnabled, FileReader, JsonHandler):
     loader = staticmethod(json.load)
 
     def read(self, fs, file):
         for line in self.loader(file):
-            yield line
+            yield self.get_output(line)
 
 
-class JsonDictReader(JsonReader):
+class JsonWriter(IOFormatEnabled, FileWriter, JsonHandler):
-    """ not api, don't use or expect breakage. """
-
-    def read(self, fs, file):
-        yield from starmap(Bag, self.loader(file).items())
-
-
-class JsonWriter(JsonHandler, FileWriter):
     @ContextProcessor
     def envelope(self, context, fs, file, lineno):
         file.write(self.prefix)
         yield
         file.write(self.suffix)
 
-    def write(self, fs, file, lineno, row):
+    def write(self, fs, file, lineno, *args, **kwargs):
         """
         Write a json row on the next line of file pointed by ctx.file.
 
         :param ctx:
         :param row:
         """
-        return super().write(fs, file, lineno, json.dumps(row))
+        row = self.get_input(*args, **kwargs)
+        self._write_line(file, (self.eol if lineno.value else '') + json.dumps(row))
+        lineno += 1
+        return NOT_MODIFIED

bonobo/nodes/io/pickle.py

@@ -0,0 +1,69 @@
+import pickle
+
+from bonobo.config import Option
+from bonobo.config.processors import ContextProcessor
+from bonobo.constants import NOT_MODIFIED
+from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
+from bonobo.nodes.io.file import FileReader, FileWriter
+from bonobo.util.objects import ValueHolder
+
+
+class PickleHandler(FileHandler):
+    """
+
+    .. attribute:: item_names
+
+        The names of the items in the pickle, if it is not defined in the first item of the pickle.
+
+    """
+
+    item_names = Option(tuple)
+
+
+class PickleReader(IOFormatEnabled, FileReader, PickleHandler):
+    """
+    Reads a Python pickle object and yields the items in dicts.
+    """
+
+    mode = Option(str, default='rb')
+
+    @ContextProcessor
+    def pickle_headers(self, context, fs, file):
+        yield ValueHolder(self.item_names)
+
+    def read(self, fs, file, pickle_headers):
+        data = pickle.load(file)
+
+        # if the data is not iterable, then wrap the object in a list so it may be iterated
+        if isinstance(data, dict):
+            is_dict = True
+            iterator = iter(data.items())
+        else:
+            is_dict = False
+            try:
+                iterator = iter(data)
+            except TypeError:
+                iterator = iter([data])
+
+        if not pickle_headers.get():
+            pickle_headers.set(next(iterator))
+
+        item_count = len(pickle_headers.value)
+
+        for i in iterator:
+            if len(i) != item_count:
+                raise ValueError('Received an object with %d items, expecting %d.' % (len(i), item_count, ))
+
+            yield self.get_output(dict(zip(i)) if is_dict else dict(zip(pickle_headers.value, i)))
+
+
+class PickleWriter(IOFormatEnabled, FileWriter, PickleHandler):
+    mode = Option(str, default='wb')
+
+    def write(self, fs, file, lineno, item):
+        """
+        Write a pickled item to the opened file.
+        """
+        file.write(pickle.dumps(item))
+        lineno += 1
+        return NOT_MODIFIED

bonobo/settings.py

@@ -1,5 +1,8 @@
+import logging
 import os
 
+from bonobo.errors import ValidationError
+
 
 def to_bool(s):
     if len(s):
@@ -9,8 +12,60 @@ def to_bool(s):
     return False
 
 
-# Debug mode.
+class Setting:
-DEBUG = to_bool(os.environ.get('BONOBO_DEBUG', 'f'))
+    def __init__(self, name, default=None, validator=None):
+        self.name = name
+
+        if default:
+            self.default = default if callable(default) else lambda: default
+        else:
+            self.default = lambda: None
+
+        if validator:
+            self.validator = validator
+        else:
+            self.validator = None
+
+    def __repr__(self):
+        return '<Setting {}={!r}>'.format(self.name, self.value)
+
+    def set(self, value):
+        if self.validator and not self.validator(value):
+            raise ValidationError('Invalid value {!r} for setting {}.'.format(value, self.name))
+        self.value = value
+
+    def get(self):
+        try:
+            return self.value
+        except AttributeError:
+            self.value = self.default()
+            return self.value
+
+
+# Debug/verbose mode.
+DEBUG = to_bool(os.environ.get('DEBUG', 'f'))
 
 # Profile mode.
-PROFILE = to_bool(os.environ.get('BONOBO_PROFILE', 'f'))
+PROFILE = to_bool(os.environ.get('PROFILE', 'f'))
+
+# Quiet mode.
+QUIET = to_bool(os.environ.get('QUIET', 'f'))
+
+# Logging level.
+LOGGING_LEVEL = logging.DEBUG if DEBUG else logging.INFO
+
+# Input/Output format for transformations
+IOFORMAT_ARG0 = 'arg0'
+IOFORMAT_KWARGS = 'kwargs'
+
+IOFORMATS = {
+    IOFORMAT_ARG0,
+    IOFORMAT_KWARGS,
+}
+
+IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__)
+
+
+def check():
+    if DEBUG and QUIET:
+        raise RuntimeError('I cannot be verbose and quiet at the same time.')

bonobo/strategies/executor.py

@@ -1,6 +1,5 @@
 import time
 import traceback
-
 from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
 
 from bonobo.constants import BEGIN, END
@@ -31,12 +30,11 @@ class ExecutorStrategy(Strategy):
         for plugin_context in context.plugins:
 
             def _runner(plugin_context=plugin_context):
+                with plugin_context:
                     try:
-                    plugin_context.start()
                         plugin_context.loop()
-                    plugin_context.stop()
                     except Exception as exc:
-                    print_error(exc, traceback.format_exc(), prefix='Error in plugin context', context=plugin_context)
+                        print_error(exc, traceback.format_exc(), context=plugin_context)
 
             futures.append(executor.submit(_runner))
 
@@ -46,9 +44,7 @@ class ExecutorStrategy(Strategy):
                 try:
                     node_context.start()
                 except Exception as exc:
-                    print_error(
+                    print_error(exc, traceback.format_exc(), context=node_context, method='start')
-                        exc, traceback.format_exc(), prefix='Could not start node context', context=node_context
-                    )
                     node_context.input.on_end()
                 else:
                     node_context.loop()
@@ -56,7 +52,7 @@ class ExecutorStrategy(Strategy):
                 try:
                     node_context.stop()
                 except Exception as exc:
-                    print_error(exc, traceback.format_exc(), prefix='Could not stop node context', context=node_context)
+                    print_error(exc, traceback.format_exc(), context=node_context, method='stop')
 
             futures.append(executor.submit(_runner))
 

bonobo/structs/bags.py

@@ -75,6 +75,14 @@ class Bag:
 
         raise TypeError('Could not apply bag to {}.'.format(func_or_iter))
 
+    def get(self):
+        """
+        Get a 2 element tuple of this bag's args and kwargs.
+
+        :return: tuple
+        """
+        return self.args, self.kwargs
+
     def extend(self, *args, **kwargs):
         return type(self)(*args, _parent=self, **kwargs)
 

bonobo/util/errors.py

@@ -1,5 +1,7 @@
 import sys
+from textwrap import indent
 
+from bonobo import settings
 from bonobo.structs.bags import ErrorBag
 
 
@@ -7,7 +9,14 @@ def is_error(bag):
     return isinstance(bag, ErrorBag)
 
 
-def print_error(exc, trace, context=None, prefix=''):
+def _get_error_message(exc):
+    if hasattr(exc, '__str__'):
+        message = str(exc)
+        return message[0].upper() + message[1:]
+    return '\n'.join(exc.args),
+
+
+def print_error(exc, trace, context=None, method=None):
     """
     Error handler. Whatever happens in a plugin or component, if it looks like an exception, taste like an exception
     or somehow make me think it is an exception, I'll handle it.
@@ -18,14 +27,20 @@ def print_error(exc, trace, context=None, prefix=''):
     """
 
     from colorama import Fore, Style
+
+    prefix = '{}{} | {}'.format(Fore.RED, Style.BRIGHT, Style.RESET_ALL)
+
     print(
         Style.BRIGHT,
         Fore.RED,
-        '\U0001F4A3 {}{}{}'.format(
+        type(exc).__name__,
-            (prefix + ': ') if prefix else '', type(exc).__name__, ' in {!r}'.format(context) if context else ''
+        ' (in {}{})'.format(type(context).__name__, '.{}()'.format(method) if method else '') if context else '',
-        ),
+        Style.RESET_ALL,
+        '\n',
+        indent(_get_error_message(exc), prefix + Style.BRIGHT),
         Style.RESET_ALL,
         sep='',
         file=sys.stderr,
     )
-    print(trace)
+    print(prefix, file=sys.stderr)
+    print(indent(trace, prefix, predicate=lambda line: True), file=sys.stderr)

bonobo/util/iterators.py

@@ -1,4 +1,5 @@
 """ Iterator utilities. """
+import functools
 
 
 def force_iterator(mixed):
@@ -23,6 +24,19 @@ def ensure_tuple(tuple_or_mixed):
     return (tuple_or_mixed, )
 
 
+def tuplize(generator):
+    """ Takes a generator and make it a tuple-returning function. As a side
+    effect, it can also decorate any iterator-returning function to force
+    return value to be a tuple.
+    """
+
+    @functools.wraps(generator)
+    def tuplized(*args, **kwargs):
+        return tuple(generator(*args, **kwargs))
+
+    return tuplized
+
+
 def iter_if_not_sequence(mixed):
     if isinstance(mixed, (dict, list, str)):
         raise TypeError(type(mixed).__name__)

bonobo/util/pkgs.py

@@ -0,0 +1,8 @@
+import pkg_resources
+from packaging.utils import canonicalize_name
+
+bonobo_packages = {}
+for p in pkg_resources.working_set:
+    name = canonicalize_name(p.project_name)
+    if name.startswith('bonobo'):
+        bonobo_packages[name] = p

bonobo/util/testing.py

@@ -1,5 +1,7 @@
+from contextlib import contextmanager
 from unittest.mock import MagicMock
 
+from bonobo import open_fs
 from bonobo.execution.node import NodeExecutionContext
 
 
@@ -7,3 +9,29 @@ class CapturingNodeExecutionContext(NodeExecutionContext):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.send = MagicMock()
+
+
+@contextmanager
+def optional_contextmanager(cm, *, ignore=False):
+    if cm is None or ignore:
+        yield
+    else:
+        with cm:
+            yield
+
+
+class FilesystemTester:
+    def __init__(self, extension='txt', mode='w'):
+        self.extension = extension
+        self.input_data = ''
+        self.mode = mode
+
+    def get_services_for_reader(self, tmpdir):
+        fs, filename = open_fs(tmpdir), 'input.' + self.extension
+        with fs.open(filename, self.mode) as fp:
+            fp.write(self.input_data)
+        return fs, filename, {'fs': fs}
+
+    def get_services_for_writer(self, tmpdir):
+        fs, filename = open_fs(tmpdir), 'output.' + self.extension
+        return fs, filename, {'fs': fs}

docs/_templates/index.html

@@ -2,46 +2,49 @@
 {% set title = _('Bonobo — Data processing for humans') %}
 {% block body %}
 
-    <div style="border: 2px solid red; font-weight: bold; margin: 1em; padding: 1em">
+<h1 style="text-align: center">
-        Bonobo is <strong>ALPHA</strong> software. Some APIs will change.
-    </div>
-
-    <h1 style="text-align: center">
     <img class="logo" src="{{ pathto('_static/bonobo.png', 1) }}" title="Bonobo" alt="Bonobo"
          style=" width: 128px; height: 128px;"/>
-    </h1>
+</h1>
 
-    <p>
+<p>
     {% trans %}
     <strong>Bonobo</strong> is a line-by-line data-processing toolkit for python 3.5+ (extract-transform-load
-            framework) emphasizing simple and atomic data transformations defined using a directed graph of plain old
+    framework, or ETL) emphasizing simple and atomic data transformations defined using a directed graph of plain old
     python objects (functions, iterables, generators, ...).
     {% endtrans %}
-    </p>
+</p>
 
-    <h2 style="margin-bottom: 0">{% trans %}Documentation{% endtrans %}</h2>
+<div style="border: 2px solid red; font-weight: bold; margin: 1em; padding: 1em">
+    Bonobo is <strong>ALPHA</strong> software. Some APIs will change.
+</div>
 
-    <table class="contentstable">
+
+<h2 style="margin-bottom: 0">{% trans %}Documentation{% endtrans %}</h2>
+
+<table class="contentstable">
     <tr>
         <td>
-                <p class="biglink"><a class="biglink" href="{{ pathto("tutorial/index") }}">{% trans %}First steps{% endtrans %}</a><br/>
+            <p class="biglink"><a class="biglink" href="{{ pathto(" tutorial/index") }}">{% trans %}First steps{%
+                endtrans %}</a><br/>
                 <span class="linkdescr">{% trans %}quick overview of basic features{% endtrans %}</span></p>
         </td>
         <td>
-                <p class="biglink"><a class="biglink" href="{{ pathto("search") }}">{% trans %}
+            <p class="biglink"><a class="biglink" href="{{ pathto(" search") }}">{% trans %}
                 Search{% endtrans %}</a><br/>
                 <span class="linkdescr">{% trans %}search the documentation{% endtrans %}</span></p>
         </td>
     </tr>
     <tr>
         <td>
-                <p class="biglink"><a class="biglink" href="{{ pathto("guide/index") }}">{% trans %}
+            <p class="biglink"><a class="biglink" href="{{ pathto(" guide/index") }}">{% trans %}
                 Guides{% endtrans %}</a><br/>
                 <span class="linkdescr">{% trans %}for a complete overview{% endtrans %}</span>
             </p>
         </td>
         <td>
-                <p class="biglink"><a class="biglink" href="{{ pathto("reference/index") }}">{% trans %}References{% endtrans %}</a>
+            <p class="biglink"><a class="biglink" href="{{ pathto(" reference/index") }}">{% trans %}References{%
+                endtrans %}</a>
                 <br/>
                 <span class="linkdescr">{% trans %}all functions, classes, terms{% endtrans %}</span>
             </p>
@@ -49,21 +52,22 @@
     </tr>
     <tr>
         <td>
-                <p class="biglink"><a class="biglink" href="{{ pathto("changes") }}">{% trans %}
+            <p class="biglink"><a class="biglink" target="_blank"
+                                  href="https://github.com/python-bonobo/bonobo/tree/master/bonobo/examples">{% trans %}
                 Cookbook{% endtrans %}</a><br/>
                 <span class="linkdescr">{% trans %}examples and recipes{% endtrans %}</span></p>
         </td>
         <td>
-                <p class="biglink"><a class="biglink" href="{{ pathto("contribute/index") }}">{% trans %}
+            <p class="biglink"><a class="biglink" href="{{ pathto(" contribute/index") }}">{% trans %}
                 Contribute{% endtrans %}</a><br/>
                 <span class="linkdescr">{% trans %}contributor guide{% endtrans %}</span></p>
         </td>
     </tr>
-    </table>
+</table>
 
-    <h2>Features</h2>
+<h2>Features</h2>
 
-    <ul>
+<ul>
     <li>
         {% trans %}
         <b>10 minutes to get started:</b> Know some python? Writing your first data processor is an affair
@@ -91,16 +95,23 @@
     </li>
     <li>
         {% trans %}
-            Bonobo is young, and the todo-list is huge. Read the <a href="https://www.bonobo-project.org/roadmap">roadmap</a>.
+        Bonobo is young, and the todo-list is huge. Read the <a
+            href="https://www.bonobo-project.org/roadmap">roadmap</a>.
         {% endtrans %}
     </li>
-    </ul>
+</ul>
 
-    <p>{% trans %}
+<p>{% trans %}
     You can also download PDF/EPUB versions of the Bonobo documentation:
     <a href="http://readthedocs.org/projects/bonobo/downloads/pdf/stable/">PDF version</a>,
     <a href="http://readthedocs.org/projects/bonobo/downloads/epub/stable/">EPUB version</a>.
     {% endtrans %}
-    </p>
+</p>
 
+<h2>Table of contents</h2>
+
+
+<div>
+    {{ toctree(maxdepth=2, collapse=False)}}
+</div>
 {% endblock %}

docs/changelog.rst

@@ -1,6 +1,186 @@
 Changelog
 =========
 
+v.0.4.2 - 18 june 2017
+::::::::::::::::::::::
+
+* [config] Implements a "requires()" service injection decorator for functions (api may change).
+* [core] Execution contexts are now context managers.
+* [fs] adds a defaut to current working directory in open_fs(...).
+* [logging] Adds logging alias for easier imports.
+* [stdlib] Fix I/O related nodes (especially json), there were bad bugs with ioformat.
+
+Dependency updates
+------------------
+
+* Update bonobo-docker from 0.2.6 to 0.2.8
+* Update dependencies.
+* Update fs from 2.0.3 to 2.0.4
+* Update requests from 2.17.3 to 2.18.1
+
+v.0.4.0 - 10 june 2017
+::::::::::::::::::::::
+
+Important highlights
+--------------------
+
+* **BC BREAK WARNING** New IOFORMAT option determines the default expected input and output format of transformations.
+  New default input/output format of transformations is now kwargs-based, instead of first-argument based. The
+  rationale behind this is that it does not make any sense to put a dict as the only argument of a transformation
+  knowing that python has a well supported syntax to do so already. Of course, it may break some of your
+  transformations but you can require the old behaviour by setting the IOFORMAT=arg0 environment variable.
+
+New features
+------------
+
+Command line interface
+......................
+
+* Allow to run directories or modules using "bonobo run".
+* Bonobo version command now shows where the package is installed, and an optional "--all/-a" flag show all
+  extensions in the same way. (#81)
+* Bonobo run flag "--install/-I" allow to pip install a requirements.txt file if run targets a directory. (#71)
+* Adds python logging facility configuration in bonobo cli commands.
+* Bonobo init now uses cookiecutter template.
+
+Configuration
+.............
+
+* `Exclusive(...)` context manager locks an object usage to one thread at a time.
+  ([docs](http://docs-dev.bonobo-project.org/en/develop/guide/services.html#solving-concurrency-problems))
+
+Standard library
+................
+
+* New PrettyPrinter and deprecate old crappy modules.
+* New pickle reader and writer (thanks @jelloslinger).
+
+Internals
+---------
+
+* ConsoleOutputPlugin now buffers stdout to avoid terminal conflicts. Side effect, output is only done every few tenth
+  of a second.
+
+Bugfixes
+--------
+
+* Fixes jupyter widget.
+
+Extensions
+----------
+
+* First release officially supporting bonobo-docker extension. See https://www.bonobo-project.org/with/docker.
+* Docker extension can be now installed using the "docker" extra on bonobo (`pip install bonobo[docker]`).
+* Jupyter widget now displays the status in topological order, like console.
+
+Miscellaneous
+-------------
+
+* Allow "main.py" as well as "__main__.py" to be the main entrypoint of an etl job.
+* Better error display (329296c).
+* Better testing.
+* Code sweeping (ecfdc81).
+* Dependencies updated.
+* Filesystem now resolve (expand) ~ in path.
+* Moving project artifact management (Projectfile) to edgy.project 0.3 format.
+* Refactoring and fixes around ioformats.
+* Some really minor changes.
+
+v.0.3.2 - 10 june 2017
+::::::::::::::::::::::
+
+Weekly maintenance release.
+
+* Updated frozen version numbers in requirements. 
+
+  * pytest==3.1.1
+  * requests==2.17.3
+  * sphinx==1.6.2
+  * stevedore==1.22.0
+
+  Note: this does not change anything when used as a dependency if you freeze your requirements, as the setup.py
+  requirement specifiers did not change.
+
+v.0.3.1 - 28 may 2017
+:::::::::::::::::::::
+
+Weekly maintenance release.
+
+* Updated project management model to edgy.project 0.3 format.
+* Updated frozen version numbers in requirements. 
+
+  * certifi==2017.4.17
+  * chardet==3.0.3
+  * coverage==4.4.1
+  * idna==2.5
+  * nbconvert==5.2.1
+  * pbr==3.0.1
+  * pytest-cov==2.5.1
+  * pytest==3.1.0
+  * requests==2.16.5
+  * sphinx==1.6.1
+  * sphinxcontrib-websupport==1.0.1
+  * testpath==0.3.1
+  * typing==3.6.1
+  * urllib3==1.21.1
+
+  Note: this does not change anything when used as a dependency if you freeze your requirements, as the setup.py
+  requirement specifiers did not change.
+
+v.0.3.0 - 22 may 2017
+:::::::::::::::::::::
+
+Features
+--------
+
+* ContextProcessors can now be implemented by getting the "yield" value (v = yield x), shortening the teardown-only
+  context processors by one line.
+* File related writers (file, csv, json ...) now returns NOT_MODIFIED, making it easier to chain something after.
+* More consistent console output, nodes are now sorted in a topological order before display.
+* Graph.add_chain(...) now takes _input and _output parameters the same way, accepting indexes, instances or names
+  (subject to change).
+* Graph.add_chain(...) now allows to "name" a chain, using _name keyword argument, to easily reference its output later
+  (subject to change).
+* New settings module (bonobo.settings) read environment for some global configuration stuff (DEBUG and PROFILE, for
+  now).
+* New Method subclass of Option allows to use Configurable objects as decorator (see bonobo.nodes.filter.Filter for a
+  simple example).
+* New Filter transformation in standard library.
+
+Internal features
+-----------------
+
+* Better ContextProcessor implementation, avoiding to use a decorator on the parent class. Now works with Configurable
+  instances like Option, Service and Method.
+* ContextCurrifier replaces the logic that was in NodeExecutionContext, that setup and teardown the context stack. Maybe
+  the name is not ideal.
+* All builtin transformations are of course updated to use the improved API, and should be 100% backward compatible.
+* The "core" package has been dismantled, and its rare remaining members are now in "structs" and "util" packages.
+* Standard transformation library has been moved under the bonobo.nodes package. It does not change anything if you used
+  bonobo.* (which you should).
+* ValueHolder is now more restrictive, not allowing to use .value anymore.
+
+Miscellaneous
+-------------
+
+* Code cleanup, dead code removal, more tests, etc.
+* More documentation.
+
+v.0.2.4 - 2 may 2017
+::::::::::::::::::::
+
+* Cosmetic release for PyPI package page formating. Same content as v.0.2.3.
+
+v.0.2.3 - 1 may 2017
+:::::::::::::::::::::
+
+* Positional options now supported, backward compatible. All FileHandler subclasses supports their path argument as
+  positional.
+* Better transformation lifecycle management (still work needed here).
+* Windows continuous integration now works.
+* Refactoring the "API" a lot to have a much cleaner first glance at it.
+* More documentation, tutorials, and tuning project artifacts.
+
 v.0.2.2 - 28 apr 2017
 :::::::::::::::::::::
 
@@ -29,10 +209,12 @@ Initial release
 
 * Migration from rdc.etl.
 * New cool name (ok, that's debatable).
-* Only supports python 3.5+, aggressively (which means, we can use async, and we remove all things from python 2/six compat)
+* Only supports python 3.5+, aggressively (which means, we can use async, and we remove all things from python 2/six
+  compat)
 * Removes all thing deprecated and/or not really convincing from rdc.etl.
 * We want transforms to be simple callables, so refactoring of the harness mess.
-* We want to use plain python data structures, so hashes are removed. If you use python 3.6, you may even get sorted dicts.
+* We want to use plain python data structures, so hashes are removed. If you use python 3.6, you may even get sorted
+  dicts.
 * Input/output MUX DEMUX removed, maybe no need for that in the real world. May come back, but not in 1.0
 * Change dependency policy. We need to include only the very basic requirements (and very required). Everything related
   to transforms that we may not use (bs, sqla, ...) should be optional dependencies.

docs/contribute/index.rst

@@ -13,6 +13,26 @@ contributions have less value, all contributions are very important.
 * You can enhance tests.
 * etc.
 
+tl;dr
+:::::
+
+1. Fork the github repository
+
+.. code-block:: shell-session
+
+    $ git clone https://github.com/python-bonobo/bonobo.git  # change this to use your fork.
+    $ cd bonobo
+    $ git remote add upstream https://github.com/python-bonobo/bonobo.git
+    $ git fetch upstream
+    $ git checkout upstream/develop -b feature/my_awesome_feature
+    $ # code, code, code, test, doc, code, test ...
+    $ git commit -m '[topic] .... blaaaah ....'
+    $ git push origin feature/my_awesome_feature
+
+2. Open pull request
+3. Rince, repeat
+
+
 Code-related contributions (including tests and examples)
 :::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 
@@ -58,7 +78,7 @@ Guidelines
 License
 :::::::
 
-`Bonobo is released under the apache license <https://github.com/python-bonobo/bonobo/blob/0.3/LICENSE>`_.
+`Bonobo is released under the apache license <https://www.bonobo-project.org/license>`_.
 
 License for non lawyers
 :::::::::::::::::::::::
@@ -67,6 +87,6 @@ Use it, change it, hack it, brew it, eat it.
 
 For pleasure, non-profit, profit or basically anything else, except stealing credit.
 
-Provided without warranty.
+Provided without any warranty.
 
 

docs/faq.rst

@@ -3,6 +3,7 @@ F.A.Q.
 
 List of questions that went up about the project, in no particuliar order.
 
+
 Too long; didn't read.
 ----------------------
 
@@ -19,8 +20,22 @@ It's lean manufacturing for data.
 
 .. note::
 
-    This is NOT a «big data» tool. We process around 5 millions database lines in around 1 hour with rdc.etl, bonobo
+    This is NOT a «big data» tool. Neither a «data analysis» tool. We process around 5 millions database lines in around
-    ancestor (algorithms are the same, we still need to run a bit of benchmarks).
+    1 hour with rdc.etl, bonobo ancestor (algorithms are the same, we still need to run a bit of benchmarks).
+
+
+What versions of python does bonobo support? Why not more?
+----------------------------------------------------------
+
+Bonobo is battle-tested against the latest python 3.5 and python 3.6. It may work well using other patch releases of those
+versions, but we cannot guarantee it.
+
+The main reasons about why 3.5+:
+
+* Creating a tool that works well under both python 2 and 3 is a lot more work.
+* Python 3 is nearly 10 years old. Consider moving on.
+* Python 3.5 contains syntaxic sugar that makes working with data a lot more convenient.
+
 
 Can a graph contain another graph?
 ----------------------------------
@@ -30,8 +45,14 @@ No, not for now. There are no tools today in bonobo to insert a graph as a subgr
 It would be great to allow it, but there is a few design questions behind this, like what node you use as input and
 output of the subgraph, etc.
 
+On another hand, if you don't consider a graph as the container but by the nodes and edges it contains, its pretty
+easy to add a set of nodes and edge to a subgraph, and thus simulate it. But there will be more threads, more copies
+of the same nodes, so it's not really an acceptable answer for big graphs. If it was possible to use a Graph as a
+node, then the problem would be correctly solved.
+
 It is something to be seriously considered post 1.0 (probably way post 1.0).
 
+
 How would one access contextual data from a transformation? Are there parameter injections like pytest's fixtures?
 ------------------------------------------------------------------------------------------------------------------
 
@@ -41,7 +62,8 @@ context.
 The API may evolve a bit though, because I feel it's a bit hackish, as it is. The concept will stay the same, but we need
 to find a better way to apply it.
 
-To understand how it works today, look at https://github.com/python-bonobo/bonobo/blob/0.3/bonobo/io/csv.py#L63 and class hierarchy.
+To understand how it works today, look at https://github.com/python-bonobo/bonobo/blob/master/bonobo/nodes/io/csv.py#L31 and class hierarchy.
+
 
 What is a plugin? Do I need to write one?
 -----------------------------------------
@@ -49,14 +71,19 @@ What is a plugin? Do I need to write one?
 Plugins are special classes added to an execution context, used to enhance or change the actual behavior of an execution
 in a generic way. You don't need to write plugins to code transformation graphs.
 
+
 Is there a difference between a transformation node and a regular python function or generator?
 -----------------------------------------------------------------------------------------------
 
-No.
+Short answer: no.
 
 Transformation callables are just regular callables, and there is nothing that differentiate it from regular python callables.
 You can even use some callables both in an imperative programming context and in a transformation graph, no problem.
 
+Longer answer: yes, sometimes, but you should not care. The function-based transformations are plain old python callable. The
+class-based transformations can be plain-old-python-objects, but can also subclass Configurable which brings a lot of
+fancy features, like options, service injections, class factories as decorators...
+
 
 Why did you include the word «marketing» in a commit message? Why is there a marketing-automation tag on the project? Isn't marketing evil?
 -------------------------------------------------------------------------------------------------------------------------------------------
@@ -83,6 +110,7 @@ See https://github.com/python-bonobo/bonobo/issues/1
 Bonobo is not a replacement for pandas, nor dask, nor luigi, nor airflow... It may be a replacement for Pentaho, Talend
 or other data integration suites but targets people more comfortable with code as an interface.
 
+
 All those references to monkeys hurt my head. Bonobos are not monkeys.
 ----------------------------------------------------------------------
 
@@ -96,6 +124,7 @@ known primate typing feature.»
 
 See https://github.com/python-bonobo/bonobo/issues/24
 
+
 Who is behind this?
 -------------------
 
@@ -104,6 +133,7 @@ Me (as an individual), and a few great people that helped me along the way. Not
 The code, documentation, and surrounding material is created using spare time and may lack a bit velocity. Feel free
 to jump in so we can go faster!
 
+
 Documentation seriously lacks X, there is a problem in Y...
 -----------------------------------------------------------
 

docs/guide/ext/docker.rst

@@ -4,11 +4,5 @@ Bonobo with Docker
 .. todo:: The `bonobo-docker` package is at a very alpha stage, and things will change. This section is here to give a
     brief overview but is neither complete nor definitive.
 
-Installation
+Read the introduction: https://www.bonobo-project.org/with/docker
-::::::::::::
 
-Overview
-::::::::
-
-Details
-:::::::

docs/guide/ext/jupyter.rst

@@ -15,20 +15,23 @@ Install `bonobo` with the **jupyter** extra::
 
 Install the jupyter extension::
 
+    jupyter nbextension enable --py --sys-prefix widgetsnbextension
     jupyter nbextension enable --py --sys-prefix bonobo.ext.jupyter
 
 Development
 :::::::::::
 
+You should favor yarn over npm to install node packages. If you prefer to use npm, it's up to you to adapt the code.
+
 To install the widget for development, make sure you're using an editable install of bonobo (see install document)::
 
     jupyter nbextension install --py --symlink --sys-prefix bonobo.ext.jupyter
     jupyter nbextension enable --py --sys-prefix bonobo.ext.jupyter
 
-If you wanna change the javascript, you should run webpack in watch mode in some terminal::
+If you want to change the javascript, you should run webpack in watch mode in some terminal::
 
     cd bonobo/ext/jupyter/js
-    npm install
+    yarn install
     ./node_modules/.bin/webpack --watch
 
 To compile the widget into a distributable version (which gets packaged on PyPI when a release is made), just run

docs/guide/ext/sqlalchemy.rst

@@ -4,6 +4,7 @@ Bonobo with SQLAlchemy
 .. todo:: The `bonobo-sqlalchemy` package is at a very alpha stage, and things will change. This section is here to
           give a brief overview but is neither complete nor definitive.
 
+Read the introduction: https://www.bonobo-project.org/with/sqlalchemy
 
 Installation
 ::::::::::::

docs/guide/services.rst

@@ -81,6 +81,26 @@ A dictionary, or dictionary-like, "services" named argument can be passed to the
 provided is pretty basic, and feature-less. But you can use much more evolved libraries instead of the provided
 stub, and as long as it works the same (a.k.a implements a dictionary-like interface), the system will use it.
 
+Solving concurrency problems
+----------------------------
+
+If a service cannot be used by more than one thread at a time, either because it's just not threadsafe, or because
+it requires to carefully order the calls made (apis that includes nonces, or work on results returned by previous
+calls are usually good candidates), you can use the :class:`bonobo.config.Exclusive` context processor to lock the
+use of a dependency for a time period.
+
+.. code-block:: python
+
+    from bonobo.config import Exclusive
+
+    def t1(api):
+        with Exclusive(api):
+            api.first_call()
+            api.second_call()
+            # ... etc
+            api.last_call()
+
+
 Service configuration (to be decided and implemented)
 :::::::::::::::::::::::::::::::::::::::::::::::::::::
 

docs/guide/transformations.rst

@@ -81,9 +81,13 @@ Services
 
 .. autoclass:: bonobo.config.Service
 
-Method
+Methods
-------
+-------
 
 .. autoclass:: bonobo.config.Method
 
+ContextProcessors
+-----------------
+
+.. autoclass:: bonobo.config.ContextProcessor
 

docs/index.rst

@@ -8,8 +8,8 @@ Bonobo
    tutorial/index
    guide/index
    reference/index
-   contribute/index
    faq
+   contribute/index
    genindex
    modindex
 

docs/install.rst

@@ -1,34 +1,54 @@
 Installation
 ============
 
-Bonobo is `available on PyPI <https://pypi.python.org/pypi/bonobo>`_, and it's the easiest solution to get started.
+Create an ETL project
+:::::::::::::::::::::
+
+Creating a project and starting to write code should take less than a minute:
+
+.. code-block:: shell-session
+
+    $ pip install --upgrade bonobo cookiecutter
+    $ bonobo init my-etl-project
+    $ bonobo run my-etl-project
+
+Once you bootstrapped a project, you can start editing the default example transformation by editing
+`my-etl-project/main.py`. Now, you can head to :doc:`tutorial/index`.
+
+Other installation options
+::::::::::::::::::::::::::
+
+Install from PyPI
+-----------------
+
+You can install it directly from the `Python Package Index <https://pypi.python.org/pypi/bonobo>`_ (like we did above).
 
 .. code-block:: shell-session
 
     $ pip install bonobo
 
 Install from source
-:::::::::::::::::::
+-------------------
 
 If you want to install an unreleased version, you can use git urls with pip. This is useful when using bonobo as a
-dependency of your code and you want to try a forked version of bonobo with your software. You can use the git+http
+dependency of your code and you want to try a forked version of bonobo with your software. You can use a `git+http`
-string in your `requirements.txt` file. However, the best option for development on bonobo directly is not this one,
+string in your `requirements.txt` file. However, the best option for development on bonobo is an editable install (see
-but editable installs (see below).
+below).
 
 .. code-block:: shell-session
 
-    $ pip install git+https://github.com/python-bonobo/bonobo.git@0.3#egg=bonobo
+    $ pip install git+https://github.com/python-bonobo/bonobo.git@develop#egg=bonobo
 
 Editable install
-::::::::::::::::
+----------------
 
-If you plan on making patches to Bonobo, you should install it as an "editable" package, which is a really great pip feature.
+If you plan on making patches to Bonobo, you should install it as an "editable" package, which is a really great pip
-Pip will clone your repository in a source directory and create a symlink for it in the site-package directory of your
+feature. Pip will clone your repository in a source directory and create a symlink for it in the site-package directory
-python interpreter.
+of your python interpreter.
 
 .. code-block:: shell-session
 
-    $ pip install --editable git+https://github.com/python-bonobo/bonobo.git@0.3#egg=bonobo
+    $ pip install --editable git+https://github.com/python-bonobo/bonobo.git@master#egg=bonobo
 
 .. note:: You can also use the `-e` flag instead of the long version.
 
@@ -54,20 +74,17 @@ I usually name the git remote for the main bonobo repository "upstream", and my
     
     $ git remote rename origin upstream
     $ git remote add origin git@github.com:hartym/bonobo.git
+    $ git fetch --all
 
 Of course, replace my github username by the one you used to fork bonobo. You should be good to go!
 
 Windows support
 :::::::::::::::
 
-There are problems on the windows platform, mostly due to the fact bonobo was not developed by experienced windows users.
+There are minor issues on the windows platform, mostly due to the fact bonobo was not developed by experienced windows
+users.
 
 We're trying to look into that but energy available to provide serious support on windows is very limited.
+
 If you have experience in this domain and you're willing to help, you're more than welcome!
 
-
-
-.. todo::
-
-    Better install docs, especially on how to use different forks or branches, etc.
-

docs/reference/examples.rst

@@ -1,10 +1,16 @@
 Examples
 ========
 
-There are a few examples bundled with **bonobo**. You'll find them under the :mod:`bonobo.examples` package, and
+There are a few examples bundled with **bonobo**.
-you can try them in a clone of bonobo by typing::
 
-    $ bonobo run bonobo/examples/.../file.py
+You'll find them under the :mod:`bonobo.examples` package, and you can run them directly as modules:
+
+    $ bonobo run -m bonobo.examples...module
+
+.. toctree::
+    :maxdepth: 4
+
+    examples/tutorials
 
 
 Datasets

docs/reference/examples/tutorials.rst

@@ -0,0 +1,50 @@
+Examples from the tutorial
+==========================
+
+Examples from :doc:`/tutorial/tut01`
+::::::::::::::::::::::::::::::::::::
+
+Example 1
+---------
+
+.. automodule:: bonobo.examples.tutorials.tut01e01
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Example 2
+---------
+
+.. automodule:: bonobo.examples.tutorials.tut01e02
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Examples from :doc:`/tutorial/tut02`
+::::::::::::::::::::::::::::::::::::
+
+Example 1: Read
+---------------
+
+.. automodule:: bonobo.examples.tutorials.tut02e01_read
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Example 2: Write
+----------------
+
+.. automodule:: bonobo.examples.tutorials.tut02e02_write
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Example 3: Write as map
+-----------------------
+
+.. automodule:: bonobo.examples.tutorials.tut02e02_writeasmap
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+

docs/roadmap.rst

@@ -1,12 +1,12 @@
-Detailed roadmap
+Internal roadmap notes
-================
+======================
 
-initialize / finalize better than start / stop ?
+Things that should be thought about and/or implemented, but that I don't know where to store.
 
 Graph and node level plugins
 ::::::::::::::::::::::::::::
 
- * Enhancers or nide-level plugins
+ * Enhancers or node-level plugins
  * Graph level plugins
  * Documentation
 
@@ -15,21 +15,19 @@ Command line interface and environment
 
 * How do we manage environment ? .env ?
 * How do we configure plugins ?
-* Console run should allow console plugin as a command line argument (or silence it).
 
 Services and Processors
 :::::::::::::::::::::::
 
-* ContextProcessors not clean
+* ContextProcessors not clean (a bit better, but still not in love with the api)
 
 Next...
 :::::::
 
 * Release process specialised for bonobo. With changelog production, etc.
 * Document how to upgrade version, like, minor need change badges, etc.
-* PyPI page looks like crap: https://pypi.python.org/pypi/bonobo/0.2.1
+* Windows console looks crappy.
-* Windows break because of readme encoding. Fix in edgy.
+* bonobo init --with sqlalchemy,docker; cookiecutter?
-* bonobo init --with sqlalchemy,docker
 * logger, vebosity level
 
 
@@ -39,22 +37,15 @@ External libs that looks good
 * dask.distributed
 * mediator (event dispatcher)
 
-Version 0.3
+Version 0.4
 :::::::::::
 
-* Services !
 * SQLAlchemy 101
 
-Version 0.2
+Design decisions
-:::::::::::
+::::::::::::::::
 
-* Autodetect if within jupyter notebook context, and apply plugin if it's the case.
+* initialize / finalize better than start / stop ?
-* New bonobo.structs package with simple data structures (bags, graphs, tokens).
-
-Plugins API
-:::::::::::
-
-* Stabilize, find other things to do.
 
 Minor stuff
 :::::::::::

docs/tutorial/index.rst

@@ -1,8 +1,6 @@
 First steps
 ===========
 
-Bonobo uses simple python and should be quick and easy to learn.
-
 What is Bonobo?
 :::::::::::::::
 
@@ -11,17 +9,34 @@ python code in charge of handling similar shaped independant lines of data.
 
 Bonobo *is not* a statistical or data-science tool. If you're looking for a data-analysis tool in python, use Pandas.
 
-Bonobo is a lean manufacturing assembly line for data that let you focus on the actual work instead of the plumbery.
+Bonobo is a lean manufacturing assembly line for data that let you focus on the actual work instead of the plumbery
+(execution contexts, parallelism, error handling, console output, logging, ...).
 
+Bonobo uses simple python and should be quick and easy to learn.
 
 Tutorial
 ::::::::
 
+.. note::
+
+    Good documentation is not easy to write. We do our best to make it better and better.
+
+    Although all content here should be accurate, you may feel a lack of completeness, for which we plaid guilty and
+    apologize.
+
+    If you're stuck, please come and ask on our `slack channel <https://bonobo-slack.herokuapp.com/>`_, we'll figure
+    something out.
+
+    If you're not stuck but had trouble understanding something, please consider contributing to the docs (via github
+    pull requests).
+
 .. toctree::
     :maxdepth: 2
 
     tut01
     tut02
+    tut03
+    tut04
 
 
 What's next?
@@ -43,6 +58,6 @@ Read about integrating external tools with bonobo
 
 * :doc:`../guide/ext/docker`: run transformation graphs in isolated containers.
 * :doc:`../guide/ext/jupyter`: run transformations within jupyter notebooks.
-* :doc:`../guide/ext/selenium`: run
+* :doc:`../guide/ext/selenium`: crawl the web using a real browser and work with the gathered data.
 * :doc:`../guide/ext/sqlalchemy`: everything you need to interract with SQL databases.
 

docs/tutorial/tut01.rst

@@ -1,58 +1,91 @@
-Basic concepts
+Let's get started!
-==============
+==================
 
-To begin with Bonobo, you need to install it in a working python 3.5+ environment:
+To begin with Bonobo, you need to install it in a working python 3.5+ environment, and you'll also need cookiecutter
+to bootstrap your project.
 
 .. code-block:: shell-session
 
-    $ pip install bonobo
+    $ pip install bonobo cookiecutter
 
 See :doc:`/install` for more options.
 
-Let's write a first data transformation
-:::::::::::::::::::::::::::::::::::::::
 
-We'll start with the simplest transformation possible.
+Create an empty project
+:::::::::::::::::::::::
 
-In **Bonobo**, a transformation is a plain old python callable, not more, not less. Let's write one that takes a string
+Your ETL code will live in ETL projects, which are basically a bunch of files, including python code, that bonobo
-and uppercases it.
+can run.
+
+.. code-block:: shell-session
+
+    $ bonobo init tutorial
+
+This will create a `tutorial` directory (`content description here <https://www.bonobo-project.org/with/cookiecutter>`_).
+
+To run this project, use:
+
+.. code-block:: shell-session
+
+    $ bonobo run tutorial
+
+
+Write a first transformation
+::::::::::::::::::::::::::::
+
+Open `tutorial/main.py`, and delete all the code here.
+
+A transformation can be whatever python can call. Simplest transformations are functions and generators.
+
+Let's write one:
 
 .. code-block:: python
 
-    def uppercase(x: str):
+    def transform(x):
         return x.upper()
 
-Pretty straightforward.
+Easy.
 
-You could even use :func:`str.upper` directly instead of writing a wrapper, as a type's method (unbound) will take an
+.. note::
-instance of this type as its first parameter (what you'd call `self` in your method).
 
-The type annotations written here are not used, but can make your code much more readable, and may very well be used as
+    This function is very similar to :func:`str.upper`, which you can use directly.
-validators in the future.
 
-Let's write two more transformations: a generator to produce the data to be transformed, and something that outputs it,
+Let's write two more transformations for the "extract" and "load" steps. In this example, we'll generate the data from
-because, yeah, feedback is cool.
+scratch, and we'll use stdout to "simulate" data-persistence.
 
 .. code-block:: python
 
-    def generate_data():
+    def extract():
         yield 'foo'
         yield 'bar'
         yield 'baz'
 
-    def output(x: str):
+    def load(x):
         print(x)
 
-Once again, you could have skipped the pain of writing this and simply use an iterable to generate the data and the
+Bonobo makes no difference between generators (yielding functions) and regular functions. It will, in all cases, iterate
-builtin :func:`print` for the output, but we'll stick to writing our own transformations for now.
+on things returned, and a normal function will just be seen as a generator that yields only once.
 
-Let's chain the three transformations together and run the transformation graph:
+.. note::
+
+    Once again, you should use the builtin :func:`print` directly instead of this `load()` function.
+
+
+Create a transformation graph
+:::::::::::::::::::::::::::::
+
+Amongst other features, Bonobo will mostly help you there with the following:
+
+* Execute the transformations in independant threads
+* Pass the outputs of one thread to other(s) thread(s) inputs.
+
+To do this, it needs to know what data-flow you want to achieve, and you'll use a :class:`bonobo.Graph` to describe it.
 
 .. code-block:: python
 
     import bonobo
 
-    graph = bonobo.Graph(generate_data, uppercase, output)
+    graph = bonobo.Graph(extract, transform, load)
 
     if __name__ == '__main__':
         bonobo.run(graph)
@@ -64,14 +97,50 @@ Let's chain the three transformations together and run the transformation graph:
         stylesheet = "../_static/graphs.css";
 
         BEGIN [shape="point"];
-        BEGIN -> "generate_data" -> "uppercase" -> "output";
+        BEGIN -> "extract" -> "transform" -> "load";
     }
 
-We use the :func:`bonobo.run` helper that hides the underlying object composition necessary to actually run the
+.. note::
-transformations in parallel, because it's simpler.
 
-Depending on what you're doing, you may use the shorthand helper method, or the verbose one. Always favor the shorter,
+    The `if __name__ == '__main__':` section is not required, unless you want to run it directly using the python
-if you don't need to tune the graph or the execution strategy (see below).
+    interpreter.
+
+
+Execute the job
+:::::::::::::::
+
+Save `tutorial/main.py` and execute your transformation again:
+
+.. code-block:: shell-session
+
+    $ bonobo run tutorial
+
+This example is available in :mod:`bonobo.examples.tutorials.tut01e01`, and you can also run it as a module:
+
+.. code-block:: shell-session
+
+    $ bonobo run -m bonobo.examples.tutorials.tut01e01
+
+
+Rewrite it using builtins
+:::::::::::::::::::::::::
+
+There is a much simpler way to describe an equivalent graph:
+
+.. literalinclude:: ../../bonobo/examples/tutorials/tut01e02.py
+    :language: python
+
+The `extract()` generator has been replaced by a list, as Bonobo will interpret non-callable iterables as a no-input
+generator.
+
+This example is also available in :mod:`bonobo.examples.tutorials.tut01e02`, and you can also run it as a module:
+
+.. code-block:: shell-session
+
+    $ bonobo run -m bonobo.examples.tutorials.tut01e02
+
+You can now jump to the next part (:doc:`tut02`), or read a small summary of concepts and definitions introduced here
+below.
 
 Takeaways
 :::::::::
@@ -79,7 +148,7 @@ Takeaways
 ① The :class:`bonobo.Graph` class is used to represent a data-processing pipeline.
 
 It can represent simple list-like linear graphs, like here, but it can also represent much more complex graphs, with
-branches and cycles.
+forks and joins.
 
 This is what the graph we defined looks like:
 
@@ -97,10 +166,10 @@ either `return` or `yield` data to send it to the next step. Regular functions (
 each call is guaranteed to return exactly one result, while generators (using `yield`) should be prefered if the
 number of output lines for a given input varies.
 
-③ The `Graph` instance, or `transformation graph` is then executed using an `ExecutionStrategy`. You did not use it
+③ The `Graph` instance, or `transformation graph` is executed using an `ExecutionStrategy`. You won't use it directly,
-directly in this tutorial, but :func:`bonobo.run` created an instance of :class:`bonobo.ThreadPoolExecutorStrategy`
+but :func:`bonobo.run` created an instance of :class:`bonobo.ThreadPoolExecutorStrategy` under the hood (the default
-under the hood (which is the default strategy). Actual behavior of an execution will depend on the strategy chosen, but
+strategy). Actual behavior of an execution will depend on the strategy chosen, but the default should be fine for most
-the default should be fine in most of the basic cases.
+cases.
 
 ④ Before actually executing the `transformations`, the `ExecutorStrategy` instance will wrap each component in an
 `execution context`, whose responsibility is to hold the state of the transformation. It enables to keep the
@@ -109,23 +178,24 @@ the default should be fine in most of the basic cases.
 Concepts and definitions
 ::::::::::::::::::::::::
 
-* Transformation: a callable that takes input (as call parameters) and returns output(s), either as its return value or
+* **Transformation**: a callable that takes input (as call parameters) and returns output(s), either as its return value or
   by yielding values (a.k.a returning a generator).
-* Transformation graph (or Graph): a set of transformations tied together in a :class:`bonobo.Graph` instance, which is a simple
+
-  directed acyclic graph (also refered as a DAG, sometimes).
+* **Transformation graph (or Graph)**: a set of transformations tied together in a :class:`bonobo.Graph` instance, which is
-* Node: a transformation within the context of a transformation graph. The node defines what to do with a
+  a directed acyclic graph (or DAG).
-  transformation's output, and especially what other nodes to feed with the output.
+
-* Execution strategy (or strategy): a way to run a transformation graph. It's responsibility is mainly to parallelize
+* **Node**: a graph element, most probably a transformation in a graph.
+
+* **Execution strategy (or strategy)**: a way to run a transformation graph. It's responsibility is mainly to parallelize
   (or not) the transformations, on one or more process and/or computer, and to setup the right queuing mechanism for
   transformations' inputs and outputs.
-* Execution context (or context): a wrapper around a node that holds the state for it. If the node needs state, there
+
-  are tools available in bonobo to feed it to the transformation using additional call parameters, and so every
+* **Execution context (or context)**: a wrapper around a node that holds the state for it. If the node needs state, there
-  transformation will be atomic.
+  are tools available in bonobo to feed it to the transformation using additional call parameters, keeping
+  transformations stateless.
 
 Next
 ::::
 
-You now know all the basic concepts necessary to build (batch-like) data processors.
+Time to jump to the second part: :doc:`tut02`.
-
-Time to jump to the second part: :doc:`tut02`
 

docs/tutorial/tut02.rst

@@ -1,11 +1,14 @@
 Working with files
 ==================
 
-Bonobo would be a bit useless if the aim was just to uppercase small lists of strings.
+Bonobo would be pointless if the aim was just to uppercase small lists of strings.
 
 In fact, Bonobo should not be used if you don't expect any gain from parallelization/distribution of tasks.
 
-Let's take the following graph as an example:
+Some background...
+::::::::::::::::::
+
+Let's take the following graph:
 
 .. graphviz::
 
@@ -16,18 +19,22 @@ Let's take the following graph as an example:
         "B" -> "D";
     }
 
-The execution strategy does a bit of under the scene work, wrapping every component in a thread (assuming you're using
+When run, the execution strategy wraps every component in a thread (assuming you're using the default
-the :class:`bonobo.strategies.ThreadPoolExecutorStrategy`).
+:class:`bonobo.strategies.ThreadPoolExecutorStrategy`).
 
 Bonobo will send each line of data in the input node's thread (here, `A`). Now, each time `A` *yields* or *returns*
-something, it will be pushed on `B` input :class:`queue.Queue`, and will be consumed by `B`'s thread.
+something, it will be pushed on `B` input :class:`queue.Queue`, and will be consumed by `B`'s thread. Meanwhile, `A`
+will continue to run, if it's not done.
 
-When there is more than one node linked as the output of a node (for example, with `B`, `C`, and `D`) , the same thing
+When there is more than one node linked as the output of a node (for example, with `B`, `C`, and `D`), the same thing
 happens except that each result coming out of `B` will be sent to both on `C` and `D` input :class:`queue.Queue`.
 
-The great thing is that you generally don't have to think about it. Just be aware that your components will be run in
+One thing to keep in mind here is that as the objects are passed from thread to thread, you need to write "pure"
-parallel (with the default strategy), and don't worry too much about blocking components, as they won't block their
+transformations (see :doc:`/guide/purity`).
-siblings when run in bonobo.
+
+You generally don't have to think about it. Just be aware that your nodes will run in parallel, and don't worry
+too much about nodes running blocking operations, as they will run in parallel. As soon as a line of output is ready,
+the next nodes will start consuming it.
 
 That being said, let's manipulate some files.
 
@@ -38,46 +45,41 @@ There are a few component builders available in **Bonobo** that let you read fro
 
 All readers work the same way. They need a filesystem to work with, and open a "path" they will read from.
 
-* :class:`bonobo.io.FileReader`
+* :class:`bonobo.CsvReader`
-* :class:`bonobo.io.JsonReader`
+* :class:`bonobo.FileReader`
-* :class:`bonobo.io.CsvReader`
+* :class:`bonobo.JsonReader`
+* :class:`bonobo.PickleReader`
 
 We'll use a text file that was generated using Bonobo from the "liste-des-cafes-a-un-euro" dataset made available by
 Mairie de Paris under the Open Database License (ODbL). You can `explore the original dataset
 <https://opendata.paris.fr/explore/dataset/liste-des-cafes-a-un-euro/information/>`_.
 
-You'll need the `example dataset <https://github.com/python-bonobo/bonobo/blob/0.3/bonobo/examples/datasets/coffeeshops.txt>`_,
+You'll need the `"coffeeshops.txt" example dataset <https://github.com/python-bonobo/bonobo/blob/master/bonobo/examples/datasets/coffeeshops.txt>`_,
-available in **Bonobo**'s repository.
+available in **Bonobo**'s repository:
 
-.. literalinclude:: ../../bonobo/examples/tutorials/tut02_01_read.py
+.. code-block:: shell-session
+
+    $ curl https://raw.githubusercontent.com/python-bonobo/bonobo/master/bonobo/examples/datasets/coffeeshops.txt > `python -c 'import bonobo; print(bonobo.get_examples_path("datasets/coffeeshops.txt"))'`
+
+.. note::
+
+    The "example dataset download" step will be easier in the future.
+
+    https://github.com/python-bonobo/bonobo/issues/134
+
+.. literalinclude:: ../../bonobo/examples/tutorials/tut02e01_read.py
     :language: python
 
-You can run this script directly using the python interpreter:
+You can also run this example as a module (but you'll still need the dataset...):
 
 .. code-block:: shell-session
 
-    $ python bonobo/examples/tutorials/tut02_01_read.py
+    $ bonobo run -m bonobo.examples.tutorials.tut02e01_read
 
-Another option is to use the bonobo cli, which allows more flexibility:
+.. note::
 
-.. code-block:: shell-session
+    Don't focus too much on the `get_services()` function for now. It is required, with this exact name, but we'll get
-
+    into that in a few minutes.
-    $ bonobo run bonobo/examples/tutorials/tut02_01_read.py
-
-Using bonobo command line has a few advantages.
-
-It will look for one and only one :class:`bonobo.Graph` instance in the file given as argument, configure an execution
-strategy, eventually plugins, and execute it. It has the benefit of allowing to tune the "artifacts" surrounding the
-transformation graph on command line (verbosity, plugins ...), and it will also ease the transition to run
-transformation graphs in containers, as the syntax will be the same. Of course, it is not required, and the
-containerization capabilities are provided by an optional and separate python package.
-
-It also change a bit the way you can configure service dependencies. The CLI won't run the `if __name__ == '__main__'`
-block,  and thus it won't get the configured services passed to :func:`bonobo.run`. Instead, one option to configure
-services is to define a `get_services()` function in a
-`_services.py <https://github.com/python-bonobo/bonobo/blob/0.3/bonobo/examples/tutorials/_services.py>`_ file.
-
-There will be more options using the CLI or environment to override things soon.
 
 Writing to files
 ::::::::::::::::
@@ -86,22 +88,34 @@ Let's split this file's each lines on the first comma and store a json file mapp
 
 Here are, like the readers, the classes available to write files
 
-* :class:`bonobo.io.FileWriter`
+* :class:`bonobo.CsvWriter`
-* :class:`bonobo.io.JsonWriter`
+* :class:`bonobo.FileWriter`
-* :class:`bonobo.io.CsvWriter`
+* :class:`bonobo.JsonWriter`
+* :class:`bonobo.PickleWriter`
 
 Let's write a first implementation:
 
-.. literalinclude:: ../../bonobo/examples/tutorials/tut02_02_write.py
+.. literalinclude:: ../../bonobo/examples/tutorials/tut02e02_write.py
     :language: python
 
-You can run it and read the output file, you'll see it misses the "map" part of the question. Let's extend
+(run it with :code:`bonobo run -m bonobo.examples.tutorials.tut02e02_write` or :code:`bonobo run myfile.py`)
-:class:`bonobo.io.JsonWriter` to finish the job:
 
-.. literalinclude:: ../../bonobo/examples/tutorials/tut02_03_writeasmap.py
+If you read the output file, you'll see it misses the "map" part of the problem.
+
+Let's extend :class:`bonobo.io.JsonWriter` to finish the job:
+
+.. literalinclude:: ../../bonobo/examples/tutorials/tut02e03_writeasmap.py
     :language: python
 
-You can now run it again, it should produce a nice map. We favored a bit hackish solution here instead of constructing a
+(run it with :code:`bonobo run -m bonobo.examples.tutorials.tut02e03_writeasmap` or :code:`bonobo run myfile.py`)
-map in python then passing the whole to :func:`json.dumps` because we want to work with streams, if you have to
-construct the whole data structure in python, you'll loose a lot of bonobo's benefits.
 
+It should produce a nice map.
+
+We favored a bit hackish solution here instead of constructing a map in python then passing the whole to
+:func:`json.dumps` because we want to work with streams, if you have to construct the whole data structure in python,
+you'll loose a lot of bonobo's benefits.
+
+Next
+::::
+
+Time to write some more advanced transformations, with service dependencies: :doc:`tut03`.

docs/tutorial/tut03.rst

@@ -0,0 +1,200 @@
+Configurables and Services
+==========================
+
+.. note::
+
+    This section lacks completeness, sorry for that (but you can still read it!).
+
+In the last section, we used a few new tools.
+
+Class-based transformations and configurables
+:::::::::::::::::::::::::::::::::::::::::::::
+
+Bonobo is a bit dumb. If something is callable, it considers it can be used as a transformation, and it's up to the
+user to provide callables that logically fits in a graph.
+
+You can use plain python objects with a `__call__()` method, and it ill just work.
+
+As a lot of transformations needs common machinery, there is a few tools to quickly build transformations, most of
+them requiring your class to subclass :class:`bonobo.config.Configurable`.
+
+Configurables allows to use the following features:
+
+* You can add **Options** (using the :class:`bonobo.config.Option` descriptor). Options can be positional, or keyword
+  based, can have a default value and will be consumed from the constructor arguments.
+
+    .. code-block:: python
+
+        from bonobo.config import Configurable, Option
+
+        class PrefixIt(Configurable):
+            prefix = Option(str, positional=True, default='>>>')
+
+            def call(self, row):
+                return self.prefix + ' ' + row
+
+        prefixer = PrefixIt('$')
+
+* You can add **Services** (using the :class:`bonobo.config.Service` descriptor). Services are a subclass of
+  :class:`bonobo.config.Option`, sharing the same basics, but specialized in the definition of "named services" that
+  will be resolved at runtime (a.k.a for which we will provide an implementation at runtime). We'll dive more into that
+  in the next section
+
+    .. code-block:: python
+
+        from bonobo.config import Configurable, Option, Service
+
+        class HttpGet(Configurable):
+            url = Option(default='https://jsonplaceholder.typicode.com/users')
+            http = Service('http.client')
+
+            def call(self, http):
+                resp = http.get(self.url)
+
+                for row in resp.json():
+                    yield row
+
+        http_get = HttpGet()
+
+
+* You can add **Methods** (using the :class:`bonobo.config.Method` descriptor). :class:`bonobo.config.Method` is a
+  subclass of :class:`bonobo.config.Option` that allows to pass callable parameters, either to the class constructor,
+  or using the class as a decorator.
+
+    .. code-block:: python
+
+        from bonobo.config import Configurable, Method
+
+        class Applier(Configurable):
+            apply = Method()
+
+            def call(self, row):
+                return self.apply(row)
+
+        @Applier
+        def Prefixer(self, row):
+            return 'Hello, ' + row
+
+        prefixer = Prefixer()
+
+* You can add **ContextProcessors**, which are an advanced feature we won't introduce here. If you're familiar with
+  pytest, you can think of them as pytest fixtures, execution wise.
+
+Services
+::::::::
+
+The motivation behind services is mostly separation of concerns, testability and deployability.
+
+Usually, your transformations will depend on services (like a filesystem, an http client, a database, a rest api, ...).
+Those services can very well be hardcoded in the transformations, but there is two main drawbacks:
+
+* You won't be able to change the implementation depending on the current environment (development laptop versus
+  production servers, bug-hunting session versus execution, etc.)
+* You won't be able to test your transformations without testing the associated services.
+
+To overcome those caveats of hardcoding things, we define Services in the configurable, which are basically
+string-options of the service names, and we provide an implementation at the last moment possible.
+
+There are two ways of providing implementations:
+
+* Either file-wide, by providing a `get_services()` function that returns a dict of named implementations (we did so
+  with filesystems in the previous step, :doc:`tut02.rst`)
+* Either directory-wide, by providing a `get_services()` function in a specially named `_services.py` file.
+
+The first is simpler if you only have one transformation graph in one file, the second allows to group coherent
+transformations together in a directory and share the implementations.
+
+Let's see how to use it, starting from the previous service example:
+
+.. code-block:: python
+
+    from bonobo.config import Configurable, Option, Service
+
+    class HttpGet(Configurable):
+        url = Option(default='https://jsonplaceholder.typicode.com/users')
+        http = Service('http.client')
+
+        def call(self, http):
+            resp = http.get(self.url)
+
+            for row in resp.json():
+                yield row
+
+We defined an "http.client" service, that obviously should have a `get()` method, returning responses that have a
+`json()` method.
+
+Let's provide two implementations for that. The first one will be using `requests <http://docs.python-requests.org/>`_,
+that coincidally satisfies the described interface:
+
+.. code-block:: python
+
+    import bonobo
+    import requests
+
+    def get_services():
+        return {
+            'http.client': requests
+        }
+
+    graph = bonobo.Graph(
+        HttpGet(),
+        print,
+    )
+
+If you run this code, you should see some mock data returned by the webservice we called (assuming it's up and you can
+reach it).
+
+Now, the second implementation will replace that with a mock, used for testing purposes:
+
+.. code-block:: python
+
+    class HttpResponseStub:
+        def json(self):
+            return [
+                {'id': 1, 'name': 'Leanne Graham', 'username': 'Bret', 'email': 'Sincere@april.biz', 'address': {'street': 'Kulas Light', 'suite': 'Apt. 556', 'city': 'Gwenborough', 'zipcode': '92998-3874', 'geo': {'lat': '-37.3159', 'lng': '81.1496'}}, 'phone': '1-770-736-8031 x56442', 'website': 'hildegard.org', 'company': {'name': 'Romaguera-Crona', 'catchPhrase': 'Multi-layered client-server neural-net', 'bs': 'harness real-time e-markets'}},
+                {'id': 2, 'name': 'Ervin Howell', 'username': 'Antonette', 'email': 'Shanna@melissa.tv', 'address': {'street': 'Victor Plains', 'suite': 'Suite 879', 'city': 'Wisokyburgh', 'zipcode': '90566-7771', 'geo': {'lat': '-43.9509', 'lng': '-34.4618'}}, 'phone': '010-692-6593 x09125', 'website': 'anastasia.net', 'company': {'name': 'Deckow-Crist', 'catchPhrase': 'Proactive didactic contingency', 'bs': 'synergize scalable supply-chains'}},
+            ]
+
+    class HttpStub:
+        def get(self, url):
+            return HttpResponseStub()
+
+    def get_services():
+        return {
+            'http.client': HttpStub()
+        }
+
+    graph = bonobo.Graph(
+        HttpGet(),
+        print,
+    )
+
+The `Graph` definition staying the exact same, you can easily substitute the `_services.py` file depending on your
+environment (the way you're doing this is out of bonobo scope and heavily depends on your usual way of managing
+configuration files on different platforms).
+
+Starting with bonobo 0.5 (not yet released), you will be able to use service injections with function-based
+transformations too, using the `bonobo.config.requires` decorator to mark a dependency.
+
+.. code-block:: python
+
+    from bonobo.config import requires
+
+    @requires('http.client')
+    def http_get(http):
+        resp = http.get('https://jsonplaceholder.typicode.com/users')
+
+        for row in resp.json():
+            yield row
+
+
+Read more
+:::::::::
+
+* :doc:`/guide/services`
+* :doc:`/reference/api_config`
+
+Next
+::::
+
+:doc:`tut04`.

docs/tutorial/tut04.rst

@@ -0,0 +1,214 @@
+Working with databases
+======================
+
+Databases (and especially SQL databases here) are not the focus of Bonobo, thus support for it is not (and will never
+be) included in the main package. Instead, working with databases is done using third party, well maintained and
+specialized packages, like SQLAlchemy, or other database access libraries from the python cheese shop.
+
+.. note::
+
+    SQLAlchemy extension is not yet complete. Things may be not optimal, and some APIs will change. You can still try,
+    of course.
+
+    Consider the following document as a "preview" (yes, it should work, yes it may break in the future).
+
+    Also, note that for early development stages, we explicitely support only PostreSQL, although it may work well
+    with `any other database supported by SQLAlchemy <http://docs.sqlalchemy.org/en/latest/core/engines.html#supported-databases>`_.
+
+First, read https://www.bonobo-project.org/with/sqlalchemy for instructions on how to install. You **do need** the
+bleeding edge version of `bonobo` and `bonobo-sqlalchemy` to make this work.
+
+Requirements
+::::::::::::
+
+Once you installed `bonobo_sqlalchemy` (read https://www.bonobo-project.org/with/sqlalchemy to use bleeding edge
+version), install the following additional packages:
+
+.. code-block:: shell-session
+
+    $ pip install -U python-dotenv psycopg2 awesome-slugify
+
+Those packages are not required by the extension, but `python-dotenv` will help us configure the database DSN, and
+`psycopg2` is required by SQLAlchemy to connect to PostgreSQL databases. Also, we'll use a slugifier to create unique
+identifiers for the database (maybe not what you'd do in the real world, but very much sufficient for example purpose).
+
+Configure a database engine
+:::::::::::::::::::::::::::
+
+Open your `_services.py` file and replace the code:
+
+.. code-block:: python
+
+    import bonobo, dotenv, logging, os
+    from bonobo_sqlalchemy.util import create_postgresql_engine
+
+    dotenv.load_dotenv(dotenv.find_dotenv())
+    logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+
+    def get_services():
+        return {
+            'fs': bonobo.open_examples_fs('datasets'),
+            'fs.output': bonobo.open_fs(),
+            'sqlalchemy.engine': create_postgresql_engine(**{
+                    'name': 'tutorial',
+                    'user': 'tutorial',
+                    'pass': 'tutorial',
+                })
+        }
+
+The `create_postgresql_engine` is a tiny function building the DSN from reasonable defaults, that you can override
+either by providing kwargs, or with system environment variables. If you want to override something, open the `.env`
+file and add values for one or more of `POSTGRES_NAME`, `POSTGRES_USER`, 'POSTGRES_PASS`, `POSTGRES_HOST`,
+`POSTGRES_PORT`. Please note that kwargs always have precedence on environment, but that you should prefer using
+environment variables for anything that is not immutable from one platform to another.
+
+Add database operation to the graph
+:::::::::::::::::::::::::::::::::::
+
+Let's create a `tutorial/pgdb.py` job:
+
+.. code-block:: python
+
+    import bonobo
+    import bonobo_sqlalchemy
+
+    from bonobo.examples.tutorials.tut02e03_writeasmap import graph, split_one_to_map
+
+    graph = graph.copy()
+    graph.add_chain(
+        bonobo_sqlalchemy.InsertOrUpdate('coffeeshops'),
+        _input=split_one_to_map
+    )
+
+Notes here:
+
+* We use the code from :doc:`tut02`, which is bundled with bonobo in the `bonobo.examples.tutorials` package.
+* We "fork" the graph, by creating a copy and appending a new "chain", starting at a point that exists in the other
+  graph.
+* We use :class:`bonobo_sqlalchemy.InsertOrUpdate` (which role, in case it is not obvious, is to create database rows if
+  they do not exist yet, or update the existing row, based on a "discriminant" criteria (by default, "id")).
+
+If we run this transformation (with `bonobo run tutorial/pgdb.py`), we should get an error:
+
+.. code-block:: text
+
+     |   File ".../lib/python3.6/site-packages/psycopg2/__init__.py", line 130, in connect
+     |     conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
+     | sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) FATAL:  database "tutorial" does not exist
+     |
+     |
+     | The above exception was the direct cause of the following exception:
+     |
+     | Traceback (most recent call last):
+     |   File ".../bonobo-devkit/bonobo/bonobo/strategies/executor.py", line 45, in _runner
+     |     node_context.start()
+     |   File ".../bonobo-devkit/bonobo/bonobo/execution/base.py", line 75, in start
+     |     self._stack.setup(self)
+     |   File ".../bonobo-devkit/bonobo/bonobo/config/processors.py", line 94, in setup
+     |     _append_to_context = next(_processed)
+     |   File ".../bonobo-devkit/bonobo-sqlalchemy/bonobo_sqlalchemy/writers.py", line 43, in create_connection
+     |     raise UnrecoverableError('Could not create SQLAlchemy connection: {}.'.format(str(exc).replace('\n', ''))) from exc
+     | bonobo.errors.UnrecoverableError: Could not create SQLAlchemy connection: (psycopg2.OperationalError) FATAL:  database "tutorial" does not exist.
+
+The database we requested do not exist. It is not the role of bonobo to do database administration, and thus there is
+no tool here to create neither the database, nor the tables we want to use.
+
+Create database and table
+:::::::::::::::::::::::::
+
+There are however tools in `sqlalchemy` to manage tables, so we'll create the database by ourselves, and ask sqlalchemy
+to create the table:
+
+.. code-block:: shell-session
+
+    $ psql -U postgres -h localhost
+
+    psql (9.6.1, server 9.6.3)
+    Type "help" for help.
+
+    postgres=# CREATE ROLE tutorial WITH LOGIN PASSWORD 'tutorial';
+    CREATE ROLE
+    postgres=# CREATE DATABASE tutorial WITH OWNER=tutorial TEMPLATE=template0 ENCODING='utf-8';
+    CREATE DATABASE
+
+Now, let's use a little trick and add this section to `pgdb.py`:
+
+.. code-block:: python
+
+    import sys
+    from sqlalchemy import Table, Column, String, Integer, MetaData
+
+    def main():
+        from bonobo.commands.run import get_default_services
+        services = get_default_services(__file__)
+        if len(sys.argv) == 1:
+            return bonobo.run(graph, services=services)
+        elif len(sys.argv) == 2 and sys.argv[1] == 'reset':
+            engine = services.get('sqlalchemy.engine')
+            metadata = MetaData()
+
+            coffee_table = Table(
+                'coffeeshops',
+                metadata,
+                Column('id', String(255), primary_key=True),
+                Column('name', String(255)),
+                Column('address', String(255)),
+            )
+
+            metadata.drop_all(engine)
+            metadata.create_all(engine)
+        else:
+            raise NotImplementedError('I do not understand.')
+
+    if __name__ == '__main__':
+        main()
+
+.. note::
+
+    We're using private API of bonobo here, which is unsatisfactory, discouraged and may change. Some way to get the
+    service dictionnary will be added to the public api in a future release of bonobo.
+
+Now run:
+
+.. code-block:: python
+
+    $ python tutorial/pgdb.py reset
+
+Database and table should now exist.
+
+Format the data
+:::::::::::::::
+
+Let's prepare our data for database, and change the `.add_chain(..)` call to do it prior to `InsertOrUpdate(...)`
+
+.. code-block:: python
+
+    from slugify import slugify_url
+
+    def format_for_db(row):
+        name, address = list(row.items())[0]
+        return {
+                'id': slugify_url(name),
+                'name': name,
+                'address': address,
+            }
+
+    # ...
+
+    graph = graph.copy()
+    graph.add_chain(
+        format_for_db,
+        bonobo_sqlalchemy.InsertOrUpdate('coffeeshops'),
+        _input=split_one_to_map
+    )
+
+Run!
+::::
+
+You can now run the script (either with `bonobo run tutorial/pgdb.py` or directly with the python interpreter, as we
+added a "main" section) and the dataset should be inserted in your database. If you run it again, no new rows are
+created.
+
+Note that as we forked the graph from :doc:`tut02`, the transformation also writes the data to `coffeeshops.json`, as
+before.
+

requirements-dev.txt

@@ -1,27 +1,34 @@
 -e .[dev]
-
 alabaster==0.7.10
-astroid==1.5.2
+arrow==0.10.0
 babel==2.4.0
-coverage==4.3.4
+binaryornot==0.4.3
+certifi==2017.4.17
+chardet==3.0.4
+click==6.7
+cookiecutter==1.5.1
+coverage==4.4.1
 docutils==0.13.1
+future==0.16.0
+idna==2.5
 imagesize==0.7.1
-isort==4.2.5
+jinja2-time==0.2.0
 jinja2==2.9.6
-lazy-object-proxy==1.2.2
 markupsafe==1.0
-mccabe==0.6.1
+poyo==0.4.1
-py==1.4.33
+py==1.4.34
 pygments==2.2.0
-pylint==1.7.1
+pytest-cov==2.5.1
-pytest-cov==2.4.0
+pytest-sugar==0.8.0
 pytest-timeout==1.2.0
-pytest==3.0.7
+pytest==3.1.2
+python-dateutil==2.6.0
 pytz==2017.2
-requests==2.13.0
+requests==2.18.1
 six==1.10.0
 snowballstemmer==1.2.1
-sphinx-rtd-theme==0.2.4
+sphinx==1.6.3
-sphinx==1.5.5
+sphinxcontrib-websupport==1.0.1
-wrapt==1.10.10
+termcolor==1.1.0
-yapf==0.16.1
+urllib3==1.21.1
+whichcraft==0.4.1

requirements-docker.txt

@@ -0,0 +1,20 @@
+-e .[docker]
+appdirs==1.4.3
+bonobo-docker==0.2.9
+certifi==2017.4.17
+chardet==3.0.4
+colorama==0.3.9
+docker-pycreds==0.2.1
+docker==2.3.0
+fs==2.0.4
+idna==2.5
+packaging==16.8
+pbr==3.1.1
+psutil==5.2.2
+pyparsing==2.2.0
+pytz==2017.2
+requests==2.18.1
+six==1.10.0
+stevedore==1.23.0
+urllib3==1.21.1
+websocket-client==0.44.0

requirements-jupyter.txt

@@ -1,31 +1,30 @@
 -e .[jupyter]
-
 appnope==0.1.0
 bleach==2.0.0
 decorator==4.0.11
-entrypoints==0.2.2
+entrypoints==0.2.3
 html5lib==0.999999999
 ipykernel==4.6.1
 ipython-genutils==0.2.0
-ipython==6.0.0
+ipython==6.1.0
 ipywidgets==6.0.0
 jedi==0.10.2
 jinja2==2.9.6
 jsonschema==2.6.0
-jupyter-client==5.0.1
+jupyter-client==5.1.0
 jupyter-console==5.1.0
 jupyter-core==4.3.0
 jupyter==1.0.0
 markupsafe==1.0
 mistune==0.7.4
-nbconvert==5.1.1
+nbconvert==5.2.1
 nbformat==4.3.0
 notebook==5.0.0
 pandocfilters==1.4.1
 pexpect==4.2.1
 pickleshare==0.7.4
 prompt-toolkit==1.0.14
-ptyprocess==0.5.1
+ptyprocess==0.5.2
 pygments==2.2.0
 python-dateutil==2.6.0
 pyzmq==16.0.2
@@ -33,7 +32,7 @@ qtconsole==4.3.0
 simplegeneric==0.8.1
 six==1.10.0
 terminado==0.6
-testpath==0.3
+testpath==0.3.1
 tornado==4.5.1
 traitlets==4.3.2
 wcwidth==0.1.7

requirements.txt

@@ -1,12 +1,16 @@
 -e .
-
 appdirs==1.4.3
+certifi==2017.4.17
+chardet==3.0.4
 colorama==0.3.9
-enum34==1.1.6
+fs==2.0.4
-fs==2.0.3
+idna==2.5
-pbr==3.0.0
+packaging==16.8
+pbr==3.1.1
 psutil==5.2.2
+pyparsing==2.2.0
 pytz==2017.2
-requests==2.13.0
+requests==2.18.1
 six==1.10.0
-stevedore==1.21.0
+stevedore==1.23.0
+urllib3==1.21.1

setup.py

@@ -18,13 +18,19 @@ except NameError:
 
 
 # Get the long description from the README file
-with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
+try:
+    with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
         long_description = f.read()
+except:
+    long_description = ''
 
 # Get the classifiers from the classifiers file
 tolines = lambda c: list(filter(None, map(lambda s: s.strip(), c.split('\n'))))
-with open(path.join(here, 'classifiers.txt'), encoding='utf-8') as f:
+try:
+    with open(path.join(here, 'classifiers.txt'), encoding='utf-8') as f:
         classifiers = tolines(f.read())
+except:
+    classifiers = []
 
 version_ns = {}
 try:
@@ -35,41 +41,36 @@ else:
     version = version_ns.get('__version__', 'dev')
 
 setup(
-    name='bonobo',
+    author='Romain Dorgueil',
+    author_email='romain@dorgueil.net',
     description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for '
                  'python 3.5+.'),
     license='Apache License, Version 2.0',
-    install_requires=[
+    name='bonobo',
-        'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0'
-    ],
     version=version,
     long_description=long_description,
     classifiers=classifiers,
     packages=find_packages(exclude=['ez_setup', 'example', 'test']),
     include_package_data=True,
-    data_files=[
+    install_requires=[
-        (
+        'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'packaging (>= 16, < 17)', 'psutil (>= 5.2, < 6.0)',
-            'share/jupyter/nbextensions/bonobo-jupyter', [
+        'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)'
-                'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js',
-                'bonobo/ext/jupyter/static/index.js.map'
-            ]
-        )
     ],
     extras_require={
         'dev': [
-            'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx',
+            'cookiecutter (>= 1.5, < 1.6)', 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)',
-            'sphinx_rtd_theme', 'yapf'
+            'pytest-cov (>= 2.5, < 3.0)', 'pytest-sugar (>= 0.8, < 0.9)', 'pytest-timeout (>= 1, < 2)',
+            'sphinx (>= 1.6, < 2.0)'
         ],
-        'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5']
+        'docker': ['bonobo-docker'],
+        'jupyter': ['ipywidgets (>= 6.0.0, < 7)', 'jupyter (>= 1.0, < 1.1)']
     },
     entry_points={
         'bonobo.commands': [
             'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register',
             'version = bonobo.commands.version:register'
         ],
-        'console_scripts': ['bonobo = bonobo.commands:entrypoint'],
+        'console_scripts': ['bonobo = bonobo.commands:entrypoint']
-        'edgy.project.features': ['bonobo = '
-                                  'bonobo.ext.edgy.project.feature:BonoboFeature']
     },
     url='https://www.bonobo-project.org/',
     download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version),

tests/config/test_configurables.py

@@ -2,7 +2,6 @@ import pytest
 
 from bonobo.config.configurables import Configurable
 from bonobo.config.options import Option
-from bonobo.config.services import Container, Service, validate_service_name
 
 
 class MyConfigurable(Configurable):
@@ -25,28 +24,6 @@ class MyConfigurableUsingPositionalOptions(MyConfigurable):
     third = Option(str, required=False, positional=True)
 
 
-class PrinterInterface():
-    def print(self, *args):
-        raise NotImplementedError()
-
-
-class ConcretePrinter(PrinterInterface):
-    def __init__(self, prefix):
-        self.prefix = prefix
-
-    def print(self, *args):
-        return ';'.join((self.prefix, *args))
-
-
-class MyServiceDependantConfigurable(Configurable):
-    printer = Service(
-        PrinterInterface,
-    )
-
-    def __call__(self, printer: PrinterInterface, *args):
-        return printer.print(*args)
-
-
 def test_missing_required_option_error():
     with pytest.raises(TypeError) as exc:
         MyConfigurable()
@@ -107,39 +84,5 @@ def test_option_resolution_order():
     assert o.integer == None
 
 
-def test_service_name_validator():
-    assert validate_service_name('foo') == 'foo'
-    assert validate_service_name('foo.bar') == 'foo.bar'
-    assert validate_service_name('Foo') == 'Foo'
-    assert validate_service_name('Foo.Bar') == 'Foo.Bar'
-    assert validate_service_name('Foo.a0') == 'Foo.a0'
-
-    with pytest.raises(ValueError):
-        validate_service_name('foo.0')
-
-    with pytest.raises(ValueError):
-        validate_service_name('0.foo')
-
-
-SERVICES = Container(
-    printer0=ConcretePrinter(prefix='0'),
-    printer1=ConcretePrinter(prefix='1'),
-)
-
-
-def test_service_dependency():
-    o = MyServiceDependantConfigurable(printer='printer0')
-
-    assert o(SERVICES.get('printer0'), 'foo', 'bar') == '0;foo;bar'
-    assert o(SERVICES.get('printer1'), 'bar', 'baz') == '1;bar;baz'
-    assert o(*SERVICES.args_for(o), 'foo', 'bar') == '0;foo;bar'
-
-
-def test_service_dependency_unavailable():
-    o = MyServiceDependantConfigurable(printer='printer2')
-    with pytest.raises(KeyError):
-        SERVICES.args_for(o)
-
-
 def test_option_positional():
     o = MyConfigurableUsingPositionalOptions('1', '2', '3', required_str='hello')

tests/config/test_methods.py

@@ -28,7 +28,10 @@ def test_define_with_decorator():
     def Concrete(self, *args, **kwargs):
         calls.append((args, kwargs, ))
 
+    assert callable(Concrete.handler)
     t = Concrete('foo', bar='baz')
+
+    assert callable(t.handler)
     assert len(calls) == 0
     t()
     assert len(calls) == 1
@@ -41,6 +44,7 @@ def test_define_with_argument():
         calls.append((args, kwargs, ))
 
     t = MethodBasedConfigurable('foo', bar='baz', handler=concrete_handler)
+    assert callable(t.handler)
     assert len(calls) == 0
     t()
     assert len(calls) == 1
@@ -54,6 +58,7 @@ def test_define_with_inheritance():
             calls.append((args, kwargs, ))
 
     t = Inheriting('foo', bar='baz')
+    assert callable(t.handler)
     assert len(calls) == 0
     t()
     assert len(calls) == 1
@@ -69,7 +74,9 @@ def test_inheritance_then_decorate():
     def Concrete(self, *args, **kwargs):
         calls.append((args, kwargs, ))
 
+    assert callable(Concrete.handler)
     t = Concrete('foo', bar='baz')
+    assert callable(t.handler)
     assert len(calls) == 0
     t()
     assert len(calls) == 1

tests/config/test_services.py

@@ -0,0 +1,96 @@
+import threading
+import time
+
+import pytest
+
+from bonobo.config import Configurable, Container, Exclusive, Service
+from bonobo.config.services import validate_service_name
+
+
+class PrinterInterface():
+    def print(self, *args):
+        raise NotImplementedError()
+
+
+class ConcretePrinter(PrinterInterface):
+    def __init__(self, prefix):
+        self.prefix = prefix
+
+    def print(self, *args):
+        return ';'.join((self.prefix, *args))
+
+
+SERVICES = Container(
+    printer0=ConcretePrinter(prefix='0'),
+    printer1=ConcretePrinter(prefix='1'),
+)
+
+
+class MyServiceDependantConfigurable(Configurable):
+    printer = Service(
+        PrinterInterface,
+    )
+
+    def __call__(self, printer: PrinterInterface, *args):
+        return printer.print(*args)
+
+
+def test_service_name_validator():
+    assert validate_service_name('foo') == 'foo'
+    assert validate_service_name('foo.bar') == 'foo.bar'
+    assert validate_service_name('Foo') == 'Foo'
+    assert validate_service_name('Foo.Bar') == 'Foo.Bar'
+    assert validate_service_name('Foo.a0') == 'Foo.a0'
+
+    with pytest.raises(ValueError):
+        validate_service_name('foo.0')
+
+    with pytest.raises(ValueError):
+        validate_service_name('0.foo')
+
+
+def test_service_dependency():
+    o = MyServiceDependantConfigurable(printer='printer0')
+
+    assert o(SERVICES.get('printer0'), 'foo', 'bar') == '0;foo;bar'
+    assert o(SERVICES.get('printer1'), 'bar', 'baz') == '1;bar;baz'
+    assert o(*SERVICES.args_for(o), 'foo', 'bar') == '0;foo;bar'
+
+
+def test_service_dependency_unavailable():
+    o = MyServiceDependantConfigurable(printer='printer2')
+    with pytest.raises(KeyError):
+        SERVICES.args_for(o)
+
+
+class VCR:
+    def __init__(self):
+        self.tape = []
+
+    def append(self, x):
+        return self.tape.append(x)
+
+
+def test_exclusive():
+    vcr = VCR()
+    vcr.append('hello')
+
+    def record(prefix, vcr=vcr):
+        with Exclusive(vcr):
+            for i in range(5):
+                vcr.append(' '.join((prefix, str(i))))
+                time.sleep(0.05)
+
+    threads = [threading.Thread(target=record, args=(str(i), )) for i in range(5)]
+
+    for thread in threads:
+        thread.start()
+        time.sleep(0.01)  # this is not good practice, how to test this without sleeping ?? XXX
+
+    for thread in threads:
+        thread.join()
+
+    assert vcr.tape == [
+        'hello', '0 0', '0 1', '0 2', '0 3', '0 4', '1 0', '1 1', '1 2', '1 3', '1 4', '2 0', '2 1', '2 2', '2 3',
+        '2 4', '3 0', '3 1', '3 2', '3 3', '3 4', '4 0', '4 1', '4 2', '4 3', '4 4'
+    ]

tests/io/test_csv.py

@@ -1,42 +1,56 @@
 import pytest
 
-from bonobo import Bag, CsvReader, CsvWriter, open_fs
+from bonobo import Bag, CsvReader, CsvWriter, settings
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
-from bonobo.util.testing import CapturingNodeExecutionContext
+from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
+
+csv_tester = FilesystemTester('csv')
+csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar'
 
 
-def test_write_csv_to_file(tmpdir):
+def test_write_csv_to_file_arg0(tmpdir):
-    fs, filename = open_fs(tmpdir), 'output.csv'
+    fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
-
-    writer = CsvWriter(path=filename)
-    context = NodeExecutionContext(writer, services={'fs': fs})
 
+    with NodeExecutionContext(CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
         context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
-
-    context.start()
         context.step()
         context.step()
-    context.stop()
 
-    assert fs.open(filename).read() == 'foo\nbar\nbaz\n'
+    with fs.open(filename) as fp:
+        assert fp.read() == 'foo\nbar\nbaz\n'
 
     with pytest.raises(AttributeError):
         getattr(context, 'file')
 
 
-def test_read_csv_from_file(tmpdir):
+@pytest.mark.parametrize('add_kwargs', ({}, {
-    fs, filename = open_fs(tmpdir), 'input.csv'
+    'ioformat': settings.IOFORMAT_KWARGS,
-    fs.open(filename, 'w').write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar')
+}, ))
+def test_write_csv_to_file_kwargs(tmpdir, add_kwargs):
+    fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
 
-    reader = CsvReader(path=filename, delimiter=',')
+    with NodeExecutionContext(CsvWriter(path=filename, **add_kwargs), services=services) as context:
+        context.write(BEGIN, Bag(**{'foo': 'bar'}), Bag(**{'foo': 'baz', 'ignore': 'this'}), END)
+        context.step()
+        context.step()
 
-    context = CapturingNodeExecutionContext(reader, services={'fs': fs})
+    with fs.open(filename) as fp:
+        assert fp.read() == 'foo\nbar\nbaz\n'
 
-    context.start()
+    with pytest.raises(AttributeError):
+        getattr(context, 'file')
+
+
+def test_read_csv_from_file_arg0(tmpdir):
+    fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
+
+    with CapturingNodeExecutionContext(
+        CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0),
+        services=services,
+    ) as context:
         context.write(BEGIN, Bag(), END)
         context.step()
-    context.stop()
 
     assert len(context.send.mock_calls) == 2
 
@@ -55,3 +69,35 @@ def test_read_csv_from_file(tmpdir):
         'b': 'b bar',
         'c': 'c bar',
     }
+
+
+def test_read_csv_from_file_kwargs(tmpdir):
+    fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
+
+    with CapturingNodeExecutionContext(
+        CsvReader(path=filename, delimiter=','),
+        services=services,
+    ) as context:
+        context.write(BEGIN, Bag(), END)
+        context.step()
+
+    assert len(context.send.mock_calls) == 2
+
+    args0, kwargs0 = context.send.call_args_list[0]
+    assert len(args0) == 1 and not len(kwargs0)
+    args1, kwargs1 = context.send.call_args_list[1]
+    assert len(args1) == 1 and not len(kwargs1)
+
+    _args, _kwargs = args0[0].get()
+    assert not len(_args) and _kwargs == {
+        'a': 'a foo',
+        'b': 'b foo',
+        'c': 'c foo',
+    }
+
+    _args, _kwargs = args1[0].get()
+    assert not len(_args) and _kwargs == {
+        'a': 'a bar',
+        'b': 'b bar',
+        'c': 'c bar',
+    }

tests/io/test_file.py

@@ -1,9 +1,22 @@
 import pytest
 
-from bonobo import Bag, FileReader, FileWriter, open_fs
+from bonobo import Bag, FileReader, FileWriter
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
-from bonobo.util.testing import CapturingNodeExecutionContext
+from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
+
+txt_tester = FilesystemTester('txt')
+txt_tester.input_data = 'Hello\nWorld\n'
+
+
+def test_file_writer_contextless(tmpdir):
+    fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
+
+    with FileWriter(path=filename).open(fs) as fp:
+        fp.write('Yosh!')
+
+    with fs.open(filename) as fp:
+        assert fp.read() == 'Yosh!'
 
 
 @pytest.mark.parametrize(
@@ -14,43 +27,23 @@ from bonobo.util.testing import CapturingNodeExecutionContext
     ]
 )
 def test_file_writer_in_context(tmpdir, lines, output):
-    fs, filename = open_fs(tmpdir), 'output.txt'
+    fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
 
-    writer = FileWriter(path=filename)
+    with NodeExecutionContext(FileWriter(path=filename), services=services) as context:
-    context = NodeExecutionContext(writer, services={'fs': fs})
-
-    context.start()
         context.write(BEGIN, *map(Bag, lines), END)
         for _ in range(len(lines)):
             context.step()
-    context.stop()
 
-    assert fs.open(filename).read() == output
+    with fs.open(filename) as fp:
+        assert fp.read() == output
 
 
-def test_file_writer_out_of_context(tmpdir):
+def test_file_reader(tmpdir):
-    fs, filename = open_fs(tmpdir), 'output.txt'
+    fs, filename, services = txt_tester.get_services_for_reader(tmpdir)
 
-    writer = FileWriter(path=filename)
+    with CapturingNodeExecutionContext(FileReader(path=filename), services=services) as context:
-
-    with writer.open(fs) as fp:
-        fp.write('Yosh!')
-
-    assert fs.open(filename).read() == 'Yosh!'
-
-
-def test_file_reader_in_context(tmpdir):
-    fs, filename = open_fs(tmpdir), 'input.txt'
-
-    fs.open(filename, 'w').write('Hello\nWorld\n')
-
-    reader = FileReader(path=filename)
-    context = CapturingNodeExecutionContext(reader, services={'fs': fs})
-
-    context.start()
         context.write(BEGIN, Bag(), END)
         context.step()
-    context.stop()
 
     assert len(context.send.mock_calls) == 2
 

tests/io/test_json.py

@@ -1,42 +1,48 @@
 import pytest
 
-from bonobo import Bag, JsonReader, JsonWriter, open_fs
+from bonobo import Bag, JsonReader, JsonWriter, settings
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
-from bonobo.util.testing import CapturingNodeExecutionContext
+from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
+
+json_tester = FilesystemTester('json')
+json_tester.input_data = '''[{"x": "foo"},{"x": "bar"}]'''
 
 
-def test_write_json_to_file(tmpdir):
+def test_write_json_arg0(tmpdir):
-    fs, filename = open_fs(tmpdir), 'output.json'
+    fs, filename, services = json_tester.get_services_for_writer(tmpdir)
 
-    writer = JsonWriter(path=filename)
+    with NodeExecutionContext(JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
-    context = NodeExecutionContext(writer, services={'fs': fs})
-
-    context.start()
         context.write(BEGIN, Bag({'foo': 'bar'}), END)
         context.step()
-    context.stop()
 
-    assert fs.open(filename).read() == '[{"foo": "bar"}]'
+    with fs.open(filename) as fp:
-
+        assert fp.read() == '[{"foo": "bar"}]'
-    with pytest.raises(AttributeError):
-        getattr(context, 'file')
-
-    with pytest.raises(AttributeError):
-        getattr(context, 'first')
 
 
-def test_read_json_from_file(tmpdir):
+@pytest.mark.parametrize('add_kwargs', ({}, {
-    fs, filename = open_fs(tmpdir), 'input.json'
+    'ioformat': settings.IOFORMAT_KWARGS,
-    fs.open(filename, 'w').write('[{"x": "foo"},{"x": "bar"}]')
+}, ))
-    reader = JsonReader(path=filename)
+def test_write_json_kwargs(tmpdir, add_kwargs):
+    fs, filename, services = json_tester.get_services_for_writer(tmpdir)
 
-    context = CapturingNodeExecutionContext(reader, services={'fs': fs})
+    with NodeExecutionContext(JsonWriter(filename, **add_kwargs), services=services) as context:
+        context.write(BEGIN, Bag(**{'foo': 'bar'}), END)
+        context.step()
 
-    context.start()
+    with fs.open(filename) as fp:
+        assert fp.read() == '[{"foo": "bar"}]'
+
+
+def test_read_json_arg0(tmpdir):
+    fs, filename, services = json_tester.get_services_for_reader(tmpdir)
+
+    with CapturingNodeExecutionContext(
+        JsonReader(filename, ioformat=settings.IOFORMAT_ARG0),
+        services=services,
+    ) as context:
         context.write(BEGIN, Bag(), END)
         context.step()
-    context.stop()
 
     assert len(context.send.mock_calls) == 2
 

tests/io/test_pickle.py

@@ -0,0 +1,54 @@
+import pickle
+
+import pytest
+
+from bonobo import Bag, PickleReader, PickleWriter, settings
+from bonobo.constants import BEGIN, END
+from bonobo.execution.node import NodeExecutionContext
+from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
+
+pickle_tester = FilesystemTester('pkl', mode='wb')
+pickle_tester.input_data = pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']])
+
+
+def test_write_pickled_dict_to_file(tmpdir):
+    fs, filename, services = pickle_tester.get_services_for_writer(tmpdir)
+
+    with NodeExecutionContext(PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
+        context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
+        context.step()
+        context.step()
+
+    with fs.open(filename, 'rb') as fp:
+        assert pickle.loads(fp.read()) == {'foo': 'bar'}
+
+    with pytest.raises(AttributeError):
+        getattr(context, 'file')
+
+
+def test_read_pickled_list_from_file(tmpdir):
+    fs, filename, services = pickle_tester.get_services_for_reader(tmpdir)
+
+    with CapturingNodeExecutionContext(
+        PickleReader(filename, ioformat=settings.IOFORMAT_ARG0), services=services
+    ) as context:
+        context.write(BEGIN, Bag(), END)
+        context.step()
+
+    assert len(context.send.mock_calls) == 2
+
+    args0, kwargs0 = context.send.call_args_list[0]
+    assert len(args0) == 1 and not len(kwargs0)
+    args1, kwargs1 = context.send.call_args_list[1]
+    assert len(args1) == 1 and not len(kwargs1)
+
+    assert args0[0].args[0] == {
+        'a': 'a foo',
+        'b': 'b foo',
+        'c': 'c foo',
+    }
+    assert args1[0].args[0] == {
+        'a': 'a bar',
+        'b': 'b bar',
+        'c': 'c bar',
+    }

tests/test_commands.py

@@ -1,10 +1,26 @@
+import runpy
+import sys
+from unittest.mock import patch
+
 import pkg_resources
 import pytest
 
-from bonobo import get_examples_path
+from bonobo import __main__, __version__, get_examples_path
 from bonobo.commands import entrypoint
 
 
+def runner_entrypoint(*args):
+    return entrypoint(list(args))
+
+
+def runner_module(*args):
+    with patch.object(sys, 'argv', ['bonobo', *args]):
+        return runpy.run_path(__main__.__file__, run_name='__main__')
+
+
+all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module])
+
+
 def test_entrypoint():
     commands = {}
 
@@ -13,23 +29,51 @@ def test_entrypoint():
 
     assert 'init' in commands
     assert 'run' in commands
+    assert 'version' in commands
 
 
-def test_no_command(capsys):
+@all_runners
+def test_no_command(runner, capsys):
     with pytest.raises(SystemExit):
-        entrypoint([])
+        runner()
     _, err = capsys.readouterr()
     assert 'error: the following arguments are required: command' in err
 
 
-def test_init():
+@all_runners
-    pass  # need ext dir
+def test_run(runner, capsys):
-
+    runner('run', '--quiet', get_examples_path('types/strings.py'))
-
-def test_run(capsys):
-    entrypoint(['run', '--quiet', get_examples_path('types/strings.py')])
     out, err = capsys.readouterr()
     out = out.split('\n')
     assert out[0].startswith('Foo ')
     assert out[1].startswith('Bar ')
     assert out[2].startswith('Baz ')
+
+
+@all_runners
+def test_run_module(runner, capsys):
+    runner('run', '--quiet', '-m', 'bonobo.examples.types.strings')
+    out, err = capsys.readouterr()
+    out = out.split('\n')
+    assert out[0].startswith('Foo ')
+    assert out[1].startswith('Bar ')
+    assert out[2].startswith('Baz ')
+
+
+@all_runners
+def test_run_path(runner, capsys):
+    runner('run', '--quiet', get_examples_path('types'))
+    out, err = capsys.readouterr()
+    out = out.split('\n')
+    assert out[0].startswith('Foo ')
+    assert out[1].startswith('Bar ')
+    assert out[2].startswith('Baz ')
+
+
+@all_runners
+def test_version(runner, capsys):
+    runner('version')
+    out, err = capsys.readouterr()
+    out = out.strip()
+    assert out.startswith('bonobo ')
+    assert __version__ in out

tests/test_publicapi.py

@@ -1,4 +1,4 @@
-import types
+import inspect
 
 
 def test_wildcard_import():
@@ -10,7 +10,7 @@ def test_wildcard_import():
         if name.startswith('_'):
             continue
         attr = getattr(bonobo, name)
-        if isinstance(attr, types.ModuleType):
+        if inspect.ismodule(attr):
             continue
 
         assert name in bonobo.__all__

tests/util/test_objects.py

@@ -1,4 +1,9 @@
+import operator
+
+import pytest
+
 from bonobo.util.objects import Wrapper, get_name, ValueHolder
+from bonobo.util.testing import optional_contextmanager
 
 
 class foo:
@@ -52,3 +57,56 @@ def test_valueholder():
     assert y == x
     assert y is not x
     assert repr(x) == repr(y) == repr(43)
+
+
+unsupported_operations = {
+    int: {operator.matmul},
+    str: {
+        operator.sub, operator.mul, operator.matmul, operator.floordiv, operator.truediv, operator.mod, divmod,
+        operator.pow, operator.lshift, operator.rshift, operator.and_, operator.xor, operator.or_
+    },
+}
+
+
+@pytest.mark.parametrize('x,y', [(5, 3), (0, 10), (0, 0), (1, 1), ('foo', 'bar'), ('', 'baz!')])
+@pytest.mark.parametrize(
+    'operation,inplace_operation', [
+        (operator.add, operator.iadd),
+        (operator.sub, operator.isub),
+        (operator.mul, operator.imul),
+        (operator.matmul, operator.imatmul),
+        (operator.truediv, operator.itruediv),
+        (operator.floordiv, operator.ifloordiv),
+        (operator.mod, operator.imod),
+        (divmod, None),
+        (operator.pow, operator.ipow),
+        (operator.lshift, operator.ilshift),
+        (operator.rshift, operator.irshift),
+        (operator.and_, operator.iand),
+        (operator.xor, operator.ixor),
+        (operator.or_, operator.ior),
+    ]
+)
+def test_valueholder_integer_operations(x, y, operation, inplace_operation):
+    v = ValueHolder(x)
+
+    is_supported = operation not in unsupported_operations.get(type(x), set())
+
+    isdiv = ('div' in operation.__name__) or ('mod' in operation.__name__)
+
+    # forward...
+    with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported):
+        with optional_contextmanager(pytest.raises(ZeroDivisionError), ignore=y or not isdiv):
+            assert operation(x, y) == operation(v, y)
+
+    # backward...
+    with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported):
+        with optional_contextmanager(pytest.raises(ZeroDivisionError), ignore=x or not isdiv):
+            assert operation(y, x) == operation(y, v)
+
+    # in place...
+    if inplace_operation is not None:
+        with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported):
+            with optional_contextmanager(pytest.raises(ZeroDivisionError), ignore=y or not isdiv):
+                inplace_operation(v, y)
+                assert v == operation(x, y)