Merge branch 'master' into develop

.codacy.yml

@@ -1,4 +1,11 @@
 ---
 exclude_paths:
-  - bonobo/examples/
+  - benchmarks/**
-  - bonobo/ext/
+  - bin/**
+  - bonobo/contrib/jupyter/**.js
+  - bonobo/examples/**
+  - bonobo/ext/**
+  - bonobo/util/testing.py
+  - docs/**
+  - setup.py
+  - tests/**

.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+
+[*.py]
+indent = '    '
+indent_size = 4
+indent_style = space
+line_length = 120
+multi_line_output = 5
+
+[Makefile]
+indent_style = tab
+

Makefile

@@ -1,4 +1,4 @@
-# Generated by Medikit 0.6.3 on 2018-07-29.
+# Generated by Medikit 0.6.3 on 2018-08-11.
 # All changes will be overriden.
 # Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
 
@@ -26,8 +26,6 @@ SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build
 SPHINX_OPTIONS ?= 
 SPHINX_SOURCEDIR ?= docs
 SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
-YAPF ?= $(PYTHON) -m yapf
-YAPF_OPTIONS ?= -rip
 SPHINX_AUTOBUILD ?= $(PYTHON_DIRNAME)/sphinx-autobuild
 MEDIKIT ?= $(PYTHON) -m medikit
 MEDIKIT_UPDATE_OPTIONS ?= 
@@ -44,7 +42,7 @@ else ifneq ($(QUICK),)
 	@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
 else
 	@printf "Applying \033[36m%s\033[0m target...\n" $(target)
-	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
+	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
 	$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_INLINE) -r $(PYTHON_REQUIREMENTS_FILE)
 	@mkdir -p .medikit; touch $@
 endif
@@ -62,7 +60,7 @@ else ifneq ($(QUICK),)
 	@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
 else
 	@printf "Applying \033[36m%s\033[0m target...\n" $(target)
-	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
+	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
 	$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DEV_INLINE) -r $(PYTHON_REQUIREMENTS_DEV_FILE)
 	@mkdir -p .medikit; touch $@
 endif
@@ -79,7 +77,7 @@ else ifneq ($(QUICK),)
 	@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
 else
 	@printf "Applying \033[36m%s\033[0m target...\n" $(target)
-	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
+	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
 	$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DOCKER_INLINE) -r $(PYTHON_REQUIREMENTS_DOCKER_FILE)
 	@mkdir -p .medikit; touch $@
 endif
@@ -93,7 +91,7 @@ else ifneq ($(QUICK),)
 	@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
 else
 	@printf "Applying \033[36m%s\033[0m target...\n" $(target)
-	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
+	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
 	$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_JUPYTER_INLINE) -r $(PYTHON_REQUIREMENTS_JUPYTER_FILE)
 	@mkdir -p .medikit; touch $@
 endif
@@ -107,7 +105,7 @@ else ifneq ($(QUICK),)
 	@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
 else
 	@printf "Applying \033[36m%s\033[0m target...\n" $(target)
-	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
+	$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
 	$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_SQLALCHEMY_INLINE) -r $(PYTHON_REQUIREMENTS_SQLALCHEMY_FILE)
 	@mkdir -p .medikit; touch $@
 endif
@@ -118,15 +116,15 @@ test: install-dev  ## Runs the test suite.
 $(SPHINX_SOURCEDIR): install-dev  ##
 	$(SPHINX_BUILD) -b html -D latex_paper_size=a4 $(SPHINX_OPTIONS) $(SPHINX_SOURCEDIR) $(SPHINX_BUILDDIR)/html
 
-format: install-dev  ## Reformats the whole python codebase using yapf.
-	$(YAPF) $(YAPF_OPTIONS) .
-	$(YAPF) $(YAPF_OPTIONS) Projectfile
-
 watch-$(SPHINX_SOURCEDIR):   ##
 	$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)
 
+format:   ## Reformats the whole codebase using our standards (requires black and isort).
+	black -l 120 --skip-string-normalization .
+	isort -rc -o mondrian -o whistle -y .
+
 medikit:   # Checks installed medikit version and updates it if it is outdated.
-	@$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=10.0" "medikit>=$(MEDIKIT_VERSION)"
+	@$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=18.0" "medikit>=$(MEDIKIT_VERSION)"
 
 update: medikit  ## Update project artifacts using medikit.
 	$(MEDIKIT) update $(MEDIKIT_UPDATE_OPTIONS)

Projectfile

@@ -6,7 +6,6 @@ make = require('make')
 pytest = require('pytest')
 python = require('python')
 sphinx = require('sphinx')
-yapf = require('yapf')
 
 python.setup(
     name='bonobo',
@@ -74,14 +73,29 @@ python.add_requirements(
 
 @listen(make.on_generate)
 def on_make_generate(event):
-    event.makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild'
+    makefile = event.makefile
-    event.makefile.add_target(
+    
+    # Sphinx
+    makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild'
+    makefile.add_target(
         'watch-$(SPHINX_SOURCEDIR)',
-        '''
+        '$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)',
-        $(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)
-    ''',
         phony=True
     )
 
+    # Formating
+    makefile.add_target(
+        'format',
+        '''
+            black -l 120 --skip-string-normalization .
+            isort -rc -o mondrian -o whistle -y .
+        ''',
+        phony=True,
+        doc='Reformats the whole codebase using our standards (requires black and isort).'
+    )
+
+
+
+
 
 # vim: ft=python:

benchmarks/parameters.py

@@ -47,10 +47,9 @@ if __name__ == '__main__':
 
     for i in 1, 2, 3:
         print(
-            'j{}'.format(i),
+            'j{}'.format(i), timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i))
-            timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i))
         )
         print(
             'k{}'.format(i),
-            timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i))
+            timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i)),
         )

bin/update_apidoc.py

@@ -1,6 +1,6 @@
 import os
 
-from jinja2 import Environment, DictLoader
+from jinja2 import DictLoader, Environment
 
 __path__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__), '..'))
 
@@ -18,11 +18,7 @@ class Module:
         return '<{} ({})>'.format(self.title, self.name)
 
     def asdict(self):
-        return {
+        return {'name': self.name, 'title': self.title, 'automodule_options': self.automodule_options}
-            'name': self.name,
-            'title': self.title,
-            'automodule_options': self.automodule_options,
-        }
 
     def get_path(self):
         return os.path.join(__path__, apidoc_root, *self.name.split('.')) + '.rst'
@@ -45,9 +41,9 @@ def underlined_filter(txt, chr):
 
 
 env = Environment(
-    loader=DictLoader({
+    loader=DictLoader(
-        'module':
+        {
-        '''
+            'module': '''
 {{ (':mod:`'~title~' <'~name~'>`') | underlined('=') }}
 
 .. currentmodule:: {{ name }}
@@ -56,8 +52,12 @@ env = Environment(
 
 .. automodule:: {{ name }}
 {% for opt in automodule_options %}   :{{ opt }}:{{ "\n" }}{% endfor %}
-    ''' [1:-1] + '\n'
+    '''[
-    })
+                1:-1
+            ]
+            + '\n'
+        }
+    )
 )
 env.filters['underlined'] = underlined_filter
 

bonobo/__init__.py

@@ -7,7 +7,8 @@
 
 import sys
 
-assert sys.version_info >= (3, 5), "Python 3.5+ is required to use Bonobo."
+if sys.version_info < (3, 5):
+    raise RuntimeError('Python 3.5+ is required to use Bonobo.')
 
 from bonobo._api import (
     run,
@@ -58,15 +59,14 @@ __version__ = __version__
 
 def _repr_html_():
     """This allows to easily display a version snippet in Jupyter."""
-    from bonobo.util.pkgs import bonobo_packages
     from bonobo.commands.version import get_versions
 
     return (
         '<div style="padding: 8px;">'
         '  <div style="float: left; width: 20px; height: 20px;">{}</div>'
         '  <pre style="white-space: nowrap; padding-left: 8px">{}</pre>'
-        "</div>"
+        '</div>'
-    ).format(__logo__, "<br/>".join(get_versions(all=True)))
+    ).format(__logo__, '<br/>'.join(get_versions(all=True)))
 
 
 del sys

bonobo/_api.py

@@ -187,7 +187,7 @@ def get_examples_path(*pathsegments):
     import os
     import pathlib
 
-    return str(pathlib.Path(os.path.dirname(__file__), "examples", *pathsegments))
+    return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments))
 
 
 @api.register

bonobo/_version.py

@@ -1 +1 @@
-__version__ = "0.6.2"
+__version__ = '0.6.3'

bonobo/commands/__init__.py

@@ -26,7 +26,9 @@ def entrypoint(args=None):
 
     commands = {}
 
-    def register_extension(ext, commands=commands):
+    def register_extension(ext):
+        nonlocal commands
+
         try:
             parser = subparsers.add_parser(ext.name)
             if isinstance(ext.plugin, type) and issubclass(ext.plugin, BaseCommand):

bonobo/commands/init.py

@@ -33,9 +33,9 @@ class InitCommand(BaseCommand):
         self.logger.info("Generated {} using template {!r}.".format(filename, template_name))
 
     def create_package(self, *, filename):
-        name, ext = os.path.splitext(filename)
+        _, ext = os.path.splitext(filename)
-        if ext != "":
+        if ext != '':
-            raise ValueError("Package names should not have an extension.")
+            raise ValueError('Package names should not have an extension.')
 
         try:
             import medikit.commands

bonobo/config/processors.py

@@ -133,7 +133,7 @@ class ContextCurrifier:
             try:
                 # todo yield from ? how to ?
                 processor.send(self._stack_values.pop())
-            except StopIteration as exc:
+            except StopIteration:
                 # This is normal, and wanted.
                 pass
             else:

bonobo/config/services.py

@@ -21,32 +21,32 @@ class Service(Option):
     identifier. For example, you can create a Configurable that has a "database" Service in its attribute, meaning that
     you'll define which database to use, by name, when creating the instance of this class, then provide an
     implementation when running the graph using a strategy.
-    
+
     Example::
-    
+
         import bonobo
-    
+
         class QueryExtractor(bonobo.Configurable):
             database = bonobo.Service(default='sqlalchemy.engine.default')
-            
+
         graph = bonobo.Graph(
             QueryExtractor(database='sqlalchemy.engine.secondary'),
             *more_transformations,
         )
-        
+
         if __name__ == '__main__':
             engine = create_engine('... dsn ...')
             bonobo.run(graph, services={
                 'sqlalchemy.engine.secondary': engine
             })
-            
+
     The main goal is not to tie transformations to actual dependencies, so the same can be run in different contexts
     (stages like preprod, prod, or tenants like client1, client2, or anything you want).
 
     .. attribute:: name
 
         Service name will be used to retrieve the implementation at runtime.
-    
+
     """
 
     def __init__(self, name, __doc__=None):
@@ -66,8 +66,13 @@ class Service(Option):
 class Container(dict):
     def __new__(cls, *args, **kwargs):
         if len(args) == 1:
-            assert not len(kwargs), "only one usage at a time, my dear."
+            if len(kwargs):
-            if not (args[0]):
+                raise ValueError(
+                    'You can either use {} with one positional argument or with keyword arguments, not both.'.format(
+                        cls.__name__
+                    )
+                )
+            if not args[0]:
                 return super().__new__(cls)
             if isinstance(args[0], cls):
                 return cls

bonobo/contrib/django/commands.py

@@ -4,11 +4,11 @@ from types import GeneratorType
 from colorama import Back, Fore, Style
 from django.core.management import BaseCommand
 from django.core.management.base import OutputWrapper
+from mondrian import term
 
 import bonobo
 from bonobo.plugins.console import ConsoleOutputPlugin
 from bonobo.util.term import CLEAR_EOL
-from mondrian import term
 
 from .utils import create_or_update
 
@@ -59,8 +59,9 @@ class ETLCommand(BaseCommand):
                 graph_coll = (graph_coll,)
 
             for i, graph in enumerate(graph_coll):
-                assert isinstance(graph, bonobo.Graph), "Invalid graph provided."
+                if not isinstance(graph, bonobo.Graph):
-                print(term.lightwhite("{}. {}".format(i + 1, graph.name)))
+                    raise ValueError('Expected a Graph instance, got {!r}.'.format(graph))
+                print(term.lightwhite('{}. {}'.format(i + 1, graph.name)))
                 result = bonobo.run(graph, services=services, strategy=strategy)
                 results.append(result)
                 print(term.lightblack(" ... return value: " + str(result)))
@@ -75,6 +76,6 @@ class ETLCommand(BaseCommand):
         self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr, ending=CLEAR_EOL + "\n")
         self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + "!" + Style.RESET_ALL + " " + x
 
-        self.run(*args, **kwargs)
+        self.run(*args, **options)
 
         self.stdout, self.stderr = _stdout_backup, _stderr_backup

bonobo/examples/datasets/coffeeshops.py

@@ -8,6 +8,11 @@ from bonobo.structs.graphs import PartialGraph
 
 
 def get_graph(graph=None, *, _limit=(), _print=()):
+    """
+    Extracts a list of cafes with on euro in Paris, renames the name, address and zipcode fields,
+    reorders the fields and formats to json and csv files.
+
+    """
     graph = graph or bonobo.Graph()
 
     producer = (

bonobo/examples/datasets/fablabs.py

@@ -1,5 +1,5 @@
 """
-Extracts a list of fablabs in the world, restrict to the ones in france, then format it both for a nice console output
+Extracts a list of fablabs in the world, restricted to the ones in france, then format its both for a nice console output
 and a flat txt file.
 
 .. graphviz::

bonobo/execution/contexts/base.py

@@ -2,16 +2,17 @@ import logging
 import sys
 from contextlib import contextmanager
 
+from mondrian import term
+
 from bonobo.util import deprecated
 from bonobo.util.objects import Wrapper, get_name
-from mondrian import term
 
 
 @contextmanager
 def recoverable(error_handler):
     try:
         yield
-    except Exception as exc:  # pylint: disable=broad-except
+    except Exception:  # pylint: disable=broad-except
         error_handler(*sys.exc_info(), level=logging.ERROR)
 
 
@@ -19,9 +20,9 @@ def recoverable(error_handler):
 def unrecoverable(error_handler):
     try:
         yield
-    except Exception as exc:  # pylint: disable=broad-except
+    except Exception:  # pylint: disable=broad-except
         error_handler(*sys.exc_info(), level=logging.ERROR)
-        raise  # raise unrecoverableerror from exc ?
+        raise  # raise unrecoverableerror from x ?
 
 
 class Lifecycle:

bonobo/execution/contexts/node.py

@@ -190,7 +190,7 @@ class NodeExecutionContext(BaseContext, WithStatistics):
         if self._stack:
             try:
                 self._stack.teardown()
-            except Exception as exc:
+            except Exception:
                 self.fatal(sys.exc_info())
 
         super().stop()
@@ -207,7 +207,7 @@ class NodeExecutionContext(BaseContext, WithStatistics):
         if self._input_type is not None:
             raise RuntimeError("Cannot override input type, already have %r.", self._input_type)
 
-        if type(input_type) is not type:
+        if not isinstance(input_type, type):
             raise UnrecoverableTypeError("Input types must be regular python types.")
 
         if not issubclass(input_type, tuple):

bonobo/execution/strategies/__init__.py

@@ -28,7 +28,7 @@ DEFAULT_STRATEGY = "threadpool"
 def create_strategy(name=None):
     """
     Create a strategy, or just returns it if it's already one.
-    
+
     :param name: 
     :return: Strategy
     """

bonobo/nodes/basics.py

@@ -3,6 +3,8 @@ import html
 import itertools
 import pprint
 
+from mondrian import term
+
 from bonobo import settings
 from bonobo.config import Configurable, Method, Option, use_context, use_no_input, use_raw_input
 from bonobo.config.functools import transformation_factory
@@ -10,7 +12,6 @@ from bonobo.config.processors import ContextProcessor, use_context_processor
 from bonobo.constants import NOT_MODIFIED
 from bonobo.util.objects import ValueHolder
 from bonobo.util.term import CLEAR_EOL
-from mondrian import term
 
 __all__ = [
     "FixedWindow",
@@ -57,8 +58,6 @@ class Limit(Configurable):
 
 
 def Tee(f):
-    from bonobo.constants import NOT_MODIFIED
-
     @functools.wraps(f)
     def wrapped(*args, **kwargs):
         nonlocal f

bonobo/nodes/filter.py

@@ -11,9 +11,9 @@ class Filter(Configurable):
     .. attribute:: filter
 
         A callable used to filter lines.
-        
+
         If the callable returns a true-ish value, the input will be passed unmodified to the next items.
-        
+
         Otherwise, it'll be burnt.
 
     """

bonobo/plugins/__init__.py

@@ -2,13 +2,13 @@ class Plugin:
     """
     A plugin is an extension to the core behavior of bonobo. If you're writing transformations, you should not need
     to use this interface.
-    
+
     For examples, you can read bonobo.plugins.console.ConsoleOutputPlugin, or bonobo.plugins.jupyter.JupyterOutputPlugin
     that respectively permits an interactive output on an ANSI console and a rich output in a jupyter notebook. Note
     that you most probably won't instanciate them by yourself at runtime, as it's the default behaviour of bonobo to use
     them if your in a compatible context (aka an interactive terminal for the console plugin, or a jupyter notebook for
     the notebook plugin.)
-    
+
     Warning: THE PLUGIN API IS PRE-ALPHA AND WILL EVOLVE BEFORE 1.0, DO NOT RELY ON IT BEING STABLE!
 
     """

bonobo/settings.py

@@ -7,7 +7,7 @@ from bonobo.errors import ValidationError
 def to_bool(s):
     if s is None:
         return False
-    if type(s) is bool:
+    if isinstance(s, bool):
         return s
     if len(s):
         if s.lower() in ("f", "false", "n", "no", "0"):
@@ -31,7 +31,7 @@ class Setting:
     def __init__(self, name, default=None, validator=None, formatter=None):
         self.name = name
 
-        if default:
+        if default is not None:
             self.default = default if callable(default) else lambda: default
         else:
             self.default = lambda: None

bonobo/structs/graphs.py

@@ -36,7 +36,6 @@ class GraphCursor:
                 "Expected something looking like a node, but got an Ellipsis (...). Did you forget to complete the graph?"
             )
 
-
         if len(nodes):
             chain = self.graph.add_chain(*nodes, _input=self.last)
             return GraphCursor(chain.graph, first=self.first, last=chain.output)

bonobo/util/bags.py

@@ -125,7 +125,7 @@ def BagType(typename, fields, *, verbose=False, module=None):
     # message or automatically replace the field name with a valid name.
 
     attrs = tuple(map(_uniquify(_make_valid_attr_name), fields))
-    if type(fields) is str:
+    if isinstance(fields, str):
         raise TypeError("BagType does not support providing fields as a string.")
     fields = list(map(str, fields))
     typename = str(typename)
@@ -133,6 +133,8 @@ def BagType(typename, fields, *, verbose=False, module=None):
     for i, name in enumerate([typename] + fields):
         if type(name) is not str:
             raise TypeError("Type names and field names must be strings, got {name!r}".format(name=name))
+        if not isinstance(name, str):
+            raise TypeError('Type names and field names must be strings, got {name!r}'.format(name=name))
         if not i:
             if not name.isidentifier():
                 raise ValueError("Type names must be valid identifiers: {name!r}".format(name=name))

bonobo/util/objects.py

@@ -24,7 +24,7 @@ class ValueHolder:
     For the sake of concistency, all operator methods have been implemented (see https://docs.python.org/3/reference/datamodel.html) or
     at least all in a certain category, but it feels like a more correct method should exist, like with a getattr-something on the
     value. Let's see later.
-    
+
     """
 
     def __init__(self, value):

bonobo/util/term.py

@@ -1,2 +1,2 @@
 CLEAR_EOL = "\033[0K"
-MOVE_CURSOR_UP = lambda n: "\033[{}A".format(n)
+MOVE_CURSOR_UP = '\033[{}A'.format

docs/_templates/alabaster/support.py

@@ -1,8 +1,9 @@
 # flake8: noqa
 
 from pygments.style import Style
-from pygments.token import Keyword, Name, Comment, String, Error, \
+from pygments.token import (
-     Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
+    Comment, Error, Generic, Keyword, Literal, Name, Number, Operator, Other, Punctuation, String, Whitespace
+)
 
 
 # Originally based on FlaskyStyle which was based on 'tango'.
@@ -12,7 +13,7 @@ class Alabaster(Style):
 
     styles = {
         # No corresponding class for the following:
-        #Text:                     "", # class:  ''
+        # Text:                     "", # class:  ''
         Whitespace: "underline #f8f8f8",  # class: 'w'
         Error: "#a40000 border:#ef2929",  # class: 'err'
         Other: "#000000",  # class 'x'
@@ -28,7 +29,6 @@ class Alabaster(Style):
         Operator: "#582800",  # class: 'o'
         Operator.Word: "bold #004461",  # class: 'ow' - like keywords
         Punctuation: "bold #000000",  # class: 'p'
-
         # because special names such as Name.Class, Name.Function, etc.
         # are not recognized as such later in the parsing, we choose them
         # to look the same as ordinary variables.

docs/conf.py

@@ -5,10 +5,11 @@ import datetime
 import os
 import sys
 
+import bonobo
+
 sys.path.insert(0, os.path.abspath('..'))
 sys.path.insert(0, os.path.abspath('_themes'))
 
-import bonobo
 
 # -- General configuration ------------------------------------------------
 
@@ -63,11 +64,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
 autoclass_content = 'both'
 autodoc_member_order = 'groupwise'
-autodoc_default_flags = [
+autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance']
-    'members',
-    'undoc-members',
-    'show-inheritance',
-]
 
 add_module_names = False
 pygments_style = 'sphinx'
@@ -112,7 +109,7 @@ html_sidebars = {
         'sourcelink.html',
         'searchbox.html',
         'sidebarinfos.html',
-    ]
+    ],
 }
 
 html_theme_path = ['_themes']
@@ -137,15 +134,12 @@ latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -154,9 +148,7 @@ latex_elements = {
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
-latex_documents = [
+latex_documents = [(master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual')]
-    (master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual'),
-]
 
 # -- Options for manual page output ---------------------------------------
 
@@ -171,9 +163,14 @@ man_pages = [(master_doc, 'bonobo', 'Bonobo Documentation', [author], 1)]
 #  dir menu entry, description, category)
 texinfo_documents = [
     (
-        master_doc, 'Bonobo', 'Bonobo Documentation', author, 'Bonobo', 'One line description of project.',
+        master_doc,
-        'Miscellaneous'
+        'Bonobo',
-    ),
+        'Bonobo Documentation',
+        author,
+        'Bonobo',
+        'One line description of project.',
+        'Miscellaneous',
+    )
 ]
 
 # -- Options for Epub output ----------------------------------------------
@@ -209,4 +206,6 @@ rst_epilog = """
 
 .. |longversion| replace:: v.{version}
 
-""".format(version=version, )
+""".format(
+    version=version
+)

docs/reference/examples.rst

@@ -5,7 +5,19 @@ There are a few examples bundled with **bonobo**.
 
 You'll find them under the :mod:`bonobo.examples` package, and you can run them directly as modules:
 
-    $ bonobo run -m bonobo.examples...module
+.. code-block:: shell-session
+
+    $ bonobo run -m bonobo.examples.module
+
+
+or
+
+.. code-block:: shell-session
+
+    $ python -m bonobo.examples.module
+
+
+
 
 .. toctree::
     :maxdepth: 4

docs/reference/examples/tutorials.rst

@@ -1,50 +0,0 @@
-Examples from the tutorial
-==========================
-
-Examples from :doc:`/tutorial/tut01`
-::::::::::::::::::::::::::::::::::::
-
-Example 1
----------
-
-.. automodule:: bonobo.examples.tutorials.tut01e01
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Example 2
----------
-
-.. automodule:: bonobo.examples.tutorials.tut01e02
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Examples from :doc:`/tutorial/tut02`
-::::::::::::::::::::::::::::::::::::
-
-Example 1: Read
----------------
-
-.. automodule:: bonobo.examples.tutorials.tut02e01_read
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Example 2: Write
-----------------
-
-.. automodule:: bonobo.examples.tutorials.tut02e02_write
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Example 3: Write as map
------------------------
-
-.. automodule:: bonobo.examples.tutorials.tut02e03_writeasmap
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-

docs/tutorial/1-init.rst

@@ -112,7 +112,7 @@ Extract
         yield 'hello'
         yield 'world'
 
-This is a first transformation, written as a python generator, that will send some strings, one after the other, to its
+This is a first transformation, written as a `python generator <https://docs.python.org/3/glossary.html#term-generator>`_, that will send some strings, one after the other, to its
 output.
 
 Transformations that take no input and yields a variable number of outputs are usually called **extractors**. You'll

docs/tutorial/3-files.rst

@@ -44,7 +44,7 @@ Now, we need to write a `writer` transformation, and apply this context processo
         f.write(repr(row) + "\n")
 
 The `f` parameter will contain the value yielded by the context processors, in order of appearance. You can chain
-multiple context processors. To find about how to implement this, check the |bonobo| guides in the documentation.
+multiple context processors. To find out about how to implement this, check the |bonobo| guides in the documentation.
 
 Please note that the :func:`bonobo.config.use_context_processor` decorator will modify the function in place, but won't
 modify its behaviour. If you want to call it out of the |bonobo| job context, it's your responsibility to provide
@@ -144,7 +144,7 @@ Reading from files is done using the same logic as writing, except that you'll p
     def get_graph(**options):
         graph = bonobo.Graph()
         graph.add_chain(
-            bonobo.CsvReader('output.csv'),
+            bonobo.CsvReader('input.csv'),
             ...
         )
         return graph

docs/tutorial/4-services.rst

@@ -2,9 +2,8 @@ Part 4: Services
 ================
 
 All external dependencies (like filesystems, network clients, database connections, etc.) should be provided to
-transformations as a service. It allows great flexibility, including the ability to test your transformations isolated
+transformations as a service. This will allow for great flexibility, including the ability to test your transformations isolated
-from the external world, and being friendly to the infrastructure people (and if you're one of them, it's also nice to
+from the external world and easily switch to production (being user-friendly for people in system administration).
-treat yourself well).
 
 In the last section, we used the `fs` service to access filesystems, we'll go even further by switching our `requests`
 call to use the `http` service, so we can switch the `requests` session at runtime. We'll use it to add an http cache,
@@ -24,7 +23,7 @@ Overriding services
 :::::::::::::::::::
 
 You can override the default services, or define your own services, by providing a dictionary to the `services=`
-argument of :obj:`bonobo.run`:
+argument of :obj:`bonobo.run`. First, let's rewrite get_services:
 
 .. code-block:: python
 
@@ -50,8 +49,8 @@ Let's replace the :obj:`requests.get` call we used in the first steps to use the
     def extract_fablabs(http):
         yield from http.get(FABLABS_API_URL).json().get('records')
 
-Tadaa, done! You're not anymore tied to a specific implementation, but to whatever :obj:`requests` compatible object the
+Tadaa, done! You're no more tied to a specific implementation, but to whatever :obj:`requests` -compatible object the
-user want to provide.
+user wants to provide.
 
 Adding cache
 ::::::::::::

docs/tutorial/5-packaging.rst

@@ -1,9 +1,7 @@
 Part 5: Projects and Packaging
 ==============================
 
-Until then, we worked with one file managing a job.
+Throughout this tutorial, we have been working with one file managing a job but real life often involves more complicated setups, with relations and imports between different files.
-
-Real life often involves more complicated setups, with relations and imports between different files.
 
 Data processing is something a wide variety of tools may want to include, and thus |bonobo| does not enforce any
 kind of project structure, as the target structure will be dictated by the hosting project. For example, a `pipelines`
@@ -17,7 +15,7 @@ Imports mechanism
 |bonobo| does not enforce anything on how the python import mechanism work. Especially, it won't add anything to your
 `sys.path`, unlike some popular projects, because we're not sure that's something you want.
 
-If you want to use imports, you should move your script in a python package, and it's up to you to have it setup
+If you want to use imports, you should move your script into a python package, and it's up to you to have it setup
 correctly.
 
 
@@ -36,8 +34,8 @@ your jobs in it. For example, it can be `mypkg.pipelines`.
 Creating a brand new package
 ::::::::::::::::::::::::::::
 
-Because you're maybe starting a project with the data-engineering part, then you may not have a python package yet. As
+Because you may be starting a project involving some data-engineering, you may not have a python package yet. As
-it can be a bit tedious to setup right, there is an helper, using `Medikit <http://medikit.rdc.li/en/latest/>`_, that
+it can be a bit tedious to setup right, there is a helper, using `Medikit <http://medikit.rdc.li/en/latest/>`_, that
 you can use to create a brand new project:
 
 .. code-block:: shell-session
@@ -72,7 +70,7 @@ created in this tutorial and extend it):
 * :doc:`/extension/jupyter`
 * :doc:`/extension/sqlalchemy`
 
-Then, you can either to jump head-first into your code, or you can have a better grasp at all concepts by
+Then, you can either jump head-first into your code, or you can have a better grasp at all concepts by
 :doc:`reading the full bonobo guide </guide/index>`.
 
 You should also `join the slack community <https://bonobo-slack.herokuapp.com/>`_ and ask all your questions there! No

requirements-dev.txt

@@ -18,8 +18,9 @@ imagesize==1.0.0
 jinja2-time==0.2.0
 jinja2==2.10
 markupsafe==1.0
-more-itertools==4.2.0
+more-itertools==4.3.0
 packaging==17.1
+pathlib2==2.3.2
 pluggy==0.7.1
 poyo==0.4.1
 py==1.5.4
@@ -27,7 +28,7 @@ pygments==2.2.0
 pyparsing==2.2.0
 pytest-cov==2.5.1
 pytest-timeout==1.3.1
-pytest==3.6.4
+pytest==3.7.1
 python-dateutil==2.7.3
 pytz==2018.5
 requests==2.19.1
@@ -38,4 +39,3 @@ sphinx==1.7.6
 sphinxcontrib-websupport==1.1.0
 urllib3==1.23
 whichcraft==0.4.1
-yapf==0.22.0

requirements-docker.txt

@@ -7,12 +7,12 @@ chardet==3.0.4
 colorama==0.3.9
 docker-pycreds==0.3.0
 docker==2.7.0
-fs==2.0.26
+fs==2.0.27
 graphviz==0.8.4
 idna==2.7
 jinja2==2.10
 markupsafe==1.0
-mondrian==0.7.0
+mondrian==0.8.0
 packaging==17.1
 pbr==4.2.0
 psutil==5.4.6

requirements-jupyter.txt

@@ -8,7 +8,7 @@ entrypoints==0.2.3
 html5lib==1.0.1
 ipykernel==4.8.2
 ipython-genutils==0.2.0
-ipython==6.4.0
+ipython==6.5.0
 ipywidgets==6.0.1
 jedi==0.12.1
 jinja2==2.10
@@ -26,12 +26,12 @@ pandocfilters==1.4.2
 parso==0.3.1
 pexpect==4.6.0
 pickleshare==0.7.4
-prometheus-client==0.3.0
+prometheus-client==0.3.1
 prompt-toolkit==1.0.15
 ptyprocess==0.6.0
 pygments==2.2.0
 python-dateutil==2.7.3
-pyzmq==17.1.0
+pyzmq==17.1.2
 qtconsole==4.3.1
 send2trash==1.5.0
 simplegeneric==0.8.1

requirements-sqlalchemy.txt

@@ -5,12 +5,12 @@ bonobo-sqlalchemy==0.6.0
 certifi==2018.4.16
 chardet==3.0.4
 colorama==0.3.9
-fs==2.0.26
+fs==2.0.27
 graphviz==0.8.4
 idna==2.7
 jinja2==2.10
 markupsafe==1.0
-mondrian==0.7.0
+mondrian==0.8.0
 packaging==17.1
 pbr==4.2.0
 psutil==5.4.6

requirements.txt

@@ -4,12 +4,12 @@ cached-property==1.4.3
 certifi==2018.4.16
 chardet==3.0.4
 colorama==0.3.9
-fs==2.0.26
+fs==2.0.27
 graphviz==0.8.4
 idna==2.7
 jinja2==2.10
 markupsafe==1.0
-mondrian==0.7.0
+mondrian==0.8.0
 packaging==17.1
 pbr==4.2.0
 psutil==5.4.6

setup.py

@@ -1,11 +1,12 @@
-# Generated by Medikit 0.6.3 on 2018-07-29.
+# Generated by Medikit 0.6.3 on 2018-08-11.
 # All changes will be overriden.
 # Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
 
-from setuptools import setup, find_packages
 from codecs import open
 from os import path
 
+from setuptools import find_packages, setup
+
 here = path.abspath(path.dirname(__file__))
 
 # Py3 compatibility hacks, borrowed from IPython.
@@ -44,14 +45,17 @@ else:
 setup(
     author='Romain Dorgueil',
     author_email='romain@dorgueil.net',
-    data_files=[(
+    data_files=[
-        'share/jupyter/nbextensions/bonobo-jupyter', [
+        (
-            'bonobo/contrib/jupyter/static/extension.js', 'bonobo/contrib/jupyter/static/index.js',
+            'share/jupyter/nbextensions/bonobo-jupyter',
-            'bonobo/contrib/jupyter/static/index.js.map'
+            [
-        ]
+                'bonobo/contrib/jupyter/static/extension.js',
-    )],
+                'bonobo/contrib/jupyter/static/index.js',
-    description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for '
+                'bonobo/contrib/jupyter/static/index.js.map',
-                 'python 3.5+.'),
+            ],
+        )
+    ],
+    description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' 'python 3.5+.'),
     license='Apache License, Version 2.0',
     name='bonobo',
     python_requires='>=3.5',
@@ -61,27 +65,43 @@ setup(
     packages=find_packages(exclude=['ez_setup', 'example', 'test']),
     include_package_data=True,
     install_requires=[
-        'cached-property (~= 1.4)', 'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (~= 2.9)', 'mondrian (~= 0.7)',
+        'cached-property (~= 1.4)',
-        'packaging (~= 17.0)', 'psutil (~= 5.4)', 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)',
+        'fs (~= 2.0)',
-        'stevedore (~= 1.27)', 'whistle (~= 1.0)'
+        'graphviz (>= 0.8, < 0.9)',
+        'jinja2 (~= 2.9)',
+        'mondrian (~= 0.7)',
+        'packaging (~= 17.0)',
+        'psutil (~= 5.4)',
+        'python-slugify (~= 1.2.0)',
+        'requests (~= 2.0)',
+        'stevedore (~= 1.27)',
+        'whistle (~= 1.0)',
     ],
     extras_require={
         'dev': [
-            'cookiecutter (>= 1.5, < 1.6)', 'coverage (~= 4.4)', 'pytest (~= 3.4)', 'pytest-cov (~= 2.5)',
+            'cookiecutter (>= 1.5, < 1.6)',
-            'pytest-timeout (>= 1, < 2)', 'sphinx (~= 1.7)', 'sphinx-sitemap (>= 0.2, < 0.3)', 'yapf'
+            'coverage (~= 4.4)',
+            'pytest (~= 3.4)',
+            'pytest-cov (~= 2.5)',
+            'pytest-timeout (>= 1, < 2)',
+            'sphinx (~= 1.7)',
+            'sphinx-sitemap (>= 0.2, < 0.3)',
         ],
         'docker': ['bonobo-docker (~= 0.6.0a1)'],
         'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'],
-        'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)']
+        'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)'],
     },
     entry_points={
         'bonobo.commands': [
-            'convert = bonobo.commands.convert:ConvertCommand', 'download = bonobo.commands.download:DownloadCommand',
+            'convert = bonobo.commands.convert:ConvertCommand',
-            'examples = bonobo.commands.examples:ExamplesCommand', 'init = bonobo.commands.init:InitCommand',
+            'download = bonobo.commands.download:DownloadCommand',
-            'inspect = bonobo.commands.inspect:InspectCommand', 'run = bonobo.commands.run:RunCommand',
+            'examples = bonobo.commands.examples:ExamplesCommand',
-            'version = bonobo.commands.version:VersionCommand'
+            'init = bonobo.commands.init:InitCommand',
+            'inspect = bonobo.commands.inspect:InspectCommand',
+            'run = bonobo.commands.run:RunCommand',
+            'version = bonobo.commands.version:VersionCommand',
         ],
-        'console_scripts': ['bonobo = bonobo.commands:entrypoint']
+        'console_scripts': ['bonobo = bonobo.commands:entrypoint'],
     },
     url='https://www.bonobo-project.org/',
     download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version),

tests/config/test_services.py

@@ -130,7 +130,21 @@ def test_requires():
     assert svcargs["output"] == vcr.append
 
 
-@pytest.mark.parametrize("services", [None, {}])
+def test_constructor():
+    c1 = Container(foo='foo', bar='bar')
+    assert 2 == len(c1)
+
+    c2 = Container({'foo': 'foo', 'bar': 'bar'})
+    assert 2 == len(c2)
+
+    assert c1['foo'] == c2['foo']
+    assert c1['bar'] == c2['bar']
+
+    with pytest.raises(ValueError):
+        Container({'bar': 'bar'}, foo='foo')
+
+
+@pytest.mark.parametrize('services', [None, {}])
 def test_create_container_empty_values(services):
     c = create_container(services)
     assert len(c) == 2

tests/nodes/io/test_csv.py

@@ -63,9 +63,9 @@ class CsvReaderTest(Csv, ReaderTest, TestCase):
     def test_output_type(self, context):
         context.write_sync(EMPTY)
         context.stop()
-        self.check_output(context, prepend=[("id", "name")])
+        self.check_output(context, prepend=[('id', 'name')])
 
-    @incontext(output_fields=("x", "y"), skip=1)
+    @incontext(output_fields=('x', 'y'), skip=1)
     def test_output_fields(self, context):
         context.write_sync(EMPTY)
         context.stop()
@@ -103,7 +103,7 @@ class CsvWriterTest(Csv, WriterTest, TestCase):
         context.write_sync((L1, L2), (L3, L4))
         context.stop()
 
-        assert self.readlines() == ("a,hey", "b,bee", "c,see", "d,dee")
+        assert self.readlines() == ('a,hey', 'b,bee', 'c,see', 'd,dee')
 
     @incontext()
     def test_nofields_multiple_args_length_mismatch(self, context):

tests/nodes/io/test_json.py

@@ -14,6 +14,7 @@ FOOBAZ = {"foo": "baz"}
 
 incontext = ConfigurableNodeTest.incontext
 
+
 ###
 # Standard JSON Readers / Writers
 ###

tests/test_settings.py

@@ -5,6 +5,7 @@ from unittest.mock import patch
 import pytest
 
 from bonobo import settings
+from bonobo.errors import ValidationError
 
 TEST_SETTING = "TEST_SETTING"
 
@@ -38,6 +39,15 @@ def test_setting():
         s.clear()
         assert s.get() == "hello"
 
+    s = settings.Setting(TEST_SETTING, default=0, validator=lambda x: x == 42)
+    with pytest.raises(ValidationError):
+        assert s.get() is 0
+
+    s.set(42)
+
+    with pytest.raises(ValidationError):
+        s.set(21)
+
 
 def test_default_settings():
     settings.clear_all()