Merge branch 'develop' into feature/io_pickle

Makefile

@@ -1,7 +1,7 @@
 # This file has been auto-generated.
 # All changes will be lost, see Projectfile.
 #
-# Updated at 2017-05-03 18:02:59.359160
+# Updated at 2017-05-22 19:54:27.969596
 
 PACKAGE ?= bonobo
 PYTHON ?= $(shell which python)

README.rst

@@ -51,12 +51,12 @@ so as though it may not yet be complete or fully stable (please, allow us to rea
 
 ----
 
+Homepage: https://www.bonobo-project.org/ (`Roadmap <https://www.bonobo-project.org/roadmap>`_)
+
 Documentation: http://docs.bonobo-project.org/
 
 Issues: https://github.com/python-bonobo/bonobo/issues
 
-Roadmap: https://www.bonobo-project.org/roadmap
-
 Slack: https://bonobo-slack.herokuapp.com/
 
 Release announcements: http://eepurl.com/csHFKL

bonobo/_api.py

@@ -1,5 +1,3 @@
-import warnings
-
 from bonobo.structs import Bag, Graph, Token
 from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
     PrettyPrint, PickleWriter, PickleReader, Tee, count, identity, noop, pprint
@@ -45,7 +43,6 @@ def run(graph, strategy=None, plugins=None, services=None):
     plugins = plugins or []
 
     from bonobo import settings
-
     settings.check()
 
     if not settings.QUIET:  # pragma: no cover
@@ -85,7 +82,9 @@ def open_fs(fs_url, *args, **kwargs):
     :returns: :class:`~fs.base.FS` object
     """
     from fs import open_fs as _open_fs
-    return _open_fs(str(fs_url), *args, **kwargs)
+    from os.path import expanduser
+
+    return _open_fs(expanduser(str(fs_url)), *args, **kwargs)
 
 
 # bonobo.nodes

bonobo/commands/run.py

@@ -20,10 +20,8 @@ def get_default_services(filename, services=None):
             '__name__': '__bonobo__',
             '__file__': services_filename,
         }
-        try:
-            exec(code, context)
-        except Exception:
-            raise
+        exec(code, context)
+
         return {
             **context[DEFAULT_SERVICES_ATTR](),
             **(services or {}),

bonobo/config/__init__.py

@@ -1,13 +1,15 @@
 from bonobo.config.configurables import Configurable
 from bonobo.config.options import Option, Method
 from bonobo.config.processors import ContextProcessor
-from bonobo.config.services import Container, Service
+from bonobo.config.services import Container, Service, Exclusive
 
+# bonobo.config public programming interface
 __all__ = [
     'Configurable',
     'Container',
     'ContextProcessor',
-    'Option',
+    'Exclusive',
     'Method',
+    'Option',
     'Service',
 ]

bonobo/config/configurables.py

@@ -1,6 +1,6 @@
 from bonobo.config.options import Method, Option
 from bonobo.config.processors import ContextProcessor
-from bonobo.errors import ConfigurationError
+from bonobo.errors import ConfigurationError, AbstractError
 
 __all__ = [
     'Configurable',

bonobo/config/processors.py

@@ -1,4 +1,3 @@
-import types
 from collections import Iterable
 from contextlib import contextmanager
 
@@ -132,14 +131,7 @@ def resolve_processors(mixed):
     try:
         yield from mixed.__processors__
     except AttributeError:
-        # old code, deprecated usage
-        if isinstance(mixed, types.FunctionType):
-            yield from getattr(mixed, _CONTEXT_PROCESSORS_ATTR, ())
-
-        for cls in reversed((mixed if isinstance(mixed, type) else type(mixed)).__mro__):
-            yield from cls.__dict__.get(_CONTEXT_PROCESSORS_ATTR, ())
-
-    return ()
+        yield from ()
 
 
 get_context_processors = deprecated_alias('get_context_processors', resolve_processors)

bonobo/config/services.py

@@ -1,7 +1,10 @@
 import re
+import threading
 import types
+from contextlib import ContextDecorator
 
 from bonobo.config.options import Option
+from bonobo.errors import MissingServiceImplementationError
 
 _service_name_re = re.compile(r"^[^\d\W]\w*(:?\.[^\d\W]\w*)*$", re.UNICODE)
 
@@ -78,8 +81,48 @@ class Container(dict):
         if not name in self:
             if default:
                 return default
-            raise KeyError('Cannot resolve service {!r} using provided service collection.'.format(name))
+            raise MissingServiceImplementationError(
+                'Cannot resolve service {!r} using provided service collection.'.format(name)
+            )
         value = super().get(name)
+        # XXX this is not documented and can lead to errors.
         if isinstance(value, types.LambdaType):
             value = value(self)
         return value
+
+
+class Exclusive(ContextDecorator):
+    """
+    Decorator and context manager used to require exclusive usage of an object, most probably a service. It's usefull
+    for example if call order matters on a service implementation (think of an http api that requires a nonce or version
+    parameter ...).
+
+    Usage:
+
+        >>> def handler(some_service):
+        ...     with Exclusive(some_service):
+        ...         some_service.call_1()
+        ...         some_service.call_2()
+        ...         some_service.call_3()
+
+    This will ensure that nobody else is using the same service while in the "with" block, using a lock primitive to
+    ensure that.
+
+    """
+    _locks = {}
+
+    def __init__(self, wrapped):
+        self._wrapped = wrapped
+
+    def get_lock(self):
+        _id = id(self._wrapped)
+        if not _id in Exclusive._locks:
+            Exclusive._locks[_id] = threading.RLock()
+        return Exclusive._locks[_id]
+
+    def __enter__(self):
+        self.get_lock().acquire()
+        return self._wrapped
+
+    def __exit__(self, *exc):
+        self.get_lock().release()

bonobo/errors.py

@@ -55,4 +55,8 @@ class ProhibitedOperationError(RuntimeError):
 
 
 class ConfigurationError(Exception):
-    pass
+    pass
+
+
+class MissingServiceImplementationError(KeyError):
+    pass

bonobo/examples/nodes/_services.py

@@ -0,0 +1,5 @@
+from bonobo import get_examples_path, open_fs
+
+
+def get_services():
+    return {'fs': open_fs(get_examples_path())}

bonobo/examples/types/__main__.py

@@ -0,0 +1,3 @@
+from bonobo.util.python import require
+
+graph = require('strings').graph

bonobo/execution/base.py

@@ -9,6 +9,14 @@ from bonobo.util.errors import print_error
 from bonobo.util.objects import Wrapper, get_name
 
 
+@contextmanager
+def recoverable(error_handler):
+    try:
+        yield
+    except Exception as exc:  # pylint: disable=broad-except
+        error_handler(exc, traceback.format_exc())
+
+
 @contextmanager
 def unrecoverable(error_handler):
     try:
@@ -55,10 +63,9 @@ class LoopingExecutionContext(Wrapper):
             raise RuntimeError('Cannot start a node twice ({}).'.format(get_name(self)))
 
         self._started = True
-        self._stack = ContextCurrifier(self.wrapped, *self._get_initial_context())
 
-        with unrecoverable(self.handle_error):
-            self._stack.setup(self)
+        self._stack = ContextCurrifier(self.wrapped, *self._get_initial_context())
+        self._stack.setup(self)
 
         for enhancer in self._enhancers:
             with unrecoverable(self.handle_error):
@@ -82,7 +89,7 @@ class LoopingExecutionContext(Wrapper):
             return
 
         try:
-            with unrecoverable(self.handle_error):
+            if self._stack:
                 self._stack.teardown()
         finally:
             self._stopped = True

bonobo/execution/plugin.py

@@ -1,6 +1,4 @@
-import traceback
-
-from bonobo.execution.base import LoopingExecutionContext
+from bonobo.execution.base import LoopingExecutionContext, recoverable
 
 
 class PluginExecutionContext(LoopingExecutionContext):
@@ -14,21 +12,14 @@ class PluginExecutionContext(LoopingExecutionContext):
     def start(self):
         super().start()
 
-        try:
+        with recoverable(self.handle_error):
             self.wrapped.initialize()
-        except Exception as exc:  # pylint: disable=broad-except
-            self.handle_error(exc, traceback.format_exc())
 
     def shutdown(self):
-        try:
+        with recoverable(self.handle_error):
             self.wrapped.finalize()
-        except Exception as exc:  # pylint: disable=broad-except
-            self.handle_error(exc, traceback.format_exc())
-        finally:
-            self.alive = False
+        self.alive = False
 
     def step(self):
-        try:
+        with recoverable(self.handle_error):
             self.wrapped.run()
-        except Exception as exc:  # pylint: disable=broad-except
-            self.handle_error(exc, traceback.format_exc())

bonobo/nodes/basics.py

@@ -1,6 +1,7 @@
 import functools
 from pprint import pprint as _pprint
 
+import itertools
 from colorama import Fore, Style
 
 from bonobo.config import Configurable, Option
@@ -69,6 +70,12 @@ def _count_counter(self, context):
     context.send(Bag(counter._value))
 
 
+class PrettyPrinter(Configurable):
+    def call(self, *args, **kwargs):
+        for i, (item, value) in enumerate(itertools.chain(enumerate(args), kwargs.items())):
+            print(' ' if i else '•', item, '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL)
+
+
 pprint = Tee(_pprint)
 
 

bonobo/nodes/io/csv.py

@@ -3,6 +3,8 @@ import csv
 from bonobo.config import Option
 from bonobo.config.processors import ContextProcessor
 from bonobo.constants import NOT_MODIFIED
+from bonobo.errors import ConfigurationError, ValidationError
+from bonobo.structs import Bag
 from bonobo.util.objects import ValueHolder
 from .file import FileHandler, FileReader, FileWriter
 
@@ -28,6 +30,14 @@ class CsvHandler(FileHandler):
     headers = Option(tuple)
 
 
+def validate_csv_output_format(v):
+    if callable(v):
+        return v
+    if v in {'dict', 'kwargs'}:
+        return v
+    raise ValidationError('Unsupported format {!r}.'.format(v))
+
+
 class CsvReader(CsvHandler, FileReader):
     """
     Reads a CSV and yield the values as dicts.
@@ -39,13 +49,23 @@ class CsvReader(CsvHandler, FileReader):
     """
 
     skip = Option(int, default=0)
+    output_format = Option(validate_csv_output_format, default='dict')
 
     @ContextProcessor
     def csv_headers(self, context, fs, file):
         yield ValueHolder(self.headers)
 
+    def get_output_formater(self):
+        if callable(self.output_format):
+            return self.output_format
+        elif isinstance(self.output_format, str):
+            return getattr(self, '_format_as_' + self.output_format)
+        else:
+            raise ConfigurationError('Unsupported format {!r} for {}.'.format(self.output_format, type(self).__name__))
+
     def read(self, fs, file, headers):
         reader = csv.reader(file, delimiter=self.delimiter, quotechar=self.quotechar)
+        formater = self.get_output_formater()
 
         if not headers.get():
             headers.set(next(reader))
@@ -60,7 +80,13 @@ class CsvReader(CsvHandler, FileReader):
             if len(row) != field_count:
                 raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count, ))
 
-            yield dict(zip(headers.value, row))
+            yield formater(headers.get(), row)
+
+    def _format_as_dict(self, headers, values):
+        return dict(zip(headers, values))
+
+    def _format_as_kwargs(self, headers, values):
+        return Bag(**dict(zip(headers, values)))
 
 
 class CsvWriter(CsvHandler, FileWriter):

bonobo/nodes/io/file.py

@@ -4,11 +4,6 @@ from bonobo.config.processors import ContextProcessor
 from bonobo.constants import NOT_MODIFIED
 from bonobo.util.objects import ValueHolder
 
-__all__ = [
-    'FileReader',
-    'FileWriter',
-]
-
 
 class FileHandler(Configurable):
     """Abstract component factory for file-related components.
@@ -23,6 +18,7 @@ class FileHandler(Configurable):
     path = Option(str, required=True, positional=True)  # type: str
     eol = Option(str, default='\n')  # type: str
     mode = Option(str)  # type: str
+    encoding = Option(str, default='utf-8')  # type: str
 
     fs = Service('fs')  # type: str
 
@@ -32,7 +28,7 @@ class FileHandler(Configurable):
             yield file
 
     def open(self, fs):
-        return fs.open(self.path, self.mode)
+        return fs.open(self.path, self.mode, encoding=self.encoding)
 
 
 class Reader(FileHandler):

bonobo/nodes/io/json.py

@@ -3,10 +3,6 @@ import json
 from bonobo.config.processors import ContextProcessor
 from .file import FileWriter, FileReader
 
-__all__ = [
-    'JsonWriter',
-]
-
 
 class JsonHandler():
     eol = ',\n'

bonobo/strategies/executor.py

@@ -36,7 +36,7 @@ class ExecutorStrategy(Strategy):
                     plugin_context.loop()
                     plugin_context.stop()
                 except Exception as exc:
-                    print_error(exc, traceback.format_exc(), prefix='Error in plugin context', context=plugin_context)
+                    print_error(exc, traceback.format_exc(), context=plugin_context)
 
             futures.append(executor.submit(_runner))
 
@@ -46,9 +46,7 @@ class ExecutorStrategy(Strategy):
                 try:
                     node_context.start()
                 except Exception as exc:
-                    print_error(
-                        exc, traceback.format_exc(), prefix='Could not start node context', context=node_context
-                    )
+                    print_error(exc, traceback.format_exc(), context=node_context, method='start')
                     node_context.input.on_end()
                 else:
                     node_context.loop()
@@ -56,7 +54,7 @@ class ExecutorStrategy(Strategy):
                 try:
                     node_context.stop()
                 except Exception as exc:
-                    print_error(exc, traceback.format_exc(), prefix='Could not stop node context', context=node_context)
+                    print_error(exc, traceback.format_exc(), context=node_context, method='stop')
 
             futures.append(executor.submit(_runner))
 

bonobo/structs/bags.py

@@ -75,6 +75,14 @@ class Bag:
 
         raise TypeError('Could not apply bag to {}.'.format(func_or_iter))
 
+    def get(self):
+        """
+        Get a 2 element tuple of this bag's args and kwargs.
+
+        :return: tuple
+        """
+        return self.args, self.kwargs
+
     def extend(self, *args, **kwargs):
         return type(self)(*args, _parent=self, **kwargs)
 

bonobo/util/errors.py

@@ -1,5 +1,7 @@
 import sys
+from textwrap import indent
 
+from bonobo import settings
 from bonobo.structs.bags import ErrorBag
 
 
@@ -7,7 +9,14 @@ def is_error(bag):
     return isinstance(bag, ErrorBag)
 
 
-def print_error(exc, trace, context=None, prefix=''):
+def _get_error_message(exc):
+    if hasattr(exc, '__str__'):
+        message = str(exc)
+        return message[0].upper() + message[1:]
+    return '\n'.join(exc.args),
+
+
+def print_error(exc, trace, context=None, method=None):
     """
     Error handler. Whatever happens in a plugin or component, if it looks like an exception, taste like an exception
     or somehow make me think it is an exception, I'll handle it.
@@ -18,14 +27,20 @@ def print_error(exc, trace, context=None, prefix=''):
     """
 
     from colorama import Fore, Style
+
+    prefix = '{}{} | {}'.format(Fore.RED, Style.BRIGHT, Style.RESET_ALL)
+
     print(
         Style.BRIGHT,
         Fore.RED,
-        '\U0001F4A3 {}{}{}'.format(
-            (prefix + ': ') if prefix else '', type(exc).__name__, ' in {!r}'.format(context) if context else ''
-        ),
+        type(exc).__name__,
+        ' (in {}{})'.format(type(context).__name__, '.{}()'.format(method) if method else '') if context else '',
+        Style.RESET_ALL,
+        '\n',
+        indent(_get_error_message(exc), prefix + Style.BRIGHT),
         Style.RESET_ALL,
         sep='',
         file=sys.stderr,
     )
-    print(trace)
+    print(prefix, file=sys.stderr)
+    print(indent(trace, prefix, predicate=lambda line: True), file=sys.stderr)

bonobo/util/testing.py

@@ -1,3 +1,4 @@
+from contextlib import contextmanager
 from unittest.mock import MagicMock
 
 from bonobo.execution.node import NodeExecutionContext
@@ -7,3 +8,12 @@ class CapturingNodeExecutionContext(NodeExecutionContext):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.send = MagicMock()
+
+
+@contextmanager
+def optional_contextmanager(cm, *, ignore=False):
+    if cm is None or ignore:
+        yield
+    else:
+        with cm:
+            yield

docs/guide/services.rst

@@ -81,6 +81,26 @@ A dictionary, or dictionary-like, "services" named argument can be passed to the
 provided is pretty basic, and feature-less. But you can use much more evolved libraries instead of the provided
 stub, and as long as it works the same (a.k.a implements a dictionary-like interface), the system will use it.
 
+Solving concurrency problems
+----------------------------
+
+If a service cannot be used by more than one thread at a time, either because it's just not threadsafe, or because
+it requires to carefully order the calls made (apis that includes nonces, or work on results returned by previous
+calls are usually good candidates), you can use the :class:`bonobo.config.Exclusive` context processor to lock the
+use of a dependency for a time period.
+
+.. code-block:: python
+
+    from bonobo.config import Exclusive
+
+    def t1(api):
+        with Exclusive(api):
+            api.first_call()
+            api.second_call()
+            # ... etc
+            api.last_call()
+
+
 Service configuration (to be decided and implemented)
 :::::::::::::::::::::::::::::::::::::::::::::::::::::
 

docs/roadmap.rst

@@ -1,12 +1,12 @@
-Detailed roadmap
-================
+Internal roadmap notes
+======================
 
-initialize / finalize better than start / stop ?
+Things that should be thought about and/or implemented, but that I don't know where to store.
 
 Graph and node level plugins
 ::::::::::::::::::::::::::::
 
- * Enhancers or nide-level plugins
+ * Enhancers or node-level plugins
  * Graph level plugins
  * Documentation
 
@@ -15,21 +15,19 @@ Command line interface and environment
 
 * How do we manage environment ? .env ?
 * How do we configure plugins ?
-* Console run should allow console plugin as a command line argument (or silence it).
 
 Services and Processors
 :::::::::::::::::::::::
 
-* ContextProcessors not clean
+* ContextProcessors not clean (a bit better, but still not in love with the api)
 
 Next...
 :::::::
 
 * Release process specialised for bonobo. With changelog production, etc.
 * Document how to upgrade version, like, minor need change badges, etc.
-* PyPI page looks like crap: https://pypi.python.org/pypi/bonobo/0.2.1
-* Windows break because of readme encoding. Fix in edgy.
-* bonobo init --with sqlalchemy,docker
+* Windows console looks crappy.
+* bonobo init --with sqlalchemy,docker; cookiecutter?
 * logger, vebosity level
 
 
@@ -39,22 +37,15 @@ External libs that looks good
 * dask.distributed
 * mediator (event dispatcher)
 
-Version 0.3
+Version 0.4
 :::::::::::
 
-* Services !
 * SQLAlchemy 101
 
-Version 0.2
-:::::::::::
+Design decisions
+::::::::::::::::
 
-* Autodetect if within jupyter notebook context, and apply plugin if it's the case.
-* New bonobo.structs package with simple data structures (bags, graphs, tokens).
-
-Plugins API
-:::::::::::
-
-* Stabilize, find other things to do.
+* initialize / finalize better than start / stop ?
 
 Minor stuff
 :::::::::::

setup.py

@@ -18,13 +18,19 @@ except NameError:
 
 
 # Get the long description from the README file
-with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
-    long_description = f.read()
+try:
+    with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
+        long_description = f.read()
+except:
+    long_description = ''
 
 # Get the classifiers from the classifiers file
 tolines = lambda c: list(filter(None, map(lambda s: s.strip(), c.split('\n'))))
-with open(path.join(here, 'classifiers.txt'), encoding='utf-8') as f:
-    classifiers = tolines(f.read())
+try:
+    with open(path.join(here, 'classifiers.txt'), encoding='utf-8') as f:
+        classifiers = tolines(f.read())
+except:
+    classifiers = []
 
 version_ns = {}
 try:

tests/config/test_configurables.py

@@ -2,7 +2,6 @@ import pytest
 
 from bonobo.config.configurables import Configurable
 from bonobo.config.options import Option
-from bonobo.config.services import Container, Service, validate_service_name
 
 
 class MyConfigurable(Configurable):
@@ -25,28 +24,6 @@ class MyConfigurableUsingPositionalOptions(MyConfigurable):
     third = Option(str, required=False, positional=True)
 
 
-class PrinterInterface():
-    def print(self, *args):
-        raise NotImplementedError()
-
-
-class ConcretePrinter(PrinterInterface):
-    def __init__(self, prefix):
-        self.prefix = prefix
-
-    def print(self, *args):
-        return ';'.join((self.prefix, *args))
-
-
-class MyServiceDependantConfigurable(Configurable):
-    printer = Service(
-        PrinterInterface,
-    )
-
-    def __call__(self, printer: PrinterInterface, *args):
-        return printer.print(*args)
-
-
 def test_missing_required_option_error():
     with pytest.raises(TypeError) as exc:
         MyConfigurable()
@@ -107,39 +84,5 @@ def test_option_resolution_order():
     assert o.integer == None
 
 
-def test_service_name_validator():
-    assert validate_service_name('foo') == 'foo'
-    assert validate_service_name('foo.bar') == 'foo.bar'
-    assert validate_service_name('Foo') == 'Foo'
-    assert validate_service_name('Foo.Bar') == 'Foo.Bar'
-    assert validate_service_name('Foo.a0') == 'Foo.a0'
-
-    with pytest.raises(ValueError):
-        validate_service_name('foo.0')
-
-    with pytest.raises(ValueError):
-        validate_service_name('0.foo')
-
-
-SERVICES = Container(
-    printer0=ConcretePrinter(prefix='0'),
-    printer1=ConcretePrinter(prefix='1'),
-)
-
-
-def test_service_dependency():
-    o = MyServiceDependantConfigurable(printer='printer0')
-
-    assert o(SERVICES.get('printer0'), 'foo', 'bar') == '0;foo;bar'
-    assert o(SERVICES.get('printer1'), 'bar', 'baz') == '1;bar;baz'
-    assert o(*SERVICES.args_for(o), 'foo', 'bar') == '0;foo;bar'
-
-
-def test_service_dependency_unavailable():
-    o = MyServiceDependantConfigurable(printer='printer2')
-    with pytest.raises(KeyError):
-        SERVICES.args_for(o)
-
-
 def test_option_positional():
     o = MyConfigurableUsingPositionalOptions('1', '2', '3', required_str='hello')

tests/config/test_methods.py

@@ -28,8 +28,6 @@ def test_define_with_decorator():
     def Concrete(self, *args, **kwargs):
         calls.append((args, kwargs, ))
 
-    print('handler', Concrete.handler)
-
     assert callable(Concrete.handler)
     t = Concrete('foo', bar='baz')
 

tests/config/test_services.py

@@ -0,0 +1,96 @@
+import threading
+import time
+
+import pytest
+
+from bonobo.config import Configurable, Container, Exclusive, Service
+from bonobo.config.services import validate_service_name
+
+
+class PrinterInterface():
+    def print(self, *args):
+        raise NotImplementedError()
+
+
+class ConcretePrinter(PrinterInterface):
+    def __init__(self, prefix):
+        self.prefix = prefix
+
+    def print(self, *args):
+        return ';'.join((self.prefix, *args))
+
+
+SERVICES = Container(
+    printer0=ConcretePrinter(prefix='0'),
+    printer1=ConcretePrinter(prefix='1'),
+)
+
+
+class MyServiceDependantConfigurable(Configurable):
+    printer = Service(
+        PrinterInterface,
+    )
+
+    def __call__(self, printer: PrinterInterface, *args):
+        return printer.print(*args)
+
+
+def test_service_name_validator():
+    assert validate_service_name('foo') == 'foo'
+    assert validate_service_name('foo.bar') == 'foo.bar'
+    assert validate_service_name('Foo') == 'Foo'
+    assert validate_service_name('Foo.Bar') == 'Foo.Bar'
+    assert validate_service_name('Foo.a0') == 'Foo.a0'
+
+    with pytest.raises(ValueError):
+        validate_service_name('foo.0')
+
+    with pytest.raises(ValueError):
+        validate_service_name('0.foo')
+
+
+def test_service_dependency():
+    o = MyServiceDependantConfigurable(printer='printer0')
+
+    assert o(SERVICES.get('printer0'), 'foo', 'bar') == '0;foo;bar'
+    assert o(SERVICES.get('printer1'), 'bar', 'baz') == '1;bar;baz'
+    assert o(*SERVICES.args_for(o), 'foo', 'bar') == '0;foo;bar'
+
+
+def test_service_dependency_unavailable():
+    o = MyServiceDependantConfigurable(printer='printer2')
+    with pytest.raises(KeyError):
+        SERVICES.args_for(o)
+
+
+class VCR:
+    def __init__(self):
+        self.tape = []
+
+    def append(self, x):
+        return self.tape.append(x)
+
+
+def test_exclusive():
+    vcr = VCR()
+    vcr.append('hello')
+
+    def record(prefix, vcr=vcr):
+        with Exclusive(vcr):
+            for i in range(5):
+                vcr.append(' '.join((prefix, str(i))))
+                time.sleep(0.05)
+
+    threads = [threading.Thread(target=record, args=(str(i), )) for i in range(5)]
+
+    for thread in threads:
+        thread.start()
+        time.sleep(0.01)  # this is not good practice, how to test this without sleeping ?? XXX
+
+    for thread in threads:
+        thread.join()
+
+    assert vcr.tape == [
+        'hello', '0 0', '0 1', '0 2', '0 3', '0 4', '1 0', '1 1', '1 2', '1 3', '1 4', '2 0', '2 1', '2 2', '2 3',
+        '2 4', '3 0', '3 1', '3 2', '3 3', '3 4', '4 0', '4 1', '4 2', '4 3', '4 4'
+    ]

tests/io/test_csv.py

@@ -55,3 +55,38 @@ def test_read_csv_from_file(tmpdir):
         'b': 'b bar',
         'c': 'c bar',
     }
+
+
+def test_read_csv_kwargs_output_formater(tmpdir):
+    fs, filename = open_fs(tmpdir), 'input.csv'
+    fs.open(filename, 'w').write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar')
+
+    reader = CsvReader(path=filename, delimiter=',', output_format='kwargs')
+
+    context = CapturingNodeExecutionContext(reader, services={'fs': fs})
+
+    context.start()
+    context.write(BEGIN, Bag(), END)
+    context.step()
+    context.stop()
+
+    assert len(context.send.mock_calls) == 2
+
+    args0, kwargs0 = context.send.call_args_list[0]
+    assert len(args0) == 1 and not len(kwargs0)
+    args1, kwargs1 = context.send.call_args_list[1]
+    assert len(args1) == 1 and not len(kwargs1)
+
+    _args, _kwargs = args0[0].get()
+    assert not len(_args) and _kwargs == {
+        'a': 'a foo',
+        'b': 'b foo',
+        'c': 'c foo',
+    }
+
+    _args, _kwargs = args1[0].get()
+    assert not len(_args) and _kwargs == {
+        'a': 'a bar',
+        'b': 'b bar',
+        'c': 'c bar',
+    }

tests/test_commands.py

@@ -1,10 +1,26 @@
+import runpy
+import sys
+from unittest.mock import patch
+
 import pkg_resources
 import pytest
 
-from bonobo import get_examples_path
+from bonobo import __main__, __version__, get_examples_path
 from bonobo.commands import entrypoint
 
 
+def runner_entrypoint(*args):
+    return entrypoint(list(args))
+
+
+def runner_module(*args):
+    with patch.object(sys, 'argv', ['bonobo', *args]):
+        return runpy.run_path(__main__.__file__, run_name='__main__')
+
+
+all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module])
+
+
 def test_entrypoint():
     commands = {}
 
@@ -13,23 +29,51 @@ def test_entrypoint():
 
     assert 'init' in commands
     assert 'run' in commands
+    assert 'version' in commands
 
 
-def test_no_command(capsys):
+@all_runners
+def test_no_command(runner, capsys):
     with pytest.raises(SystemExit):
-        entrypoint([])
+        runner()
     _, err = capsys.readouterr()
     assert 'error: the following arguments are required: command' in err
 
 
-def test_init():
-    pass  # need ext dir
-
-
-def test_run(capsys):
-    entrypoint(['run', '--quiet', get_examples_path('types/strings.py')])
+@all_runners
+def test_run(runner, capsys):
+    runner('run', '--quiet', get_examples_path('types/strings.py'))
     out, err = capsys.readouterr()
     out = out.split('\n')
     assert out[0].startswith('Foo ')
     assert out[1].startswith('Bar ')
     assert out[2].startswith('Baz ')
+
+
+@all_runners
+def test_run_module(runner, capsys):
+    runner('run', '--quiet', '-m', 'bonobo.examples.types.strings')
+    out, err = capsys.readouterr()
+    out = out.split('\n')
+    assert out[0].startswith('Foo ')
+    assert out[1].startswith('Bar ')
+    assert out[2].startswith('Baz ')
+
+
+@all_runners
+def test_run_path(runner, capsys):
+    runner('run', '--quiet', get_examples_path('types'))
+    out, err = capsys.readouterr()
+    out = out.split('\n')
+    assert out[0].startswith('Foo ')
+    assert out[1].startswith('Bar ')
+    assert out[2].startswith('Baz ')
+
+
+@all_runners
+def test_version(runner, capsys):
+    runner('version')
+    out, err = capsys.readouterr()
+    out = out.strip()
+    assert out.startswith('bonobo ')
+    assert out.endswith(__version__)

tests/test_publicapi.py

@@ -1,4 +1,4 @@
-import types
+import inspect
 
 
 def test_wildcard_import():
@@ -10,7 +10,7 @@ def test_wildcard_import():
         if name.startswith('_'):
             continue
         attr = getattr(bonobo, name)
-        if isinstance(attr, types.ModuleType):
+        if inspect.ismodule(attr):
             continue
 
         assert name in bonobo.__all__

tests/util/test_objects.py

@@ -1,4 +1,9 @@
+import operator
+
+import pytest
+
 from bonobo.util.objects import Wrapper, get_name, ValueHolder
+from bonobo.util.testing import optional_contextmanager
 
 
 class foo:
@@ -52,3 +57,56 @@ def test_valueholder():
     assert y == x
     assert y is not x
     assert repr(x) == repr(y) == repr(43)
+
+
+unsupported_operations = {
+    int: {operator.matmul},
+    str: {
+        operator.sub, operator.mul, operator.matmul, operator.floordiv, operator.truediv, operator.mod, divmod,
+        operator.pow, operator.lshift, operator.rshift, operator.and_, operator.xor, operator.or_
+    },
+}
+
+
+@pytest.mark.parametrize('x,y', [(5, 3), (0, 10), (0, 0), (1, 1), ('foo', 'bar'), ('', 'baz!')])
+@pytest.mark.parametrize(
+    'operation,inplace_operation', [
+        (operator.add, operator.iadd),
+        (operator.sub, operator.isub),
+        (operator.mul, operator.imul),
+        (operator.matmul, operator.imatmul),
+        (operator.truediv, operator.itruediv),
+        (operator.floordiv, operator.ifloordiv),
+        (operator.mod, operator.imod),
+        (divmod, None),
+        (operator.pow, operator.ipow),
+        (operator.lshift, operator.ilshift),
+        (operator.rshift, operator.irshift),
+        (operator.and_, operator.iand),
+        (operator.xor, operator.ixor),
+        (operator.or_, operator.ior),
+    ]
+)
+def test_valueholder_integer_operations(x, y, operation, inplace_operation):
+    v = ValueHolder(x)
+
+    is_supported = operation not in unsupported_operations.get(type(x), set())
+
+    isdiv = ('div' in operation.__name__) or ('mod' in operation.__name__)
+
+    # forward...
+    with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported):
+        with optional_contextmanager(pytest.raises(ZeroDivisionError), ignore=y or not isdiv):
+            assert operation(x, y) == operation(v, y)
+
+    # backward...
+    with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported):
+        with optional_contextmanager(pytest.raises(ZeroDivisionError), ignore=x or not isdiv):
+            assert operation(y, x) == operation(y, v)
+
+    # in place...
+    if inplace_operation is not None:
+        with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported):
+            with optional_contextmanager(pytest.raises(ZeroDivisionError), ignore=y or not isdiv):
+                inplace_operation(v, y)
+                assert v == operation(x, y)