paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into transformation_factory

Browse Source
This commit is contained in:
Romain Dorgueil
2017-07-15 14:38:09 +02:00
parent 3059f112f1 9ab14f9c94
commit f02d92c7e8
97 changed files with 4278 additions and 1098 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@ -2,6 +2,7 @@
*,cover
*.egg
*.egg-info/
*.iml
*.log
*.manifest
*.mo
@ -20,25 +21,17 @@
.installed.cfg
.ipynb_checkpoints
.python-version
.tox/
.webassets-cache
/.idea
/.release
/bonobo.iml
/bonobo/examples/work_in_progress/
/bonobo/ext/jupyter/js/node_modules/
/build/
/coverage.xml
/develop-eggs/
/dist/
/docs/_build/
/downloads/
/eggs/
/examples/private
/htmlcov/
/sdist/
/tags
celerybeat-schedule
parts/
pip-delete-this-directory.txt
pip-log.txt

4
.travis.yml
View File

@ -9,8 +9,6 @@ install:
- make install-dev
- pip install coveralls
script:
- make clean docs test
- pip install pycountry
- bin/run_all_examples.sh
- make clean test
after_success:
- coveralls

11
Makefile
View File

@ -1,7 +1,7 @@
# This file has been auto-generated.
# All changes will be lost, see Projectfile.
#
# Updated at 2017-05-03 18:02:59.359160
# Updated at 2017-07-04 10:50:55.775681
PACKAGE ?= bonobo
PYTHON ?= $(shell which python)
@ -18,10 +18,11 @@ SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build
SPHINX_OPTIONS ?=
SPHINX_SOURCEDIR ?= docs
SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
YAPF ?= $(PYTHON_DIRNAME)/yapf
YAPF ?= $(PYTHON) -m yapf
YAPF_OPTIONS ?= -rip
VERSION ?= $(shell git describe 2>/dev/null || echo dev)
.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev lint test
.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev test
# Installs the local project dependencies.
install:
@ -39,9 +40,6 @@ install-dev:
clean:
rm -rf build dist *.egg-info
lint: install-dev
$(PYTHON_DIRNAME)/pylint --py3k $(PACKAGE) -f html > pylint.html
test: install-dev
$(PYTEST) $(PYTEST_OPTIONS) tests
@ -50,3 +48,4 @@ $(SPHINX_SOURCEDIR): install-dev
format: install-dev
$(YAPF) $(YAPF_OPTIONS) .
$(YAPF) $(YAPF_OPTIONS) Projectfile

103
Projectfile
View File

@ -1,73 +1,58 @@
# bonobo (see github.com/python-edgy/project)
name = 'bonobo'
description = 'Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.'
license = 'Apache License, Version 2.0'
from edgy.project import require
url = 'https://www.bonobo-project.org/'
download_url = 'https://github.com/python-bonobo/bonobo/tarball/{version}'
pytest = require('pytest')
python = require('python')
sphinx = require('sphinx')
yapf = require('yapf')
author = 'Romain Dorgueil'
author_email = 'romain@dorgueil.net'
python.setup(
name='bonobo',
description='Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.',
license='Apache License, Version 2.0',
url='https://www.bonobo-project.org/',
download_url='https://github.com/python-bonobo/bonobo/tarball/{version}',
author='Romain Dorgueil',
author_email='romain@dorgueil.net',
data_files=[
(
'share/jupyter/nbextensions/bonobo-jupyter', [
'bonobo/ext/jupyter/static/extension.js',
'bonobo/ext/jupyter/static/index.js',
'bonobo/ext/jupyter/static/index.js.map',
]
),
],
entry_points={
'console_scripts': [
'bonobo = bonobo.commands:entrypoint',
],
'bonobo.commands': [
'init = bonobo.commands.init:register',
'run = bonobo.commands.run:register',
'version = bonobo.commands.version:register',
],
}
)
enable_features = {
'make',
'sphinx',
'pytest',
'git',
'pylint',
'python',
'yapf',
}
# stricts deendencies in requirements.txt
install_requires = [
python.add_requirements(
'colorama >=0.3,<1.0',
'fs >=2.0,<3.0',
'packaging >=16,<17',
'psutil >=5.2,<6.0',
'requests >=2.0,<3.0',
'stevedore >=1.21,<2.0',
]
extras_require = {
'jupyter': [
'jupyter >=1.0,<1.1',
'ipywidgets >=6.0.0.beta5'
],
'dev': [
'coverage >=4,<5',
'pylint >=1,<2',
'pytest >=3,<4',
'pytest-cov >=2,<3',
dev=[
'cookiecutter >=1.5,<1.6',
'pytest-sugar >=0.8,<0.9',
'pytest-timeout >=1,<2',
'sphinx',
'sphinx_rtd_theme',
'yapf',
],
}
data_files = [
('share/jupyter/nbextensions/bonobo-jupyter', [
'bonobo/ext/jupyter/static/extension.js',
'bonobo/ext/jupyter/static/index.js',
'bonobo/ext/jupyter/static/index.js.map',
]),
]
entry_points = {
'console_scripts': [
'bonobo = bonobo.commands:entrypoint',
docker=[
'bonobo-docker',
],
'bonobo.commands': [
'init = bonobo.commands.init:register',
'run = bonobo.commands.run:register',
'version = bonobo.commands.version:register',
],
'edgy.project.features': [
'bonobo = bonobo.ext.edgy.project.feature:BonoboFeature'
jupyter=[
'jupyter >=1.0,<1.1',
'ipywidgets >=6.0.0,<7',
]
}
@listen('edgy.project.feature.make.on_generate', priority=10)
def on_make_generate_docker_targets(event):
event.makefile['SPHINX_SOURCEDIR'] = 'docs'
)

17
README.rst
View File

@ -7,28 +7,29 @@ Data-processing for humans.
.. image:: https://img.shields.io/pypi/v/bonobo.svg
:target: https://pypi.python.org/pypi/bonobo
:alt: PyPI
.. image:: https://img.shields.io/pypi/pyversions/bonobo.svg
:target: https://pypi.python.org/pypi/bonobo
:alt: Versions
.. image:: https://readthedocs.org/projects/bonobo/badge/?version=0.3
.. image:: https://readthedocs.org/projects/bonobo/badge/?version=latest
:target: http://docs.bonobo-project.org/
:alt: Documentation
.. image:: https://travis-ci.org/python-bonobo/bonobo.svg?branch=0.3
.. image:: https://travis-ci.org/python-bonobo/bonobo.svg?branch=master
:target: https://travis-ci.org/python-bonobo/bonobo
:alt: Continuous Integration (Linux)
.. image:: https://ci.appveyor.com/api/projects/status/github/python-bonobo/bonobo?retina=true&branch=0.3&svg=true
:target: https://ci.appveyor.com/project/hartym/bonobo?branch=0.3
.. image:: https://ci.appveyor.com/api/projects/status/github/python-bonobo/bonobo?retina=true&branch=master&svg=true
:target: https://ci.appveyor.com/project/hartym/bonobo?branch=master
:alt: Continuous Integration (Windows)
.. image:: https://codeclimate.com/github/python-bonobo/bonobo/badges/gpa.svg
:target: https://codeclimate.com/github/python-bonobo/bonobo
:alt: Code Climate
.. image:: https://img.shields.io/coveralls/python-bonobo/bonobo/0.3.svg
:target: https://coveralls.io/github/python-bonobo/bonobo?branch=0.3
.. image:: https://img.shields.io/coveralls/python-bonobo/bonobo/master.svg
:target: https://coveralls.io/github/python-bonobo/bonobo?branch=master
:alt: Coverage
Bonobo is an extract-transform-load framework for python 3.5+ (see comparisons with other data tools).
@ -50,12 +51,12 @@ so as though it may not yet be complete or fully stable (please, allow us to rea
----
Homepage: https://www.bonobo-project.org/ (`Roadmap <https://www.bonobo-project.org/roadmap>`_)
Documentation: http://docs.bonobo-project.org/
Issues: https://github.com/python-bonobo/bonobo/issues
Roadmap: https://www.bonobo-project.org/roadmap
Slack: https://bonobo-slack.herokuapp.com/
Release announcements: http://eepurl.com/csHFKL

38
bonobo/_api.py
View File

@ -1,8 +1,6 @@
import warnings
from bonobo.structs import Bag, Graph, Token
from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
PrettyPrint, Tee, count, identity, noop, pprint
PrettyPrinter, PickleWriter, PickleReader, Tee, count, identity, noop, pprint
from bonobo.strategies import create_strategy
from bonobo.util.objects import get_name
@ -44,15 +42,19 @@ def run(graph, strategy=None, plugins=None, services=None):
plugins = plugins or []
if _is_interactive_console(): # pragma: no cover
from bonobo.ext.console import ConsoleOutputPlugin
if ConsoleOutputPlugin not in plugins:
plugins.append(ConsoleOutputPlugin)
from bonobo import settings
settings.check()
if _is_jupyter_notebook(): # pragma: no cover
from bonobo.ext.jupyter import JupyterOutputPlugin
if JupyterOutputPlugin not in plugins:
plugins.append(JupyterOutputPlugin)
if not settings.QUIET: # pragma: no cover
if _is_interactive_console():
from bonobo.ext.console import ConsoleOutputPlugin
if ConsoleOutputPlugin not in plugins:
plugins.append(ConsoleOutputPlugin)
if _is_jupyter_notebook():
from bonobo.ext.jupyter import JupyterOutputPlugin
if JupyterOutputPlugin not in plugins:
plugins.append(JupyterOutputPlugin)
return strategy.execute(graph, plugins=plugins, services=services)
@ -66,7 +68,7 @@ register_api(create_strategy)
# Shortcut to filesystem2's open_fs, that we make available there for convenience.
@register_api
def open_fs(fs_url, *args, **kwargs):
def open_fs(fs_url=None, *args, **kwargs):
"""
Wraps :func:`fs.open_fs` function with a few candies.
@ -80,7 +82,13 @@ def open_fs(fs_url, *args, **kwargs):
:returns: :class:`~fs.base.FS` object
"""
from fs import open_fs as _open_fs
return _open_fs(str(fs_url), *args, **kwargs)
from os.path import expanduser
from os import getcwd
if fs_url is None:
fs_url = getcwd()
return _open_fs(expanduser(str(fs_url)), *args, **kwargs)
# bonobo.nodes
@ -93,7 +101,9 @@ register_api_group(
JsonReader,
JsonWriter,
Limit,
PrettyPrint,
PrettyPrinter,
PickleReader,
PickleWriter,
Tee,
count,
identity,

2
bonobo/_version.py
View File

@ -1 +1 @@
__version__ = '0.3.0a1'
__version__ = '0.4.3'

19
bonobo/commands/__init__.py
View File

@ -1,11 +1,13 @@
import argparse
import logging
from stevedore import ExtensionManager
from bonobo import logging, settings
logger = logging.get_logger()
def entrypoint(args=None):
parser = argparse.ArgumentParser()
parser.add_argument('--debug', '-D', action='store_true')
subparsers = parser.add_subparsers(dest='command')
subparsers.required = True
@ -17,12 +19,17 @@ def entrypoint(args=None):
parser = subparsers.add_parser(ext.name)
commands[ext.name] = ext.plugin(parser)
except Exception:
logging.exception('Error while loading command {}.'.format(ext.name))
logger.exception('Error while loading command {}.'.format(ext.name))
mgr = ExtensionManager(
namespace='bonobo.commands',
)
from stevedore import ExtensionManager
mgr = ExtensionManager(namespace='bonobo.commands')
mgr.map(register_extension)
args = parser.parse_args(args).__dict__
if args.pop('debug', False):
settings.DEBUG = True
settings.LOGGING_LEVEL = logging.DEBUG
logging.set_level(settings.LOGGING_LEVEL)
logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args))
commands[args.pop('command')](**args)

18
bonobo/commands/init.py
View File

@ -1,16 +1,20 @@
import os
def execute():
def execute(name, branch):
try:
from edgy.project.__main__ import handle_init
from cookiecutter.main import cookiecutter
except ImportError as exc:
raise ImportError(
'You must install "edgy.project" to use this command.\n\n $ pip install edgy.project\n'
'You must install "cookiecutter" to use this command.\n\n $ pip install cookiecutter\n'
) from exc
return handle_init(os.path.join(os.getcwd(), 'Projectfile'))
return cookiecutter(
'https://github.com/python-bonobo/cookiecutter-bonobo.git',
extra_context={'name': name},
no_input=True,
checkout=branch
)
def register(parser):
parser.add_argument('name')
parser.add_argument('--branch', '-b', default='master')
return execute

86
bonobo/commands/run.py
View File

@ -1,12 +1,11 @@
import argparse
import os
import bonobo
DEFAULT_SERVICES_FILENAME = '_services.py'
DEFAULT_SERVICES_ATTR = 'get_services'
DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', )
DEFAULT_GRAPH_ATTR = 'get_graph'
def get_default_services(filename, services=None):
dirname = os.path.dirname(filename)
@ -18,10 +17,8 @@ def get_default_services(filename, services=None):
'__name__': '__bonobo__',
'__file__': services_filename,
}
try:
exec(code, context)
except Exception:
raise
exec(code, context)
return {
**context[DEFAULT_SERVICES_ATTR](),
**(services or {}),
@ -29,26 +26,47 @@ def get_default_services(filename, services=None):
return services or {}
def execute(file, quiet=False):
with file:
code = compile(file.read(), file.name, 'exec')
def execute(filename, module, install=False, quiet=False, verbose=False):
import runpy
from bonobo import Graph, run, settings
# TODO: A few special variables should be set before running the file:
#
# See:
# - https://docs.python.org/3/reference/import.html#import-mod-attrs
# - https://docs.python.org/3/library/runpy.html#runpy.run_module
context = {
'__name__': '__bonobo__',
'__file__': file.name,
}
if quiet:
settings.QUIET = True
try:
exec(code, context)
except Exception as exc:
raise
if verbose:
settings.DEBUG = True
graphs = dict((k, v) for k, v in context.items() if isinstance(v, bonobo.Graph))
if filename:
if os.path.isdir(filename):
if install:
import importlib
import pip
requirements = os.path.join(filename, 'requirements.txt')
pip.main(['install', '-r', requirements])
# Some shenanigans to be sure everything is importable after this, especially .egg-link files which
# are referenced in *.pth files and apparently loaded by site.py at some magic bootstrap moment of the
# python interpreter.
pip.utils.pkg_resources = importlib.reload(pip.utils.pkg_resources)
import site
importlib.reload(site)
pathname = filename
for filename in DEFAULT_GRAPH_FILENAMES:
filename = os.path.join(pathname, filename)
if os.path.exists(filename):
break
if not os.path.exists(filename):
raise IOError('Could not find entrypoint (candidates: {}).'.format(', '.join(DEFAULT_GRAPH_FILENAMES)))
elif install:
raise RuntimeError('Cannot --install on a file (only available for dirs containing requirements.txt).')
context = runpy.run_path(filename, run_name='__bonobo__')
elif module:
context = runpy.run_module(module, run_name='__bonobo__')
filename = context['__file__']
else:
raise RuntimeError('UNEXPECTED: argparse should not allow this.')
graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph))
assert len(graphs) == 1, (
'Having zero or more than one graph definition in one file is unsupported for now, '
@ -59,16 +77,26 @@ def execute(file, quiet=False):
# todo if console and not quiet, then add the console plugin
# todo when better console plugin, add it if console and just disable display
return bonobo.run(
return run(
graph,
plugins=[],
services=get_default_services(
file.name, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
)
)
def register_generic_run_arguments(parser, required=True):
source_group = parser.add_mutually_exclusive_group(required=required)
source_group.add_argument('filename', nargs='?', type=str)
source_group.add_argument('--module', '-m', type=str)
return parser
def register(parser):
parser.add_argument('file', type=argparse.FileType())
parser.add_argument('--quiet', action='store_true')
parser = register_generic_run_arguments(parser)
verbosity_group = parser.add_mutually_exclusive_group()
verbosity_group.add_argument('--quiet', '-q', action='store_true')
verbosity_group.add_argument('--verbose', '-v', action='store_true')
parser.add_argument('--install', '-I', action='store_true')
return execute

37
bonobo/commands/version.py
View File

@ -1,9 +1,40 @@
import bonobo
def format_version(mod, *, name=None, quiet=False):
from bonobo.util.pkgs import bonobo_packages
args = {
'name': name or mod.__name__,
'version': mod.__version__,
'location': bonobo_packages[name or mod.__name__].location
}
if not quiet:
return '{name} v.{version} (in {location})'.format(**args)
if quiet < 2:
return '{name} {version}'.format(**args)
if quiet < 3:
return '{version}'.format(**args)
raise RuntimeError('Hard to be so quiet...')
def execute():
print('{} v.{}'.format(bonobo.__name__, bonobo.__version__))
def execute(all=False, quiet=False):
import bonobo
from bonobo.util.pkgs import bonobo_packages
print(format_version(bonobo, quiet=quiet))
if all:
for name in sorted(bonobo_packages):
if name != 'bonobo':
try:
mod = __import__(name.replace('-', '_'))
try:
print(format_version(mod, name=name, quiet=quiet))
except Exception as exc:
print('{} ({})'.format(name, exc))
except ImportError as exc:
print('{} is not importable ({}).'.format(name, exc))
def register(parser):
parser.add_argument('--all', '-a', action='store_true')
parser.add_argument('--quiet', '-q', action='count')
return execute

9
bonobo/config/__init__.py
View File

@ -1,13 +1,16 @@
from bonobo.config.configurables import Configurable
from bonobo.config.options import Option, Method
from bonobo.config.options import Method, Option
from bonobo.config.processors import ContextProcessor
from bonobo.config.services import Container, Service
from bonobo.config.services import Container, Exclusive, Service, requires
# bonobo.config public programming interface
__all__ = [
'Configurable',
'Container',
'ContextProcessor',
'Option',
'Exclusive',
'Method',
'Option',
'Service',
'requires',
]

15
bonobo/config/configurables.py
View File

@ -1,6 +1,6 @@
from bonobo.config.options import Method, Option
from bonobo.config.processors import ContextProcessor
from bonobo.errors import ConfigurationError
from bonobo.errors import ConfigurationError, AbstractError
__all__ = [
'Configurable',
@ -26,16 +26,19 @@ class ConfigurableMeta(type):
if isinstance(value, ContextProcessor):
cls.__processors__.append(value)
else:
if not value.name:
value.name = name
if isinstance(value, Method):
if cls.__wrappable__:
raise ConfigurationError(
'Cannot define more than one "Method" option in a configurable. That may change in the future.'
)
cls.__wrappable__ = name
if not value.name:
value.name = name
if not name in cls.__options__:
cls.__options__[name] = value
if value.positional:
cls.__positional_options__.append(name)
@ -53,11 +56,9 @@ class Configurable(metaclass=ConfigurableMeta):
def __new__(cls, *args, **kwargs):
if cls.__wrappable__ and len(args) == 1 and hasattr(args[0], '__call__'):
wrapped, args = args[0], args[1:]
return type(wrapped.__name__, (cls, ), {cls.__wrappable__: wrapped})
return type(args[0].__name__, (cls, ), {cls.__wrappable__: args[0]})
# XXX is that correct ??? how does it pass args/kwargs to __init__ ???
return super().__new__(cls)
return super(Configurable, cls).__new__(cls)
def __init__(self, *args, **kwargs):
super().__init__()

10
bonobo/config/options.py
View File

@ -67,12 +67,12 @@ class Option:
def __set__(self, inst, value):
inst.__options_values__[self.name] = self.clean(value)
def get_default(self):
return self.default() if callable(self.default) else self.default
def clean(self, value):
return self.type(value) if self.type else value
def get_default(self):
return self.default() if callable(self.default) else self.default
class Method(Option):
"""
@ -106,7 +106,7 @@ class Method(Option):
"""
def __init__(self):
super().__init__(None, required=False, positional=True)
super().__init__(None, required=False)
def __get__(self, inst, typ):
if not self.name in inst.__options_values__:
@ -114,6 +114,8 @@ class Method(Option):
return inst.__options_values__[self.name]
def __set__(self, inst, value):
if isinstance(value, str):
raise ValueError('should be callable')
inst.__options_values__[self.name] = self.type(value) if self.type else value
def clean(self, value):

38
bonobo/config/processors.py
View File

@ -1,4 +1,3 @@
import types
from collections import Iterable
from contextlib import contextmanager
@ -10,6 +9,34 @@ _CONTEXT_PROCESSORS_ATTR = '__processors__'
class ContextProcessor(Option):
"""
A ContextProcessor is a kind of transformation decorator that can setup and teardown a transformation and runtime
related dependencies, at the execution level.
It works like a yielding context manager, and is the recommended way to setup and teardown objects you'll need
in the context of one execution. It's the way to overcome the stateless nature of transformations.
The yielded values will be passed as positional arguments to the next context processors (order do matter), and
finally to the __call__ method of the transformation.
Warning: this may change for a similar but simpler implementation, don't relly too much on it (yet).
Example:
>>> from bonobo.config import Configurable
>>> from bonobo.util.objects import ValueHolder
>>> class Counter(Configurable):
... @ContextProcessor
... def counter(self, context):
... yield ValueHolder(0)
...
... def __call__(self, counter, *args, **kwargs):
... counter += 1
... yield counter.get()
"""
@property
def __name__(self):
return self.func.__name__
@ -104,14 +131,7 @@ def resolve_processors(mixed):
try:
yield from mixed.__processors__
except AttributeError:
# old code, deprecated usage
if isinstance(mixed, types.FunctionType):
yield from getattr(mixed, _CONTEXT_PROCESSORS_ATTR, ())
for cls in reversed((mixed if isinstance(mixed, type) else type(mixed)).__mro__):
yield from cls.__dict__.get(_CONTEXT_PROCESSORS_ATTR, ())
return ()
yield from ()
get_context_processors = deprecated_alias('get_context_processors', resolve_processors)

67
bonobo/config/services.py
View File

@ -1,7 +1,10 @@
import re
import threading
import types
from contextlib import ContextDecorator
from bonobo.config.options import Option
from bonobo.errors import MissingServiceImplementationError
_service_name_re = re.compile(r"^[^\d\W]\w*(:?\.[^\d\W]\w*)*$", re.UNICODE)
@ -53,7 +56,11 @@ class Service(Option):
inst.__options_values__[self.name] = validate_service_name(value)
def resolve(self, inst, services):
return services.get(getattr(inst, self.name))
try:
name = getattr(inst, self.name)
except AttributeError:
name = self.name
return services.get(name)
class Container(dict):
@ -78,8 +85,64 @@ class Container(dict):
if not name in self:
if default:
return default
raise KeyError('Cannot resolve service {!r} using provided service collection.'.format(name))
raise MissingServiceImplementationError(
'Cannot resolve service {!r} using provided service collection.'.format(name)
)
value = super().get(name)
# XXX this is not documented and can lead to errors.
if isinstance(value, types.LambdaType):
value = value(self)
return value
class Exclusive(ContextDecorator):
"""
Decorator and context manager used to require exclusive usage of an object, most probably a service. It's usefull
for example if call order matters on a service implementation (think of an http api that requires a nonce or version
parameter ...).
Usage:
>>> def handler(some_service):
... with Exclusive(some_service):
... some_service.call_1()
... some_service.call_2()
... some_service.call_3()
This will ensure that nobody else is using the same service while in the "with" block, using a lock primitive to
ensure that.
"""
_locks = {}
def __init__(self, wrapped):
self._wrapped = wrapped
def get_lock(self):
_id = id(self._wrapped)
if not _id in Exclusive._locks:
Exclusive._locks[_id] = threading.RLock()
return Exclusive._locks[_id]
def __enter__(self):
self.get_lock().acquire()
return self._wrapped
def __exit__(self, *exc):
self.get_lock().release()
def requires(*service_names):
def decorate(mixed):
try:
options = mixed.__options__
except AttributeError:
mixed.__options__ = options = {}
for service_name in service_names:
service = Service(service_name)
service.name = service_name
options[service_name] = service
return mixed
return decorate

6
bonobo/errors.py
View File

@ -55,4 +55,8 @@ class ProhibitedOperationError(RuntimeError):
class ConfigurationError(Exception):
pass
pass
class MissingServiceImplementationError(KeyError):
pass

242
bonobo/examples/datasets/coffeeshops.json
View File

@ -1,182 +1,182 @@
{"O q de poule": "53 rue du ruisseau, 75018 Paris, France",
{"Le Reynou": "2 bis quai de la m\u00e9gisserie, 75001 Paris, France",
"les montparnos": "65 boulevard Pasteur, 75015 Paris, France",
"Le Saint Jean": "23 rue des abbesses, 75018 Paris, France",
"Le Felteu": "1 rue Pecquay, 75004 Paris, France",
"O q de poule": "53 rue du ruisseau, 75018 Paris, France",
"Le chantereine": "51 Rue Victoire, 75009 Paris, France",
"Le M\u00fcller": "11 rue Feutrier, 75018 Paris, France",
"La Caravane": "Rue de la Fontaine au Roi, 75011 Paris, France",
"Le Pas Sage": "1 Passage du Grand Cerf, 75002 Paris, France",
"La Renaissance": "112 Rue Championnet, 75018 Paris, France",
"Ext\u00e9rieur Quai": "5, rue d'Alsace, 75010 Paris, France",
"Le Reynou": "2 bis quai de la m\u00e9gisserie, 75001 Paris, France",
"les montparnos": "65 boulevard Pasteur, 75015 Paris, France",
"Le Sully": "6 Bd henri IV, 75004 Paris, France",
"Le drapeau de la fidelit\u00e9": "21 rue Copreaux, 75015 Paris, France",
"Le caf\u00e9 des amis": "125 rue Blomet, 75015 Paris, France",
"Le Kleemend's": "34 avenue Pierre Mend\u00e8s-France, 75013 Paris, France",
"Assaporare Dix sur Dix": "75, avenue Ledru-Rollin, 75012 Paris, France",
"Caf\u00e9 Pierre": "202 rue du faubourg st antoine, 75012 Paris, France",
"Le Caf\u00e9 Livres": "10 rue Saint Martin, 75004 Paris, France",
"Le Chaumontois": "12 rue Armand Carrel, 75018 Paris, France",
"Le Square": "31 rue Saint-Dominique, 75007 Paris, France",
"Les Arcades": "61 rue de Ponthieu, 75008 Paris, France",
"Le Bosquet": "46 avenue Bosquet, 75007 Paris, France",
"Le bistrot de Ma\u00eblle et Augustin": "42 rue coquill\u00e8re, 75001 Paris, France",
"D\u00e9d\u00e9 la frite": "52 rue Notre-Dame des Victoires, 75002 Paris, France",
"Assaporare Dix sur Dix": "75, avenue Ledru-Rollin, 75012 Paris, France",
"Cardinal Saint-Germain": "11 boulevard Saint-Germain, 75005 Paris, France",
"Caf\u00e9 antoine": "17 rue Jean de la Fontaine, 75016 Paris, France",
"Au cerceau d'or": "129 boulevard sebastopol, 75002 Paris, France",
"Le Chaumontois": "12 rue Armand Carrel, 75018 Paris, France",
"Aux cadrans": "21 ter boulevard Diderot, 75012 Paris, France",
"Le Saint Jean": "23 rue des abbesses, 75018 Paris, France",
"Le Square": "31 rue Saint-Dominique, 75007 Paris, France",
"Les Arcades": "61 rue de Ponthieu, 75008 Paris, France",
"Caf\u00e9 Lea": "5 rue Claude Bernard, 75005 Paris, France",
"Le Bellerive": "71 quai de Seine, 75019 Paris, France",
"La Bauloise": "36 rue du hameau, 75015 Paris, France",
"Le Dellac": "14 rue Rougemont, 75009 Paris, France",
"Le Bosquet": "46 avenue Bosquet, 75007 Paris, France",
"Le Sully": "6 Bd henri IV, 75004 Paris, France",
"Le Felteu": "1 rue Pecquay, 75004 Paris, France",
"Le drapeau de la fidelit\u00e9": "21 rue Copreaux, 75015 Paris, France",
"Le caf\u00e9 des amis": "125 rue Blomet, 75015 Paris, France",
"Le Kleemend's": "34 avenue Pierre Mend\u00e8s-France, 75013 Paris, France",
"Caf\u00e9 Pierre": "202 rue du faubourg st antoine, 75012 Paris, France",
"Le M\u00fcller": "11 rue Feutrier, 75018 Paris, France",
"Le Caf\u00e9 Livres": "10 rue Saint Martin, 75004 Paris, France",
"La Cordonnerie": "142 Rue Saint-Denis 75002 Paris, 75002 Paris, France",
"Invitez vous chez nous": "7 rue Ep\u00e9e de Bois, 75005 Paris, France",
"Au bon coin": "49 rue des Cloys, 75018 Paris, France",
"La Br\u00fblerie des Ternes": "111 rue mouffetard, 75005 Paris, France",
"Le Petit Choiseul": "23 rue saint augustin, 75002 Paris, France",
"O'Breizh": "27 rue de Penthi\u00e8vre, 75008 Paris, France",
"Le Supercoin": "3, rue Baudelique, 75018 Paris, France",
"Populettes": "86 bis rue Riquet, 75018 Paris, France",
"Le Couvent": "69 rue Broca, 75013 Paris, France",
"Caf\u00e9 Zen": "46 rue Victoire, 75009 Paris, France",
"Le Chat bossu": "126, rue du Faubourg Saint Antoine, 75012 Paris, France",
"Le petit club": "55 rue de la tombe Issoire, 75014 Paris, France",
"Le Relais Haussmann": "146, boulevard Haussmann, 75008 Paris, France",
"Denfert caf\u00e9": "58 boulvevard Saint Jacques, 75014 Paris, France",
"Bagels & Coffee Corner": "Place de Clichy, 75017 Paris, France",
"Le Plein soleil": "90 avenue Parmentier, 75011 Paris, France",
"La Perle": "78 rue vieille du temple, 75003 Paris, France",
"Le Caf\u00e9 frapp\u00e9": "95 rue Montmartre, 75002 Paris, France",
"L'\u00c9cir": "59 Boulevard Saint-Jacques, 75014 Paris, France",
"Le Descartes": "1 rue Thouin, 75005 Paris, France",
"Br\u00fblerie San Jos\u00e9": "30 rue des Petits-Champs, 75002 Paris, France",
"Caf\u00e9 de la Mairie (du VIII)": "rue de Lisbonne, 75008 Paris, France",
"Au panini de la place": "47 rue Belgrand, 75020 Paris, France",
"Extra old caf\u00e9": "307 fg saint Antoine, 75011 Paris, France",
"Le General Beuret": "9 Place du General Beuret, 75015 Paris, France",
"Le Cap Bourbon": "1 rue Louis le Grand, 75002 Paris, France",
"En attendant l'or": "3 rue Faidherbe, 75011 Paris, France",
"Le Pure caf\u00e9": "14 rue Jean Mac\u00e9, 75011 Paris, France",
"Le Village": "182 rue de Courcelles, 75017 Paris, France",
"Le Malar": "88 rue Saint-Dominique, 75007 Paris, France",
"Pause Caf\u00e9": "41 rue de Charonne, 75011 Paris, France",
"Chez Fafa": "44 rue Vinaigriers, 75010 Paris, France",
"La Recoleta au Manoir": "229 avenue Gambetta, 75020 Paris, France",
"Le Pareloup": "80 Rue Saint-Charles, 75015 Paris, France",
"Caf\u00e9 Martin": "2 place Martin Nadaud, 75001 Paris, France",
"Etienne": "14 rue Turbigo, Paris, 75001 Paris, France",
"L'ing\u00e9nu": "184 bd Voltaire, 75011 Paris, France",
"Le Biz": "18 rue Favart, 75002 Paris, France",
"L'Olive": "8 rue L'Olive, 75018 Paris, France",
"Le pari's caf\u00e9": "104 rue caulaincourt, 75018 Paris, France",
"Le Poulailler": "60 rue saint-sabin, 75011 Paris, France",
"La Marine": "55 bis quai de valmy, 75010 Paris, France",
"American Kitchen": "49 rue bichat, 75010 Paris, France",
"Chai 33": "33 Cour Saint Emilion, 75012 Paris, France",
"Face Bar": "82 rue des archives, 75003 Paris, France",
"Le Bloc": "21 avenue Brochant, 75017 Paris, France",
"La Bricole": "52 rue Liebniz, 75018 Paris, France",
"le ronsard": "place maubert, 75005 Paris, France",
"l'Usine": "1 rue d'Avron, 75020 Paris, France",
"La Brasserie Gait\u00e9": "3 rue de la Gait\u00e9, 75014 Paris, France",
"Le General Beuret": "9 Place du General Beuret, 75015 Paris, France",
"Le Cap Bourbon": "1 rue Louis le Grand, 75002 Paris, France",
"La Cordonnerie": "142 Rue Saint-Denis 75002 Paris, 75002 Paris, France",
"Invitez vous chez nous": "7 rue Ep\u00e9e de Bois, 75005 Paris, France",
"Le sully": "13 rue du Faubourg Saint Denis, 75010 Paris, France",
"Le Ragueneau": "202 rue Saint-Honor\u00e9, 75001 Paris, France",
"Le Germinal": "95 avenue Emile Zola, 75015 Paris, France",
"Caf\u00e9 Martin": "2 place Martin Nadaud, 75001 Paris, France",
"Etienne": "14 rue Turbigo, Paris, 75001 Paris, France",
"L'ing\u00e9nu": "184 bd Voltaire, 75011 Paris, France",
"Le refuge": "72 rue lamarck, 75018 Paris, France",
"Le Biz": "18 rue Favart, 75002 Paris, France",
"L'Olive": "8 rue L'Olive, 75018 Paris, France",
"Le sully": "13 rue du Faubourg Saint Denis, 75010 Paris, France",
"Drole d'endroit pour une rencontre": "58 rue de Montorgueil, 75002 Paris, France",
"Le Petit Choiseul": "23 rue saint augustin, 75002 Paris, France",
"O'Breizh": "27 rue de Penthi\u00e8vre, 75008 Paris, France",
"Le Supercoin": "3, rue Baudelique, 75018 Paris, France",
"Populettes": "86 bis rue Riquet, 75018 Paris, France",
"La Recoleta au Manoir": "229 avenue Gambetta, 75020 Paris, France",
"L'Assassin": "99 rue Jean-Pierre Timbaud, 75011 Paris, France",
"Le pari's caf\u00e9": "104 rue caulaincourt, 75018 Paris, France",
"Le Poulailler": "60 rue saint-sabin, 75011 Paris, France",
"Chai 33": "33 Cour Saint Emilion, 75012 Paris, France",
"Le Pareloup": "80 Rue Saint-Charles, 75015 Paris, France",
"Caf\u00e9 Zen": "46 rue Victoire, 75009 Paris, France",
"La Brasserie Gait\u00e9": "3 rue de la Gait\u00e9, 75014 Paris, France",
"Au bon coin": "49 rue des Cloys, 75018 Paris, France",
"La Br\u00fblerie des Ternes": "111 rue mouffetard, 75005 Paris, France",
"Le Chat bossu": "126, rue du Faubourg Saint Antoine, 75012 Paris, France",
"Denfert caf\u00e9": "58 boulvevard Saint Jacques, 75014 Paris, France",
"Le Couvent": "69 rue Broca, 75013 Paris, France",
"Bagels & Coffee Corner": "Place de Clichy, 75017 Paris, France",
"La Perle": "78 rue vieille du temple, 75003 Paris, France",
"Le Caf\u00e9 frapp\u00e9": "95 rue Montmartre, 75002 Paris, France",
"L'\u00c9cir": "59 Boulevard Saint-Jacques, 75014 Paris, France",
"Le Descartes": "1 rue Thouin, 75005 Paris, France",
"Le petit club": "55 rue de la tombe Issoire, 75014 Paris, France",
"Le Relais Haussmann": "146, boulevard Haussmann, 75008 Paris, France",
"Au panini de la place": "47 rue Belgrand, 75020 Paris, France",
"Extra old caf\u00e9": "307 fg saint Antoine, 75011 Paris, France",
"Le Plein soleil": "90 avenue Parmentier, 75011 Paris, France",
"Le Pure caf\u00e9": "14 rue Jean Mac\u00e9, 75011 Paris, France",
"Le Village": "182 rue de Courcelles, 75017 Paris, France",
"Le Malar": "88 rue Saint-Dominique, 75007 Paris, France",
"Pause Caf\u00e9": "41 rue de Charonne, 75011 Paris, France",
"Chez Fafa": "44 rue Vinaigriers, 75010 Paris, France",
"Caf\u00e9 dans l'aerogare Air France Invalides": "2 rue Robert Esnault Pelterie, 75007 Paris, France",
"Le relais de la victoire": "73 rue de la Victoire, 75009 Paris, France",
"Caprice caf\u00e9": "12 avenue Jean Moulin, 75014 Paris, France",
"Caves populaires": "22 rue des Dames, 75017 Paris, France",
"Cafe de grenelle": "188 rue de Grenelle, 75007 Paris, France",
"Chez Prune": "36 rue Beaurepaire, 75010 Paris, France",
"L'anjou": "1 rue de Montholon, 75009 Paris, France",
"Le Brio": "216, rue Marcadet, 75018 Paris, France",
"Tamm Bara": "7 rue Clisson, 75013 Paris, France",
"La chaumi\u00e8re gourmande": "Route de la Muette \u00e0 Neuilly",
"Club hippique du Jardin d\u2019Acclimatation": "75016 Paris, France",
"Les P\u00e8res Populaires": "46 rue de Buzenval, 75020 Paris, France",
"Epicerie Musicale": "55bis quai de Valmy, 75010 Paris, France",
"Le relais de la victoire": "73 rue de la Victoire, 75009 Paris, France",
"Le Centenaire": "104 rue amelot, 75011 Paris, France",
"Caf\u00e9 Pistache": "9 rue des petits champs, 75001 Paris, France",
"La Cagnotte": "13 Rue Jean-Baptiste Dumay, 75020 Paris, France",
"Le bal du pirate": "60 rue des bergers, 75015 Paris, France",
"bistrot les timbr\u00e9s": "14 rue d'alleray, 75015 Paris, France",
"Le Killy Jen": "28 bis boulevard Diderot, 75012 Paris, France",
"le 1 cinq": "172 rue de vaugirard, 75015 Paris, France",
"Les Artisans": "106 rue Lecourbe, 75015 Paris, France",
"Peperoni": "83 avenue de Wagram, 75001 Paris, France",
"Le Zazabar": "116 Rue de M\u00e9nilmontant, 75020 Paris, France",
"Ragueneau": "202 rue Saint Honor\u00e9, 75001 Paris, France",
"l'orillon bar": "35 rue de l'orillon, 75011 Paris, France",
"zic zinc": "95 rue claude decaen, 75012 Paris, France",
"L'In\u00e9vitable": "22 rue Linn\u00e9, 75005 Paris, France",
"Le Brio": "216, rue Marcadet, 75018 Paris, France",
"Le Dunois": "77 rue Dunois, 75013 Paris, France",
"La Montagne Sans Genevi\u00e8ve": "13 Rue du Pot de Fer, 75005 Paris, France",
"Le Caminito": "48 rue du Dessous des Berges, 75013 Paris, France",
"Le petit Bretonneau": "Le petit Bretonneau - \u00e0 l'int\u00e9rieur de l'H\u00f4pital, 75018 Paris, France",
"La chaumi\u00e8re gourmande": "Route de la Muette \u00e0 Neuilly",
"Club hippique du Jardin d\u2019Acclimatation": "75016 Paris, France",
"Caprice caf\u00e9": "12 avenue Jean Moulin, 75014 Paris, France",
"Le Zazabar": "116 Rue de M\u00e9nilmontant, 75020 Paris, France",
"Caf\u00e9 beauveau": "9 rue de Miromesnil, 75008 Paris, France",
"Caves populaires": "22 rue des Dames, 75017 Paris, France",
"Cafe de grenelle": "188 rue de Grenelle, 75007 Paris, France",
"Au Vin Des Rues": "21 rue Boulard, 75014 Paris, France",
"Le bal du pirate": "60 rue des bergers, 75015 Paris, France",
"L'antre d'eux": "16 rue DE MEZIERES, 75006 Paris, France",
"Chez Prune": "36 rue Beaurepaire, 75010 Paris, France",
"L'anjou": "1 rue de Montholon, 75009 Paris, France",
"Tamm Bara": "7 rue Clisson, 75013 Paris, France",
"La cantoche de Paname": "40 Boulevard Beaumarchais, 75011 Paris, France",
"Le Saint Ren\u00e9": "148 Boulevard de Charonne, 75020 Paris, France",
"Caf\u00e9 Clochette": "16 avenue Richerand, 75010 Paris, France",
"L'europ\u00e9en": "21 Bis Boulevard Diderot, 75012 Paris, France",
"l'orillon bar": "35 rue de l'orillon, 75011 Paris, France",
"zic zinc": "95 rue claude decaen, 75012 Paris, France",
"Caf\u00e9 Pistache": "9 rue des petits champs, 75001 Paris, France",
"La Cagnotte": "13 Rue Jean-Baptiste Dumay, 75020 Paris, France",
"bistrot les timbr\u00e9s": "14 rue d'alleray, 75015 Paris, France",
"Le Killy Jen": "28 bis boulevard Diderot, 75012 Paris, France",
"Caf\u00e9 beauveau": "9 rue de Miromesnil, 75008 Paris, France",
"le 1 cinq": "172 rue de vaugirard, 75015 Paris, France",
"Au Vin Des Rues": "21 rue Boulard, 75014 Paris, France",
"Les Artisans": "106 rue Lecourbe, 75015 Paris, France",
"Peperoni": "83 avenue de Wagram, 75001 Paris, France",
"Le BB (Bouchon des Batignolles)": "2 rue Lemercier, 75017 Paris, France",
"La Libert\u00e9": "196 rue du faubourg saint-antoine, 75012 Paris, France",
"Chez Rutabaga": "16 rue des Petits Champs, 75002 Paris, France",
"Les caves populaires": "22 rue des Dames, 75017 Paris, France",
"Le Plomb du cantal": "3 rue Ga\u00eet\u00e9, 75014 Paris, France",
"Bistrot Saint-Antoine": "58 rue du Fbg Saint-Antoine, 75012 Paris, France",
"Trois pi\u00e8ces cuisine": "101 rue des dames, 75017 Paris, France",
"La cantoche de Paname": "40 Boulevard Beaumarchais, 75011 Paris, France",
"Le Saint Ren\u00e9": "148 Boulevard de Charonne, 75020 Paris, France",
"La Brocante": "10 rue Rossini, 75009 Paris, France",
"Le Zinc": "61 avenue de la Motte Picquet, 75015 Paris, France",
"Chez Oscar": "11/13 boulevard Beaumarchais, 75004 Paris, France",
"Le Piquet": "48 avenue de la Motte Picquet, 75015 Paris, France",
"L'avant comptoir": "3 carrefour de l'Od\u00e9on, 75006 Paris, France",
"le chateau d'eau": "67 rue du Ch\u00e2teau d'eau, 75010 Paris, France",
"maison du vin": "52 rue des plantes, 75014 Paris, France",
"Les Vendangeurs": "6/8 rue Stanislas, 75006 Paris, France",
"Caf\u00e9 Clochette": "16 avenue Richerand, 75010 Paris, France",
"L'europ\u00e9en": "21 Bis Boulevard Diderot, 75012 Paris, France",
"NoMa": "39 rue Notre Dame de Nazareth, 75003 Paris, France",
"Chez Luna": "108 rue de M\u00e9nilmontant, 75020 Paris, France",
"Le Tournebride": "104 rue Mouffetard, 75005 Paris, France",
"le lutece": "380 rue de vaugirard, 75015 Paris, France",
"Le bar Fleuri": "1 rue du Plateau, 75019 Paris, France",
"Le Fronton": "63 rue de Ponthieu, 75008 Paris, France",
"O'Paris": "1 Rue des Envierges, 75020 Paris, France",
"Rivolux": "16 rue de Rivoli, 75004 Paris, France",
"Brasiloja": "16 rue Ganneron, 75018 Paris, France",
"Botak cafe": "1 rue Paul albert, 75018 Paris, France",
"La cantine de Zo\u00e9": "136 rue du Faubourg poissonni\u00e8re, 75010 Paris, France",
"Institut des Cultures d'Islam": "19-23 rue L\u00e9on, 75018 Paris, France",
"Chez Miamophile": "6 rue M\u00e9lingue, 75019 Paris, France",
"Canopy Caf\u00e9 associatif": "19 rue Pajol, 75018 Paris, France",
"Caf\u00e9 rallye tournelles": "11 Quai de la Tournelle, 75005 Paris, France",
"Petits Freres des Pauvres": "47 rue de Batignolles, 75017 Paris, France",
"Les caves populaires": "22 rue des Dames, 75017 Paris, France",
"Le Plomb du cantal": "3 rue Ga\u00eet\u00e9, 75014 Paris, France",
"Trois pi\u00e8ces cuisine": "101 rue des dames, 75017 Paris, France",
"Le Zinc": "61 avenue de la Motte Picquet, 75015 Paris, France",
"L'avant comptoir": "3 carrefour de l'Od\u00e9on, 75006 Paris, France",
"Les Vendangeurs": "6/8 rue Stanislas, 75006 Paris, France",
"Chez Luna": "108 rue de M\u00e9nilmontant, 75020 Paris, France",
"Le bar Fleuri": "1 rue du Plateau, 75019 Paris, France",
"Bistrot Saint-Antoine": "58 rue du Fbg Saint-Antoine, 75012 Paris, France",
"Chez Oscar": "11/13 boulevard Beaumarchais, 75004 Paris, France",
"Le Piquet": "48 avenue de la Motte Picquet, 75015 Paris, France",
"le chateau d'eau": "67 rue du Ch\u00e2teau d'eau, 75010 Paris, France",
"maison du vin": "52 rue des plantes, 75014 Paris, France",
"Le Tournebride": "104 rue Mouffetard, 75005 Paris, France",
"Le Fronton": "63 rue de Ponthieu, 75008 Paris, France",
"le lutece": "380 rue de vaugirard, 75015 Paris, France",
"Rivolux": "16 rue de Rivoli, 75004 Paris, France",
"Brasiloja": "16 rue Ganneron, 75018 Paris, France",
"Le caf\u00e9 Monde et M\u00e9dias": "Place de la R\u00e9publique, 75003 Paris, France",
"L'entrep\u00f4t": "157 rue Bercy 75012 Paris, 75012 Paris, France",
"Coffee Chope": "344Vrue Vaugirard, 75015 Paris, France",
"Le Comptoir": "354 bis rue Vaugirard, 75015 Paris, France",
"L'empreinte": "54, avenue Daumesnil, 75012 Paris, France",
"Caf\u00e9 Victor": "10 boulevard Victor, 75015 Paris, France",
"Caf\u00e9 Varenne": "36 rue de Varenne, 75007 Paris, France",
"Le Brigadier": "12 rue Blanche, 75009 Paris, France",
"Melting Pot": "3 rue de Lagny, 75020 Paris, France",
"L'Entracte": "place de l'opera, 75002 Paris, France",
"le Zango": "58 rue Daguerre, 75014 Paris, France",
"Panem": "18 rue de Crussol, 75011 Paris, France",
"Waikiki": "10 rue d\"Ulm, 75005 Paris, France",
"l'El\u00e9phant du nil": "125 Rue Saint-Antoine, 75004 Paris, France",
"Le Parc Vaugirard": "358 rue de Vaugirard, 75015 Paris, France",
"Pari's Caf\u00e9": "174 avenue de Clichy, 75017 Paris, France",
"Le Comptoir": "354 bis rue Vaugirard, 75015 Paris, France",
"Caf\u00e9 Varenne": "36 rue de Varenne, 75007 Paris, France",
"Melting Pot": "3 rue de Lagny, 75020 Paris, France",
"le Zango": "58 rue Daguerre, 75014 Paris, France",
"Chez Miamophile": "6 rue M\u00e9lingue, 75019 Paris, France",
"Institut des Cultures d'Islam": "19-23 rue L\u00e9on, 75018 Paris, France",
"Canopy Caf\u00e9 associatif": "19 rue Pajol, 75018 Paris, France",
"Caf\u00e9 rallye tournelles": "11 Quai de la Tournelle, 75005 Paris, France",
"Petits Freres des Pauvres": "47 rue de Batignolles, 75017 Paris, France",
"Brasserie le Morvan": "61 rue du ch\u00e2teau d'eau, 75010 Paris, France",
"L'Angle": "28 rue de Ponthieu, 75008 Paris, France",
"Caf\u00e9 Dupont": "198 rue de la Convention, 75015 Paris, France",
"L'Entracte": "place de l'opera, 75002 Paris, France",
"Panem": "18 rue de Crussol, 75011 Paris, France",
"Au pays de Vannes": "34 bis rue de Wattignies, 75012 Paris, France",
"Le Lucernaire": "53 rue Notre-Dame des Champs, 75006 Paris, France",
"L'Angle": "28 rue de Ponthieu, 75008 Paris, France",
"Le Caf\u00e9 d'avant": "35 rue Claude Bernard, 75005 Paris, France",
"Caf\u00e9 Dupont": "198 rue de la Convention, 75015 Paris, France",
"L'\u00e2ge d'or": "26 rue du Docteur Magnan, 75013 Paris, France",
"Le S\u00e9vign\u00e9": "15 rue du Parc Royal, 75003 Paris, France",
"L'horizon": "93, rue de la Roquette, 75011 Paris, France"}
"L'horizon": "93, rue de la Roquette, 75011 Paris, France",
"L'empreinte": "54, avenue Daumesnil, 75012 Paris, France",
"Caf\u00e9 Victor": "10 boulevard Victor, 75015 Paris, France",
"Le Brigadier": "12 rue Blanche, 75009 Paris, France",
"Waikiki": "10 rue d\"Ulm, 75005 Paris, France"}

240
bonobo/examples/datasets/coffeeshops.txt
View File

@ -1,182 +1,182 @@
Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
les montparnos, 65 boulevard Pasteur, 75015 Paris, France
Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
Le Felteu, 1 rue Pecquay, 75004 Paris, France
O q de poule, 53 rue du ruisseau, 75018 Paris, France
Le chantereine, 51 Rue Victoire, 75009 Paris, France
Le Müller, 11 rue Feutrier, 75018 Paris, France
La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France
Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France
La Renaissance, 112 Rue Championnet, 75018 Paris, France
Extérieur Quai, 5, rue d'Alsace, 75010 Paris, France
Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
les montparnos, 65 boulevard Pasteur, 75015 Paris, France
Le Sully, 6 Bd henri IV, 75004 Paris, France
Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
Le café des amis, 125 rue Blomet, 75015 Paris, France
Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
Le Café Livres, 10 rue Saint Martin, 75004 Paris, France
Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France
Le Square, 31 rue Saint-Dominique, 75007 Paris, France
Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
Le bistrot de Maëlle et Augustin, 42 rue coquillère, 75001 Paris, France
Dédé la frite, 52 rue Notre-Dame des Victoires, 75002 Paris, France
Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
Cardinal Saint-Germain, 11 boulevard Saint-Germain, 75005 Paris, France
Café antoine, 17 rue Jean de la Fontaine, 75016 Paris, France
Au cerceau d'or, 129 boulevard sebastopol, 75002 Paris, France
Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France
Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France
Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
Le Square, 31 rue Saint-Dominique, 75007 Paris, France
Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
Café Lea, 5 rue Claude Bernard, 75005 Paris, France
Le Bellerive, 71 quai de Seine, 75019 Paris, France
La Bauloise, 36 rue du hameau, 75015 Paris, France
Le Dellac, 14 rue Rougemont, 75009 Paris, France
Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
Le Sully, 6 Bd henri IV, 75004 Paris, France
Le Felteu, 1 rue Pecquay, 75004 Paris, France
Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
Le café des amis, 125 rue Blomet, 75015 Paris, France
Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
Le Müller, 11 rue Feutrier, 75018 Paris, France
Le Café Livres, 10 rue Saint Martin, 75004 Paris, France
La Cordonnerie, 142 Rue Saint-Denis 75002 Paris, 75002 Paris, France
Invitez vous chez nous, 7 rue Epée de Bois, 75005 Paris, France
Au bon coin, 49 rue des Cloys, 75018 Paris, France
La Brûlerie des Ternes, 111 rue mouffetard, 75005 Paris, France
Le Petit Choiseul, 23 rue saint augustin, 75002 Paris, France
O'Breizh, 27 rue de Penthièvre, 75008 Paris, France
Le Supercoin, 3, rue Baudelique, 75018 Paris, France
Populettes, 86 bis rue Riquet, 75018 Paris, France
Le Couvent, 69 rue Broca, 75013 Paris, France
Café Zen, 46 rue Victoire, 75009 Paris, France
Le Chat bossu, 126, rue du Faubourg Saint Antoine, 75012 Paris, France
Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France
Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France
Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France
Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France
La Perle, 78 rue vieille du temple, 75003 Paris, France
Le Café frappé, 95 rue Montmartre, 75002 Paris, France
L'Écir, 59 Boulevard Saint-Jacques, 75014 Paris, France
Le Descartes, 1 rue Thouin, 75005 Paris, France
Brûlerie San José, 30 rue des Petits-Champs, 75002 Paris, France
Café de la Mairie (du VIII), rue de Lisbonne, 75008 Paris, France
Au panini de la place, 47 rue Belgrand, 75020 Paris, France
Extra old café, 307 fg saint Antoine, 75011 Paris, France
Le General Beuret, 9 Place du General Beuret, 75015 Paris, France
Le Cap Bourbon, 1 rue Louis le Grand, 75002 Paris, France
En attendant l'or, 3 rue Faidherbe, 75011 Paris, France
Le Pure café, 14 rue Jean Macé, 75011 Paris, France
Le Village, 182 rue de Courcelles, 75017 Paris, France
Le Malar, 88 rue Saint-Dominique, 75007 Paris, France
Pause Café, 41 rue de Charonne, 75011 Paris, France
Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France
La Recoleta au Manoir, 229 avenue Gambetta, 75020 Paris, France
Le Pareloup, 80 Rue Saint-Charles, 75015 Paris, France
Café Martin, 2 place Martin Nadaud, 75001 Paris, France
Etienne, 14 rue Turbigo, Paris, 75001 Paris, France
L'ingénu, 184 bd Voltaire, 75011 Paris, France
Le Biz, 18 rue Favart, 75002 Paris, France
L'Olive, 8 rue L'Olive, 75018 Paris, France
Le pari's café, 104 rue caulaincourt, 75018 Paris, France
Le Poulailler, 60 rue saint-sabin, 75011 Paris, France
La Marine, 55 bis quai de valmy, 75010 Paris, France
American Kitchen, 49 rue bichat, 75010 Paris, France
Chai 33, 33 Cour Saint Emilion, 75012 Paris, France
Face Bar, 82 rue des archives, 75003 Paris, France
Le Bloc, 21 avenue Brochant, 75017 Paris, France
La Bricole, 52 rue Liebniz, 75018 Paris, France
le ronsard, place maubert, 75005 Paris, France
l'Usine, 1 rue d'Avron, 75020 Paris, France
La Brasserie Gaité, 3 rue de la Gaité, 75014 Paris, France
Le General Beuret, 9 Place du General Beuret, 75015 Paris, France
Le Cap Bourbon, 1 rue Louis le Grand, 75002 Paris, France
La Cordonnerie, 142 Rue Saint-Denis 75002 Paris, 75002 Paris, France
Invitez vous chez nous, 7 rue Epée de Bois, 75005 Paris, France
Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France
Le Ragueneau, 202 rue Saint-Honoré, 75001 Paris, France
Le Germinal, 95 avenue Emile Zola, 75015 Paris, France
Café Martin, 2 place Martin Nadaud, 75001 Paris, France
Etienne, 14 rue Turbigo, Paris, 75001 Paris, France
L'ingénu, 184 bd Voltaire, 75011 Paris, France
Le refuge, 72 rue lamarck, 75018 Paris, France
Le Biz, 18 rue Favart, 75002 Paris, France
L'Olive, 8 rue L'Olive, 75018 Paris, France
Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France
Drole d'endroit pour une rencontre, 58 rue de Montorgueil, 75002 Paris, France
Le Petit Choiseul, 23 rue saint augustin, 75002 Paris, France
O'Breizh, 27 rue de Penthièvre, 75008 Paris, France
Le Supercoin, 3, rue Baudelique, 75018 Paris, France
Populettes, 86 bis rue Riquet, 75018 Paris, France
La Recoleta au Manoir, 229 avenue Gambetta, 75020 Paris, France
L'Assassin, 99 rue Jean-Pierre Timbaud, 75011 Paris, France
Le pari's café, 104 rue caulaincourt, 75018 Paris, France
Le Poulailler, 60 rue saint-sabin, 75011 Paris, France
Chai 33, 33 Cour Saint Emilion, 75012 Paris, France
Le Pareloup, 80 Rue Saint-Charles, 75015 Paris, France
Café Zen, 46 rue Victoire, 75009 Paris, France
La Brasserie Gaité, 3 rue de la Gaité, 75014 Paris, France
Au bon coin, 49 rue des Cloys, 75018 Paris, France
La Brûlerie des Ternes, 111 rue mouffetard, 75005 Paris, France
Le Chat bossu, 126, rue du Faubourg Saint Antoine, 75012 Paris, France
Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France
Le Couvent, 69 rue Broca, 75013 Paris, France
Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
La Perle, 78 rue vieille du temple, 75003 Paris, France
Le Café frappé, 95 rue Montmartre, 75002 Paris, France
L'Écir, 59 Boulevard Saint-Jacques, 75014 Paris, France
Le Descartes, 1 rue Thouin, 75005 Paris, France
Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France
Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France
Au panini de la place, 47 rue Belgrand, 75020 Paris, France
Extra old café, 307 fg saint Antoine, 75011 Paris, France
Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France
Le Pure café, 14 rue Jean Macé, 75011 Paris, France
Le Village, 182 rue de Courcelles, 75017 Paris, France
Le Malar, 88 rue Saint-Dominique, 75007 Paris, France
Pause Café, 41 rue de Charonne, 75011 Paris, France
Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France
Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France
Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
Caves populaires, 22 rue des Dames, 75017 Paris, France
Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
L'anjou, 1 rue de Montholon, 75009 Paris, France
Le Brio, 216, rue Marcadet, 75018 Paris, France
Tamm Bara, 7 rue Clisson, 75013 Paris, France
La chaumière gourmande, Route de la Muette à Neuilly
Club hippique du Jardin d’Acclimatation, 75016 Paris, France
Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France
Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France
Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
Le Centenaire, 104 rue amelot, 75011 Paris, France
Café Pistache, 9 rue des petits champs, 75001 Paris, France
La Cagnotte, 13 Rue Jean-Baptiste Dumay, 75020 Paris, France
Le bal du pirate, 60 rue des bergers, 75015 Paris, France
bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
le 1 cinq, 172 rue de vaugirard, 75015 Paris, France
Les Artisans, 106 rue Lecourbe, 75015 Paris, France
Peperoni, 83 avenue de Wagram, 75001 Paris, France
Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
Ragueneau, 202 rue Saint Honoré, 75001 Paris, France
l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
zic zinc, 95 rue claude decaen, 75012 Paris, France
L'Inévitable, 22 rue Linné, 75005 Paris, France
Le Brio, 216, rue Marcadet, 75018 Paris, France
Le Dunois, 77 rue Dunois, 75013 Paris, France
La Montagne Sans Geneviève, 13 Rue du Pot de Fer, 75005 Paris, France
Le Caminito, 48 rue du Dessous des Berges, 75013 Paris, France
Le petit Bretonneau, Le petit Bretonneau - à l'intérieur de l'Hôpital, 75018 Paris, France
La chaumière gourmande, Route de la Muette à Neuilly
Club hippique du Jardin d’Acclimatation, 75016 Paris, France
Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
Caves populaires, 22 rue des Dames, 75017 Paris, France
Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
Le bal du pirate, 60 rue des bergers, 75015 Paris, France
L'antre d'eux, 16 rue DE MEZIERES, 75006 Paris, France
Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
L'anjou, 1 rue de Montholon, 75009 Paris, France
Tamm Bara, 7 rue Clisson, 75013 Paris, France
La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
Café Clochette, 16 avenue Richerand, 75010 Paris, France
L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France
l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
zic zinc, 95 rue claude decaen, 75012 Paris, France
Café Pistache, 9 rue des petits champs, 75001 Paris, France
La Cagnotte, 13 Rue Jean-Baptiste Dumay, 75020 Paris, France
bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
le 1 cinq, 172 rue de vaugirard, 75015 Paris, France
Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
Les Artisans, 106 rue Lecourbe, 75015 Paris, France
Peperoni, 83 avenue de Wagram, 75001 Paris, France
Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France
La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France
Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France
Les caves populaires, 22 rue des Dames, 75017 Paris, France
Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
La Brocante, 10 rue Rossini, 75009 Paris, France
Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
maison du vin, 52 rue des plantes, 75014 Paris, France
Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
Café Clochette, 16 avenue Richerand, 75010 Paris, France
L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France
NoMa, 39 rue Notre Dame de Nazareth, 75003 Paris, France
Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
le lutece, 380 rue de vaugirard, 75015 Paris, France
Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
O'Paris, 1 Rue des Envierges, 75020 Paris, France
Rivolux, 16 rue de Rivoli, 75004 Paris, France
Brasiloja, 16 rue Ganneron, 75018 Paris, France
Botak cafe, 1 rue Paul albert, 75018 Paris, France
La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France
Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France
Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
Les caves populaires, 22 rue des Dames, 75017 Paris, France
Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
maison du vin, 52 rue des plantes, 75014 Paris, France
Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
le lutece, 380 rue de vaugirard, 75015 Paris, France
Rivolux, 16 rue de Rivoli, 75004 Paris, France
Brasiloja, 16 rue Ganneron, 75018 Paris, France
Le café Monde et Médias, Place de la République, 75003 Paris, France
L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France
Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France
Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
Café Victor, 10 boulevard Victor, 75015 Paris, France
Café Varenne, 36 rue de Varenne, 75007 Paris, France
Le Brigadier, 12 rue Blanche, 75009 Paris, France
Melting Pot, 3 rue de Lagny, 75020 Paris, France
L'Entracte, place de l'opera, 75002 Paris, France
le Zango, 58 rue Daguerre, 75014 Paris, France
Panem, 18 rue de Crussol, 75011 Paris, France
Waikiki, 10 rue d"Ulm, 75005 Paris, France
l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France
Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France
Pari's Café, 174 avenue de Clichy, 75017 Paris, France
Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
Café Varenne, 36 rue de Varenne, 75007 Paris, France
Melting Pot, 3 rue de Lagny, 75020 Paris, France
le Zango, 58 rue Daguerre, 75014 Paris, France
Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France
Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
Brasserie le Morvan, 61 rue du château d'eau, 75010 Paris, France
L'Angle, 28 rue de Ponthieu, 75008 Paris, France
Café Dupont, 198 rue de la Convention, 75015 Paris, France
L'Entracte, place de l'opera, 75002 Paris, France
Panem, 18 rue de Crussol, 75011 Paris, France
Au pays de Vannes, 34 bis rue de Wattignies, 75012 Paris, France
Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France
L'Angle, 28 rue de Ponthieu, 75008 Paris, France
Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France
Café Dupont, 198 rue de la Convention, 75015 Paris, France
L'âge d'or, 26 rue du Docteur Magnan, 75013 Paris, France
Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France
L'horizon, 93, rue de la Roquette, 75011 Paris, France
L'horizon, 93, rue de la Roquette, 75011 Paris, France
L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
Café Victor, 10 boulevard Victor, 75015 Paris, France
Le Brigadier, 12 rue Blanche, 75009 Paris, France
Waikiki, 10 rue d"Ulm, 75005 Paris, France

BIN
bonobo/examples/datasets/spam.tgz Normal file
View File

Binary file not shown.

4
bonobo/examples/files/csv_handlers.py
View File

@ -2,8 +2,8 @@ import bonobo
from bonobo.commands.run import get_default_services
graph = bonobo.Graph(
bonobo.CsvReader('datasets/coffeeshops.txt'),
print,
bonobo.CsvReader('datasets/coffeeshops.txt', headers=('item', )),
bonobo.PrettyPrinter(),
)
if __name__ == '__main__':

7
bonobo/examples/files/json_handlers.py
View File

@ -1,15 +1,16 @@
import bonobo
from bonobo import Bag
from bonobo.commands.run import get_default_services
def get_fields(row):
return row['fields']
def get_fields(**row):
return Bag(**row['fields'])
graph = bonobo.Graph(
bonobo.JsonReader('datasets/theaters.json'),
get_fields,
bonobo.PrettyPrint(title_keys=('eq_nom_equipement', )),
bonobo.PrettyPrinter(),
)
if __name__ == '__main__':

59
bonobo/examples/files/pickle_handlers.py Normal file
View File

@ -0,0 +1,59 @@
'''
This example shows how a different file system service can be injected
into a transformation (as compressing pickled objects often makes sense
anyways). The pickle itself contains a list of lists as follows:
```
[
['category', 'sms'],
['ham', 'Go until jurong point, crazy..'],
['ham', 'Ok lar... Joking wif u oni...'],
['spam', 'Free entry in 2 a wkly comp to win...'],
['ham', 'U dun say so early hor... U c already then say...'],
['ham', 'Nah I don't think he goes to usf, he lives around here though'],
['spam', 'FreeMsg Hey there darling it's been 3 week's now...'],
...
]
```
where the first column categorizes and sms as "ham" or "spam". The second
column contains the sms itself.
Data set taken from:
https://www.kaggle.com/uciml/sms-spam-collection-dataset/downloads/sms-spam-collection-dataset.zip
The transformation (1) reads the pickled data, (2) marks and shortens
messages categorized as spam, and (3) prints the output.
'''
import bonobo
from bonobo.commands.run import get_default_services
from fs.tarfs import TarFS
def cleanse_sms(**row):
if row['category'] == 'spam':
row['sms_clean'] = '**MARKED AS SPAM** ' + row['sms'][0:50] + (
'...' if len(row['sms']) > 50 else ''
)
else:
row['sms_clean'] = row['sms']
return row['sms_clean']
graph = bonobo.Graph(
# spam.pkl is within the gzipped tarball
bonobo.PickleReader('spam.pkl'),
cleanse_sms,
bonobo.PrettyPrinter(),
)
def get_services():
return {'fs': TarFS(bonobo.get_examples_path('datasets/spam.tgz'))}
if __name__ == '__main__':
bonobo.run(graph, services=get_default_services(__file__))

16
bonobo/examples/nodes/slow.py Normal file
View File

@ -0,0 +1,16 @@
import bonobo
import time
from bonobo.constants import NOT_MODIFIED
def pause(*args, **kwargs):
time.sleep(0.1)
return NOT_MODIFIED
graph = bonobo.Graph(
lambda: tuple(range(20)),
pause,
print,
)

23
bonobo/examples/tutorials/tut01e01.py Normal file
View File

@ -0,0 +1,23 @@
import bonobo
def extract():
yield 'foo'
yield 'bar'
yield 'baz'
def transform(x):
return x.upper()
def load(x):
print(x)
graph = bonobo.Graph(extract, transform, load)
graph.__doc__ = 'hello'
if __name__ == '__main__':
bonobo.run(graph)

14
bonobo/examples/tutorials/tut01e02.py Normal file
View File

@ -0,0 +1,14 @@
import bonobo
graph = bonobo.Graph(
[
'foo',
'bar',
'baz',
],
str.upper,
print,
)
if __name__ == '__main__':
bonobo.run(graph)

11
bonobo/examples/tutorials/tut02_01_read.py
View File

@ -1,11 +0,0 @@
import bonobo
graph = bonobo.Graph(
bonobo.FileReader('coffeeshops.txt'),
print,
)
if __name__ == '__main__':
bonobo.run(
graph, services={'fs': bonobo.open_examples_fs('datasets')}
)

17
bonobo/examples/tutorials/tut02_02_write.py
View File

@ -1,17 +0,0 @@
import bonobo
def split_one(line):
return line.split(', ', 1)
graph = bonobo.Graph(
bonobo.FileReader('coffeeshops.txt'),
split_one,
bonobo.JsonWriter('coffeeshops.json'),
)
if __name__ == '__main__':
bonobo.run(
graph, services={'fs': bonobo.open_examples_fs('datasets')}
)

14
bonobo/examples/tutorials/tut02e01_read.py Normal file
View File

@ -0,0 +1,14 @@
import bonobo
graph = bonobo.Graph(
bonobo.FileReader('coffeeshops.txt'),
print,
)
def get_services():
return {'fs': bonobo.open_examples_fs('datasets')}
if __name__ == '__main__':
bonobo.run(graph, services=get_services())

25
bonobo/examples/tutorials/tut02e02_write.py Normal file
View File

@ -0,0 +1,25 @@
import bonobo
def split_one(line):
return line.split(', ', 1)
graph = bonobo.Graph(
bonobo.FileReader('coffeeshops.txt'),
split_one,
bonobo.JsonWriter(
'coffeeshops.json', fs='fs.output', ioformat='arg0'
),
)
def get_services():
return {
'fs': bonobo.open_examples_fs('datasets'),
'fs.output': bonobo.open_fs(),
}
if __name__ == '__main__':
bonobo.run(graph, services=get_services())

18
bonobo/examples/tutorials/tut02_03_writeasmap.py → bonobo/examples/tutorials/tut02e03_writeasmap.py
View File

@ -1,4 +1,6 @@
import bonobo, json
import json
import bonobo
def split_one_to_map(line):
@ -18,10 +20,16 @@ class MyJsonWriter(bonobo.JsonWriter):
graph = bonobo.Graph(
bonobo.FileReader('coffeeshops.txt'),
split_one_to_map,
MyJsonWriter('coffeeshops.json'),
MyJsonWriter('coffeeshops.json', fs='fs.output', ioformat='arg0'),
)
def get_services():
return {
'fs': bonobo.open_examples_fs('datasets'),
'fs.output': bonobo.open_fs(),
}
if __name__ == '__main__':
bonobo.run(
graph, services={'fs': bonobo.open_examples_fs('datasets')}
)
bonobo.run(graph, services=get_services())

3
bonobo/examples/types/__main__.py Normal file
View File

@ -0,0 +1,3 @@
from bonobo.util.python import require
graph = require('strings').graph

22
bonobo/execution/base.py
View File

@ -9,6 +9,14 @@ from bonobo.util.errors import print_error
from bonobo.util.objects import Wrapper, get_name
@contextmanager
def recoverable(error_handler):
try:
yield
except Exception as exc: # pylint: disable=broad-except
error_handler(exc, traceback.format_exc())
@contextmanager
def unrecoverable(error_handler):
try:
@ -50,15 +58,21 @@ class LoopingExecutionContext(Wrapper):
# XXX enhancers
self._enhancers = get_enhancers(self.wrapped)
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
self.stop()
def start(self):
if self.started:
raise RuntimeError('Cannot start a node twice ({}).'.format(get_name(self)))
self._started = True
self._stack = ContextCurrifier(self.wrapped, *self._get_initial_context())
with unrecoverable(self.handle_error):
self._stack.setup(self)
self._stack = ContextCurrifier(self.wrapped, *self._get_initial_context())
self._stack.setup(self)
for enhancer in self._enhancers:
with unrecoverable(self.handle_error):
@ -82,7 +96,7 @@ class LoopingExecutionContext(Wrapper):
return
try:
with unrecoverable(self.handle_error):
if self._stack:
self._stack.teardown()
finally:
self._stopped = True

19
bonobo/execution/plugin.py
View File

@ -1,6 +1,4 @@
import traceback
from bonobo.execution.base import LoopingExecutionContext
from bonobo.execution.base import LoopingExecutionContext, recoverable
class PluginExecutionContext(LoopingExecutionContext):
@ -14,21 +12,14 @@ class PluginExecutionContext(LoopingExecutionContext):
def start(self):
super().start()
try:
with recoverable(self.handle_error):
self.wrapped.initialize()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
def shutdown(self):
try:
with recoverable(self.handle_error):
self.wrapped.finalize()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
finally:
self.alive = False
self.alive = False
def step(self):
try:
with recoverable(self.handle_error):
self.wrapped.run()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())

70
bonobo/ext/console.py
View File

@ -1,5 +1,6 @@
import functools
import io
import sys
from contextlib import redirect_stdout
from colorama import Style, Fore
@ -8,6 +9,21 @@ from bonobo.plugins import Plugin
from bonobo.util.term import CLEAR_EOL, MOVE_CURSOR_UP
class IOBuffer():
def __init__(self):
self.current = io.StringIO()
self.write = self.current.write
def switch(self):
previous = self.current
self.current = io.StringIO()
self.write = self.current.write
try:
return previous.getvalue()
finally:
previous.close()
class ConsoleOutputPlugin(Plugin):
"""
Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor
@ -21,30 +37,32 @@ class ConsoleOutputPlugin(Plugin):
def initialize(self):
self.prefix = ''
self.counter = 0
self._append_cache = ''
self.isatty = sys.stdout.isatty()
def _write(self, graph_context, rewind):
if settings.PROFILE:
append = (
('Memory', '{0:.2f} Mb'.format(memory_usage())),
# ('Total time', '{0} s'.format(execution_time(harness))),
)
else:
append = ()
self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind)
self._stdout = sys.stdout
self.stdout = IOBuffer()
self.redirect_stdout = redirect_stdout(self.stdout)
self.redirect_stdout.__enter__()
def run(self):
if sys.stdout.isatty():
if self.isatty:
self._write(self.context.parent, rewind=True)
else:
pass # not a tty
def finalize(self):
self._write(self.context.parent, rewind=False)
self.redirect_stdout.__exit__(None, None, None)
@staticmethod
def write(context, prefix='', rewind=True, append=None):
def write(self, context, prefix='', rewind=True, append=None):
t_cnt = len(context)
buffered = self.stdout.switch()
for line in buffered.split('\n')[:-1]:
print(line + CLEAR_EOL, file=sys.stderr)
for i in context.graph.topologically_sorted_indexes:
node = context[i]
name_suffix = '({})'.format(i) if settings.DEBUG else ''
@ -62,7 +80,7 @@ class ConsoleOutputPlugin(Plugin):
Style.RESET_ALL, ' ',
)
)
print(prefix + _line + '\033[0K')
print(prefix + _line + '\033[0K', file=sys.stderr)
if append:
# todo handle multiline
@ -72,17 +90,31 @@ class ConsoleOutputPlugin(Plugin):
' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v)
for k, v in append), CLEAR_EOL
)
)
),
file=sys.stderr
)
t_cnt += 1
if rewind:
print(CLEAR_EOL)
print(MOVE_CURSOR_UP(t_cnt + 2))
print(CLEAR_EOL, file=sys.stderr)
print(MOVE_CURSOR_UP(t_cnt + 2), file=sys.stderr)
def _write(self, graph_context, rewind):
if settings.PROFILE:
if self.counter % 10 and self._append_cache:
append = self._append_cache
else:
self._append_cache = append = (
('Memory', '{0:.2f} Mb'.format(memory_usage())),
# ('Total time', '{0} s'.format(execution_time(harness))),
)
else:
append = ()
self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind)
self.counter += 1
@functools.lru_cache(1)
def memory_usage():
import os, psutil
process = psutil.Process(os.getpid())
return process.memory_info()[0] / float(2**20)
return process.memory_info()[0] / float(2**20)

22
bonobo/ext/edgy/project/feature.py
View File

@ -1,22 +0,0 @@
try:
import edgy.project
except ImportError as e:
import logging
logging.exception('You must install edgy.project to use this.')
import os
from edgy.project.events import subscribe
from edgy.project.feature import Feature, SUPPORT_PRIORITY
class BonoboFeature(Feature):
requires = {'python'}
@subscribe('edgy.project.on_start', priority=SUPPORT_PRIORITY)
def on_start(self, event):
package_path = event.setup['name'].replace('.', os.sep)
for file in ('example_graph'):
self.render_file(os.path.join(package_path, file + '.py'), os.path.join('tornado', file + '.py.j2'))

21
bonobo/ext/jupyter/js/dist/index.js vendored
View File

@ -42,7 +42,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
/************************************************************************/
/******/ ([
/* 0 */
/***/ function(module, exports, __webpack_require__) {
/***/ (function(module, exports, __webpack_require__) {
// Entry point for the unpkg bundle containing custom model definitions.
//
@ -55,14 +55,13 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
module.exports['version'] = __webpack_require__(4).version;
/***/ },
/***/ }),
/* 1 */
/***/ function(module, exports, __webpack_require__) {
/***/ (function(module, exports, __webpack_require__) {
var widgets = __webpack_require__(2);
var _ = __webpack_require__(3);
// Custom Model. Custom widgets models must at least provide default values
// for model attributes, including `_model_name`, `_view_name`, `_model_module`
// and `_view_module` when different from the base class.
@ -102,15 +101,15 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
};
/***/ },
/***/ }),
/* 2 */
/***/ function(module, exports) {
/***/ (function(module, exports) {
module.exports = __WEBPACK_EXTERNAL_MODULE_2__;
/***/ },
/***/ }),
/* 3 */
/***/ function(module, exports, __webpack_require__) {
/***/ (function(module, exports, __webpack_require__) {
var __WEBPACK_AMD_DEFINE_ARRAY__, __WEBPACK_AMD_DEFINE_RESULT__;// Underscore.js 1.8.3
// http://underscorejs.org
@ -1662,9 +1661,9 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
}.call(this));
/***/ },
/***/ }),
/* 4 */
/***/ function(module, exports) {
/***/ (function(module, exports) {
module.exports = {
"name": "bonobo-jupyter",
@ -1696,6 +1695,6 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
}
};
/***/ }
/***/ })
/******/ ])});;
//# sourceMappingURL=index.js.map

2
bonobo/ext/jupyter/js/dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

1
bonobo/ext/jupyter/js/src/bonobo.js
View File

@ -1,7 +1,6 @@
var widgets = require('jupyter-js-widgets');
var _ = require('underscore');
// Custom Model. Custom widgets models must at least provide default values
// for model attributes, including `_model_name`, `_view_name`, `_model_module`
// and `_view_module` when different from the base class.

1441
bonobo/ext/jupyter/js/yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff

4
bonobo/ext/jupyter/plugin.py
View File

@ -19,6 +19,8 @@ class JupyterOutputPlugin(Plugin):
IPython.core.display.display(self.widget)
def run(self):
self.widget.value = [repr(node) for node in self.context.parent.nodes]
self.widget.value = [
str(self.context.parent[i]) for i in self.context.parent.graph.topologically_sorted_indexes
]
finalize = run

4
bonobo/ext/jupyter/static/extension.js
View File

@ -42,7 +42,7 @@ define(function() { return /******/ (function(modules) { // webpackBootstrap
/************************************************************************/
/******/ ([
/* 0 */
/***/ function(module, exports) {
/***/ (function(module, exports) {
// This file contains the javascript that is run when the notebook is loaded.
// It contains some requirejs configuration and the `load_ipython_extension`
@ -66,5 +66,5 @@ define(function() { return /******/ (function(modules) { // webpackBootstrap
};
/***/ }
/***/ })
/******/ ])});;

21
bonobo/ext/jupyter/static/index.js
View File

@ -42,7 +42,7 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
/************************************************************************/
/******/ ([
/* 0 */
/***/ function(module, exports, __webpack_require__) {
/***/ (function(module, exports, __webpack_require__) {
// Entry point for the notebook bundle containing custom model definitions.
//
@ -58,14 +58,13 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
module.exports['version'] = __webpack_require__(4).version;
/***/ },
/***/ }),
/* 1 */
/***/ function(module, exports, __webpack_require__) {
/***/ (function(module, exports, __webpack_require__) {
var widgets = __webpack_require__(2);
var _ = __webpack_require__(3);
// Custom Model. Custom widgets models must at least provide default values
// for model attributes, including `_model_name`, `_view_name`, `_model_module`
// and `_view_module` when different from the base class.
@ -105,15 +104,15 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
};
/***/ },
/***/ }),
/* 2 */
/***/ function(module, exports) {
/***/ (function(module, exports) {
module.exports = __WEBPACK_EXTERNAL_MODULE_2__;
/***/ },
/***/ }),
/* 3 */
/***/ function(module, exports, __webpack_require__) {
/***/ (function(module, exports, __webpack_require__) {
var __WEBPACK_AMD_DEFINE_ARRAY__, __WEBPACK_AMD_DEFINE_RESULT__;// Underscore.js 1.8.3
// http://underscorejs.org
@ -1665,9 +1664,9 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
}.call(this));
/***/ },
/***/ }),
/* 4 */
/***/ function(module, exports) {
/***/ (function(module, exports) {
module.exports = {
"name": "bonobo-jupyter",
@ -1699,6 +1698,6 @@ define(["jupyter-js-widgets"], function(__WEBPACK_EXTERNAL_MODULE_2__) { return
}
};
/***/ }
/***/ })
/******/ ])});;
//# sourceMappingURL=index.js.map

2
bonobo/ext/jupyter/static/index.js.map
View File

File diff suppressed because one or more lines are too long

2
bonobo/ext/jupyter/widget.py
View File

@ -2,7 +2,7 @@ import ipywidgets as widgets
from traitlets import List, Unicode
@widgets.register('bonobo-widget.Bonobo')
@widgets.register('bonobo-widget.bonobo')
class BonoboWidget(widgets.DOMWidget):
_view_name = Unicode('BonoboView').tag(sync=True)
_model_name = Unicode('BonoboModel').tag(sync=True)

78
bonobo/logging.py Normal file
View File

@ -0,0 +1,78 @@
import logging
import sys
import textwrap
from logging import CRITICAL, DEBUG, ERROR, INFO, WARNING
from colorama import Fore, Style
from bonobo import settings
from bonobo.util.term import CLEAR_EOL
def get_format():
yield '{b}[%(fg)s%(levelname)s{b}][{w}'
yield '{b}][{w}'.join(('%(spent)04d', '%(name)s'))
yield '{b}]'
yield ' %(fg)s%(message)s{r}'
yield CLEAR_EOL
colors = {
'b': Fore.BLACK,
'w': Fore.LIGHTBLACK_EX,
'r': Style.RESET_ALL,
}
format = (''.join(get_format())).format(**colors)
class Filter(logging.Filter):
def filter(self, record):
record.spent = record.relativeCreated // 1000
if record.levelname == 'DEBG':
record.fg = Fore.LIGHTBLACK_EX
elif record.levelname == 'INFO':
record.fg = Fore.LIGHTWHITE_EX
elif record.levelname == 'WARN':
record.fg = Fore.LIGHTYELLOW_EX
elif record.levelname == 'ERR ':
record.fg = Fore.LIGHTRED_EX
elif record.levelname == 'CRIT':
record.fg = Fore.RED
else:
record.fg = Fore.LIGHTWHITE_EX
return True
class Formatter(logging.Formatter):
def formatException(self, ei):
tb = super().formatException(ei)
return textwrap.indent(tb, Fore.BLACK + ' | ' + Fore.WHITE)
def setup(level):
logging.addLevelName(DEBUG, 'DEBG')
logging.addLevelName(INFO, 'INFO')
logging.addLevelName(WARNING, 'WARN')
logging.addLevelName(ERROR, 'ERR ')
logging.addLevelName(CRITICAL, 'CRIT')
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(Formatter(format))
handler.addFilter(Filter())
root = logging.getLogger()
root.addHandler(handler)
root.setLevel(level)
def set_level(level):
logging.getLogger().setLevel(level)
def get_logger(name='bonobo'):
return logging.getLogger(name)
# Compatibility with python logging
getLogger = get_logger
# Setup formating and level.
setup(level=settings.LOGGING_LEVEL)

26
bonobo/nodes/basics.py
View File

@ -1,9 +1,12 @@
import functools
from pprint import pprint as _pprint
import itertools
from colorama import Fore, Style
from bonobo.config import Configurable, ContextProcessor, Option
from bonobo import settings
from bonobo.config import Configurable, Option
from bonobo.config.processors import ContextProcessor
from bonobo.structs.bags import Bag
from bonobo.util.objects import ValueHolder
from bonobo.util.term import CLEAR_EOL
@ -15,7 +18,7 @@ __all__ = [
'Tee',
'count',
'pprint',
'PrettyPrint',
'PrettyPrinter',
'noop',
]
@ -68,7 +71,24 @@ def _count_counter(self, context):
context.send(Bag(counter._value))
pprint = Tee(_pprint)
class PrettyPrinter(Configurable):
def call(self, *args, **kwargs):
formater = self._format_quiet if settings.QUIET else self._format_console
for i, (item, value) in enumerate(itertools.chain(enumerate(args), kwargs.items())):
print(formater(i, item, value))
def _format_quiet(self, i, item, value):
return ' '.join(((' ' if i else '-'), str(item), ':', str(value).strip()))
def _format_console(self, i, item, value):
return ' '.join(
((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL)
)
pprint = PrettyPrinter()
pprint.__name__ = 'pprint'
def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):

3
bonobo/nodes/io/__init__.py
View File

@ -3,6 +3,7 @@
from .file import FileReader, FileWriter
from .json import JsonReader, JsonWriter
from .csv import CsvReader, CsvWriter
from .pickle import PickleReader, PickleWriter
__all__ = [
'CsvReader',
@ -11,4 +12,6 @@ __all__ = [
'FileWriter',
'JsonReader',
'JsonWriter',
'PickleReader',
'PickleWriter',
]

82
bonobo/nodes/io/base.py Normal file
View File

@ -0,0 +1,82 @@
from bonobo import settings
from bonobo.config import Configurable, ContextProcessor, Option, Service
from bonobo.structs.bags import Bag
class IOFormatEnabled(Configurable):
ioformat = Option(default=settings.IOFORMAT.get)
def get_input(self, *args, **kwargs):
if self.ioformat == settings.IOFORMAT_ARG0:
if len(args) != 1 or len(kwargs):
raise ValueError(
'Wrong input formating: IOFORMAT=ARG0 implies one arg and no kwargs, got args={!r} and kwargs={!r}.'.
format(args, kwargs)
)
return args[0]
if self.ioformat == settings.IOFORMAT_KWARGS:
if len(args) or not len(kwargs):
raise ValueError(
'Wrong input formating: IOFORMAT=KWARGS ioformat implies no arg, got args={!r} and kwargs={!r}.'.
format(args, kwargs)
)
return kwargs
raise NotImplementedError('Unsupported format.')
def get_output(self, row):
if self.ioformat == settings.IOFORMAT_ARG0:
return row
if self.ioformat == settings.IOFORMAT_KWARGS:
return Bag(**row)
raise NotImplementedError('Unsupported format.')
class FileHandler(Configurable):
"""Abstract component factory for file-related components.
Args:
path (str): which path to use within the provided filesystem.
eol (str): which character to use to separate lines.
mode (str): which mode to use when opening the file.
fs (str): service name to use for filesystem.
"""
path = Option(str, required=True, positional=True) # type: str
eol = Option(str, default='\n') # type: str
mode = Option(str) # type: str
encoding = Option(str, default='utf-8') # type: str
fs = Service('fs') # type: str
@ContextProcessor
def file(self, context, fs):
with self.open(fs) as file:
yield file
def open(self, fs):
return fs.open(self.path, self.mode, encoding=self.encoding)
class Reader:
"""Abstract component factory for readers.
"""
def __call__(self, *args, **kwargs):
yield from self.read(*args, **kwargs)
def read(self, *args, **kwargs):
raise NotImplementedError('Abstract.')
class Writer:
"""Abstract component factory for writers.
"""
def __call__(self, *args, **kwargs):
return self.write(*args, **kwargs)
def write(self, *args, **kwargs):
raise NotImplementedError('Abstract.')

13
bonobo/nodes/io/csv.py
View File

@ -3,8 +3,9 @@ import csv
from bonobo.config import Option
from bonobo.config.processors import ContextProcessor
from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.file import FileReader, FileWriter
from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
from bonobo.util.objects import ValueHolder
from .file import FileHandler, FileReader, FileWriter
class CsvHandler(FileHandler):
@ -28,7 +29,7 @@ class CsvHandler(FileHandler):
headers = Option(tuple)
class CsvReader(CsvHandler, FileReader):
class CsvReader(IOFormatEnabled, FileReader, CsvHandler):
"""
Reads a CSV and yield the values as dicts.
@ -49,6 +50,7 @@ class CsvReader(CsvHandler, FileReader):
if not headers.get():
headers.set(next(reader))
_headers = headers.get()
field_count = len(headers)
@ -60,17 +62,18 @@ class CsvReader(CsvHandler, FileReader):
if len(row) != field_count:
raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count, ))
yield dict(zip(headers.value, row))
yield self.get_output(dict(zip(_headers, row)))
class CsvWriter(CsvHandler, FileWriter):
class CsvWriter(IOFormatEnabled, FileWriter, CsvHandler):
@ContextProcessor
def writer(self, context, fs, file, lineno):
writer = csv.writer(file, delimiter=self.delimiter, quotechar=self.quotechar, lineterminator=self.eol)
headers = ValueHolder(list(self.headers) if self.headers else None)
yield writer, headers
def write(self, fs, file, lineno, writer, headers, row):
def write(self, fs, file, lineno, writer, headers, *args, **kwargs):
row = self.get_input(*args, **kwargs)
if not lineno:
headers.set(headers.value or row.keys())
writer.writerow(headers.get())

64
bonobo/nodes/io/file.py
View File

@ -1,63 +1,11 @@
from bonobo.config import Option, Service
from bonobo.config.configurables import Configurable
from bonobo.config import Option
from bonobo.config.processors import ContextProcessor
from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.base import FileHandler, Reader, Writer
from bonobo.util.objects import ValueHolder
__all__ = [
'FileReader',
'FileWriter',
]
class FileHandler(Configurable):
"""Abstract component factory for file-related components.
Args:
path (str): which path to use within the provided filesystem.
eol (str): which character to use to separate lines.
mode (str): which mode to use when opening the file.
fs (str): service name to use for filesystem.
"""
path = Option(str, required=True, positional=True) # type: str
eol = Option(str, default='\n') # type: str
mode = Option(str) # type: str
fs = Service('fs') # type: str
@ContextProcessor
def file(self, context, fs):
with self.open(fs) as file:
yield file
def open(self, fs):
return fs.open(self.path, self.mode)
class Reader(FileHandler):
"""Abstract component factory for readers.
"""
def __call__(self, *args):
yield from self.read(*args)
def read(self, *args):
raise NotImplementedError('Abstract.')
class Writer(FileHandler):
"""Abstract component factory for writers.
"""
def __call__(self, *args):
return self.write(*args)
def write(self, *args):
raise NotImplementedError('Abstract.')
class FileReader(Reader):
class FileReader(Reader, FileHandler):
"""Component factory for file-like readers.
On its own, it can be used to read a file and yield one row per line, trimming the "eol" character at the end if
@ -75,7 +23,7 @@ class FileReader(Reader):
yield line.rstrip(self.eol)
class FileWriter(Writer):
class FileWriter(Writer, FileHandler):
"""Component factory for file or file-like writers.
On its own, it can be used to write in a file one line per row that comes into this component. Extending it is
@ -89,11 +37,11 @@ class FileWriter(Writer):
lineno = ValueHolder(0)
yield lineno
def write(self, fs, file, lineno, row):
def write(self, fs, file, lineno, line):
"""
Write a row on the next line of opened file in context.
"""
self._write_line(file, (self.eol if lineno.value else '') + row)
self._write_line(file, (self.eol if lineno.value else '') + line)
lineno += 1
return NOT_MODIFIED

30
bonobo/nodes/io/json.py
View File

@ -3,45 +3,39 @@ from itertools import starmap
from bonobo.structs.bags import Bag
from bonobo.config.processors import ContextProcessor
from .file import FileWriter, FileReader
__all__ = [
'JsonWriter',
]
from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
from bonobo.nodes.io.file import FileReader, FileWriter
class JsonHandler():
class JsonHandler(FileHandler):
eol = ',\n'
prefix, suffix = '[', ']'
class JsonReader(JsonHandler, FileReader):
class JsonReader(IOFormatEnabled, FileReader, JsonHandler):
loader = staticmethod(json.load)
def read(self, fs, file):
for line in self.loader(file):
yield line
yield self.get_output(line)
class JsonDictReader(JsonReader):
""" not api, don't use or expect breakage. """
def read(self, fs, file):
yield from starmap(Bag, self.loader(file).items())
class JsonWriter(JsonHandler, FileWriter):
class JsonWriter(IOFormatEnabled, FileWriter, JsonHandler):
@ContextProcessor
def envelope(self, context, fs, file, lineno):
file.write(self.prefix)
yield
file.write(self.suffix)
def write(self, fs, file, lineno, row):
def write(self, fs, file, lineno, *args, **kwargs):
"""
Write a json row on the next line of file pointed by ctx.file.
:param ctx:
:param row:
"""
return super().write(fs, file, lineno, json.dumps(row))
row = self.get_input(*args, **kwargs)
self._write_line(file, (self.eol if lineno.value else '') + json.dumps(row))
lineno += 1
return NOT_MODIFIED

69
bonobo/nodes/io/pickle.py Normal file
View File

@ -0,0 +1,69 @@
import pickle
from bonobo.config import Option
from bonobo.config.processors import ContextProcessor
from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
from bonobo.nodes.io.file import FileReader, FileWriter
from bonobo.util.objects import ValueHolder
class PickleHandler(FileHandler):
"""
.. attribute:: item_names
The names of the items in the pickle, if it is not defined in the first item of the pickle.
"""
item_names = Option(tuple)
class PickleReader(IOFormatEnabled, FileReader, PickleHandler):
"""
Reads a Python pickle object and yields the items in dicts.
"""
mode = Option(str, default='rb')
@ContextProcessor
def pickle_headers(self, context, fs, file):
yield ValueHolder(self.item_names)
def read(self, fs, file, pickle_headers):
data = pickle.load(file)
# if the data is not iterable, then wrap the object in a list so it may be iterated
if isinstance(data, dict):
is_dict = True
iterator = iter(data.items())
else:
is_dict = False
try:
iterator = iter(data)
except TypeError:
iterator = iter([data])
if not pickle_headers.get():
pickle_headers.set(next(iterator))
item_count = len(pickle_headers.value)
for i in iterator:
if len(i) != item_count:
raise ValueError('Received an object with %d items, expecting %d.' % (len(i), item_count, ))
yield self.get_output(dict(zip(i)) if is_dict else dict(zip(pickle_headers.value, i)))
class PickleWriter(IOFormatEnabled, FileWriter, PickleHandler):
mode = Option(str, default='wb')
def write(self, fs, file, lineno, item):
"""
Write a pickled item to the opened file.
"""
file.write(pickle.dumps(item))
lineno += 1
return NOT_MODIFIED

0
bonobo/ext/edgy/__init__.py → bonobo/nodes/io/xml.py
View File

61
bonobo/settings.py
View File

@ -1,5 +1,8 @@
import logging
import os
from bonobo.errors import ValidationError
def to_bool(s):
if len(s):
@ -9,8 +12,60 @@ def to_bool(s):
return False
# Debug mode.
DEBUG = to_bool(os.environ.get('BONOBO_DEBUG', 'f'))
class Setting:
def __init__(self, name, default=None, validator=None):
self.name = name
if default:
self.default = default if callable(default) else lambda: default
else:
self.default = lambda: None
if validator:
self.validator = validator
else:
self.validator = None
def __repr__(self):
return '<Setting {}={!r}>'.format(self.name, self.value)
def set(self, value):
if self.validator and not self.validator(value):
raise ValidationError('Invalid value {!r} for setting {}.'.format(value, self.name))
self.value = value
def get(self):
try:
return self.value
except AttributeError:
self.value = self.default()
return self.value
# Debug/verbose mode.
DEBUG = to_bool(os.environ.get('DEBUG', 'f'))
# Profile mode.
PROFILE = to_bool(os.environ.get('BONOBO_PROFILE', 'f'))
PROFILE = to_bool(os.environ.get('PROFILE', 'f'))
# Quiet mode.
QUIET = to_bool(os.environ.get('QUIET', 'f'))
# Logging level.
LOGGING_LEVEL = logging.DEBUG if DEBUG else logging.INFO
# Input/Output format for transformations
IOFORMAT_ARG0 = 'arg0'
IOFORMAT_KWARGS = 'kwargs'
IOFORMATS = {
IOFORMAT_ARG0,
IOFORMAT_KWARGS,
}
IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__)
def check():
if DEBUG and QUIET:
raise RuntimeError('I cannot be verbose and quiet at the same time.')

18
bonobo/strategies/executor.py
View File

@ -1,6 +1,5 @@
import time
import traceback
from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
from bonobo.constants import BEGIN, END
@ -31,12 +30,11 @@ class ExecutorStrategy(Strategy):
for plugin_context in context.plugins:
def _runner(plugin_context=plugin_context):
try:
plugin_context.start()
plugin_context.loop()
plugin_context.stop()
except Exception as exc:
print_error(exc, traceback.format_exc(), prefix='Error in plugin context', context=plugin_context)
with plugin_context:
try:
plugin_context.loop()
except Exception as exc:
print_error(exc, traceback.format_exc(), context=plugin_context)
futures.append(executor.submit(_runner))
@ -46,9 +44,7 @@ class ExecutorStrategy(Strategy):
try:
node_context.start()
except Exception as exc:
print_error(
exc, traceback.format_exc(), prefix='Could not start node context', context=node_context
)
print_error(exc, traceback.format_exc(), context=node_context, method='start')
node_context.input.on_end()
else:
node_context.loop()
@ -56,7 +52,7 @@ class ExecutorStrategy(Strategy):
try:
node_context.stop()
except Exception as exc:
print_error(exc, traceback.format_exc(), prefix='Could not stop node context', context=node_context)
print_error(exc, traceback.format_exc(), context=node_context, method='stop')
futures.append(executor.submit(_runner))

8
bonobo/structs/bags.py
View File

@ -75,6 +75,14 @@ class Bag:
raise TypeError('Could not apply bag to {}.'.format(func_or_iter))
def get(self):
"""
Get a 2 element tuple of this bag's args and kwargs.
:return: tuple
"""
return self.args, self.kwargs
def extend(self, *args, **kwargs):
return type(self)(*args, _parent=self, **kwargs)

25
bonobo/util/errors.py
View File

@ -1,5 +1,7 @@
import sys
from textwrap import indent
from bonobo import settings
from bonobo.structs.bags import ErrorBag
@ -7,7 +9,14 @@ def is_error(bag):
return isinstance(bag, ErrorBag)
def print_error(exc, trace, context=None, prefix=''):
def _get_error_message(exc):
if hasattr(exc, '__str__'):
message = str(exc)
return message[0].upper() + message[1:]
return '\n'.join(exc.args),
def print_error(exc, trace, context=None, method=None):
"""
Error handler. Whatever happens in a plugin or component, if it looks like an exception, taste like an exception
or somehow make me think it is an exception, I'll handle it.
@ -18,14 +27,20 @@ def print_error(exc, trace, context=None, prefix=''):
"""
from colorama import Fore, Style
prefix = '{}{} | {}'.format(Fore.RED, Style.BRIGHT, Style.RESET_ALL)
print(
Style.BRIGHT,
Fore.RED,
'\U0001F4A3 {}{}{}'.format(
(prefix + ': ') if prefix else '', type(exc).__name__, ' in {!r}'.format(context) if context else ''
),
type(exc).__name__,
' (in {}{})'.format(type(context).__name__, '.{}()'.format(method) if method else '') if context else '',
Style.RESET_ALL,
'\n',
indent(_get_error_message(exc), prefix + Style.BRIGHT),
Style.RESET_ALL,
sep='',
file=sys.stderr,
)
print(trace)
print(prefix, file=sys.stderr)
print(indent(trace, prefix, predicate=lambda line: True), file=sys.stderr)

14
bonobo/util/iterators.py
View File

@ -1,4 +1,5 @@
""" Iterator utilities. """
import functools
def force_iterator(mixed):
@ -23,6 +24,19 @@ def ensure_tuple(tuple_or_mixed):
return (tuple_or_mixed, )
def tuplize(generator):
""" Takes a generator and make it a tuple-returning function. As a side
effect, it can also decorate any iterator-returning function to force
return value to be a tuple.
"""
@functools.wraps(generator)
def tuplized(*args, **kwargs):
return tuple(generator(*args, **kwargs))
return tuplized
def iter_if_not_sequence(mixed):
if isinstance(mixed, (dict, list, str)):
raise TypeError(type(mixed).__name__)

8
bonobo/util/pkgs.py Normal file
View File

@ -0,0 +1,8 @@
import pkg_resources
from packaging.utils import canonicalize_name
bonobo_packages = {}
for p in pkg_resources.working_set:
name = canonicalize_name(p.project_name)
if name.startswith('bonobo'):
bonobo_packages[name] = p

28
bonobo/util/testing.py
View File

@ -1,5 +1,7 @@
from contextlib import contextmanager
from unittest.mock import MagicMock
from bonobo import open_fs
from bonobo.execution.node import NodeExecutionContext
@ -7,3 +9,29 @@ class CapturingNodeExecutionContext(NodeExecutionContext):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.send = MagicMock()
@contextmanager
def optional_contextmanager(cm, *, ignore=False):
if cm is None or ignore:
yield
else:
with cm:
yield
class FilesystemTester:
def __init__(self, extension='txt', mode='w'):
self.extension = extension
self.input_data = ''
self.mode = mode
def get_services_for_reader(self, tmpdir):
fs, filename = open_fs(tmpdir), 'input.' + self.extension
with fs.open(filename, self.mode) as fp:
fp.write(self.input_data)
return fs, filename, {'fs': fs}
def get_services_for_writer(self, tmpdir):
fs, filename = open_fs(tmpdir), 'output.' + self.extension
return fs, filename, {'fs': fs}

207
docs/_templates/index.html vendored
View File

@ -2,105 +2,116 @@
{% set title = _('Bonobo — Data processing for humans') %}
{% block body %}
<div style="border: 2px solid red; font-weight: bold; margin: 1em; padding: 1em">
Bonobo is <strong>ALPHA</strong> software. Some APIs will change.
</div>
<h1 style="text-align: center">
<img class="logo" src="{{ pathto('_static/bonobo.png', 1) }}" title="Bonobo" alt="Bonobo"
style=" width: 128px; height: 128px;"/>
</h1>
<h1 style="text-align: center">
<img class="logo" src="{{ pathto('_static/bonobo.png', 1) }}" title="Bonobo" alt="Bonobo"
style=" width: 128px; height: 128px;"/>
</h1>
<p>
{% trans %}
<strong>Bonobo</strong> is a line-by-line data-processing toolkit for python 3.5+ (extract-transform-load
framework) emphasizing simple and atomic data transformations defined using a directed graph of plain old
python objects (functions, iterables, generators, ...).
{% endtrans %}
</p>
<h2 style="margin-bottom: 0">{% trans %}Documentation{% endtrans %}</h2>
<table class="contentstable">
<tr>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto("tutorial/index") }}">{% trans %}First steps{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}quick overview of basic features{% endtrans %}</span></p>
</td>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto("search") }}">{% trans %}
Search{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}search the documentation{% endtrans %}</span></p>
</td>
</tr>
<tr>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto("guide/index") }}">{% trans %}
Guides{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}for a complete overview{% endtrans %}</span>
</p>
</td>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto("reference/index") }}">{% trans %}References{% endtrans %}</a>
<br/>
<span class="linkdescr">{% trans %}all functions, classes, terms{% endtrans %}</span>
</p>
</td>
</tr>
<tr>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto("changes") }}">{% trans %}
Cookbook{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}examples and recipes{% endtrans %}</span></p>
</td>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto("contribute/index") }}">{% trans %}
Contribute{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}contributor guide{% endtrans %}</span></p>
</td>
</tr>
</table>
<h2>Features</h2>
<ul>
<li>
{% trans %}
<b>10 minutes to get started:</b> Know some python? Writing your first data processor is an affair
of minutes.
{% endtrans %}
</li>
<li>
{% trans %}
<b>Data sources and targets:</b> HTML, JSON, XML, SQL databases, NoSQL databases, HTTP/REST APIs,
streaming APIs, python objects...
{% endtrans %}
</li>
<li>
{% trans %}
<b>Service injection:</b> Abstract the transformation dependencies to easily switch data sources and
dependant libraries. You'll be able to specify the concrete implementations or configurations at
runtime, for example to switch a database connection string or an API endpoint.
{% endtrans %}
</li>
<li>
{% trans %}
<b>Plugins:</b> Easily add features to all your transformations by using builtin plugins (Jupyter,
Console, ...) or write your own.
{% endtrans %}
</li>
<li>
{% trans %}
Bonobo is young, and the todo-list is huge. Read the <a href="https://www.bonobo-project.org/roadmap">roadmap</a>.
{% endtrans %}
</li>
</ul>
<p>{% trans %}
You can also download PDF/EPUB versions of the Bonobo documentation:
<a href="http://readthedocs.org/projects/bonobo/downloads/pdf/stable/">PDF version</a>,
<a href="http://readthedocs.org/projects/bonobo/downloads/epub/stable/">EPUB version</a>.
<p>
{% trans %}
<strong>Bonobo</strong> is a line-by-line data-processing toolkit for python 3.5+ (extract-transform-load
framework, or ETL) emphasizing simple and atomic data transformations defined using a directed graph of plain old
python objects (functions, iterables, generators, ...).
{% endtrans %}
</p>
</p>
<div style="border: 2px solid red; font-weight: bold; margin: 1em; padding: 1em">
Bonobo is <strong>ALPHA</strong> software. Some APIs will change.
</div>
<h2 style="margin-bottom: 0">{% trans %}Documentation{% endtrans %}</h2>
<table class="contentstable">
<tr>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto(" tutorial/index") }}">{% trans %}First steps{%
endtrans %}</a><br/>
<span class="linkdescr">{% trans %}quick overview of basic features{% endtrans %}</span></p>
</td>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto(" search") }}">{% trans %}
Search{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}search the documentation{% endtrans %}</span></p>
</td>
</tr>
<tr>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto(" guide/index") }}">{% trans %}
Guides{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}for a complete overview{% endtrans %}</span>
</p>
</td>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto(" reference/index") }}">{% trans %}References{%
endtrans %}</a>
<br/>
<span class="linkdescr">{% trans %}all functions, classes, terms{% endtrans %}</span>
</p>
</td>
</tr>
<tr>
<td>
<p class="biglink"><a class="biglink" target="_blank"
href="https://github.com/python-bonobo/bonobo/tree/master/bonobo/examples">{% trans %}
Cookbook{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}examples and recipes{% endtrans %}</span></p>
</td>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto(" contribute/index") }}">{% trans %}
Contribute{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}contributor guide{% endtrans %}</span></p>
</td>
</tr>
</table>
<h2>Features</h2>
<ul>
<li>
{% trans %}
<b>10 minutes to get started:</b> Know some python? Writing your first data processor is an affair
of minutes.
{% endtrans %}
</li>
<li>
{% trans %}
<b>Data sources and targets:</b> HTML, JSON, XML, SQL databases, NoSQL databases, HTTP/REST APIs,
streaming APIs, python objects...
{% endtrans %}
</li>
<li>
{% trans %}
<b>Service injection:</b> Abstract the transformation dependencies to easily switch data sources and
dependant libraries. You'll be able to specify the concrete implementations or configurations at
runtime, for example to switch a database connection string or an API endpoint.
{% endtrans %}
</li>
<li>
{% trans %}
<b>Plugins:</b> Easily add features to all your transformations by using builtin plugins (Jupyter,
Console, ...) or write your own.
{% endtrans %}
</li>
<li>
{% trans %}
Bonobo is young, and the todo-list is huge. Read the <a
href="https://www.bonobo-project.org/roadmap">roadmap</a>.
{% endtrans %}
</li>
</ul>
<p>{% trans %}
You can also download PDF/EPUB versions of the Bonobo documentation:
<a href="http://readthedocs.org/projects/bonobo/downloads/pdf/stable/">PDF version</a>,
<a href="http://readthedocs.org/projects/bonobo/downloads/epub/stable/">EPUB version</a>.
{% endtrans %}
</p>
<h2>Table of contents</h2>
<div>
{{ toctree(maxdepth=2, collapse=False)}}
</div>
{% endblock %}

188
docs/changelog.rst
View File

@ -1,6 +1,186 @@
Changelog
=========
v.0.4.2 - 18 june 2017
::::::::::::::::::::::
* [config] Implements a "requires()" service injection decorator for functions (api may change).
* [core] Execution contexts are now context managers.
* [fs] adds a defaut to current working directory in open_fs(...).
* [logging] Adds logging alias for easier imports.
* [stdlib] Fix I/O related nodes (especially json), there were bad bugs with ioformat.
Dependency updates
------------------
* Update bonobo-docker from 0.2.6 to 0.2.8
* Update dependencies.
* Update fs from 2.0.3 to 2.0.4
* Update requests from 2.17.3 to 2.18.1
v.0.4.0 - 10 june 2017
::::::::::::::::::::::
Important highlights
--------------------
* **BC BREAK WARNING** New IOFORMAT option determines the default expected input and output format of transformations.
New default input/output format of transformations is now kwargs-based, instead of first-argument based. The
rationale behind this is that it does not make any sense to put a dict as the only argument of a transformation
knowing that python has a well supported syntax to do so already. Of course, it may break some of your
transformations but you can require the old behaviour by setting the IOFORMAT=arg0 environment variable.
New features
------------
Command line interface
......................
* Allow to run directories or modules using "bonobo run".
* Bonobo version command now shows where the package is installed, and an optional "--all/-a" flag show all
extensions in the same way. (#81)
* Bonobo run flag "--install/-I" allow to pip install a requirements.txt file if run targets a directory. (#71)
* Adds python logging facility configuration in bonobo cli commands.
* Bonobo init now uses cookiecutter template.
Configuration
.............
* `Exclusive(...)` context manager locks an object usage to one thread at a time.
([docs](http://docs-dev.bonobo-project.org/en/develop/guide/services.html#solving-concurrency-problems))
Standard library
................
* New PrettyPrinter and deprecate old crappy modules.
* New pickle reader and writer (thanks @jelloslinger).
Internals
---------
* ConsoleOutputPlugin now buffers stdout to avoid terminal conflicts. Side effect, output is only done every few tenth
of a second.
Bugfixes
--------
* Fixes jupyter widget.
Extensions
----------
* First release officially supporting bonobo-docker extension. See https://www.bonobo-project.org/with/docker.
* Docker extension can be now installed using the "docker" extra on bonobo (`pip install bonobo[docker]`).
* Jupyter widget now displays the status in topological order, like console.
Miscellaneous
-------------
* Allow "main.py" as well as "__main__.py" to be the main entrypoint of an etl job.
* Better error display (329296c).
* Better testing.
* Code sweeping (ecfdc81).
* Dependencies updated.
* Filesystem now resolve (expand) ~ in path.
* Moving project artifact management (Projectfile) to edgy.project 0.3 format.
* Refactoring and fixes around ioformats.
* Some really minor changes.
v.0.3.2 - 10 june 2017
::::::::::::::::::::::
Weekly maintenance release.
* Updated frozen version numbers in requirements.
* pytest==3.1.1
* requests==2.17.3
* sphinx==1.6.2
* stevedore==1.22.0
Note: this does not change anything when used as a dependency if you freeze your requirements, as the setup.py
requirement specifiers did not change.
v.0.3.1 - 28 may 2017
:::::::::::::::::::::
Weekly maintenance release.
* Updated project management model to edgy.project 0.3 format.
* Updated frozen version numbers in requirements.
* certifi==2017.4.17
* chardet==3.0.3
* coverage==4.4.1
* idna==2.5
* nbconvert==5.2.1
* pbr==3.0.1
* pytest-cov==2.5.1
* pytest==3.1.0
* requests==2.16.5
* sphinx==1.6.1
* sphinxcontrib-websupport==1.0.1
* testpath==0.3.1
* typing==3.6.1
* urllib3==1.21.1
Note: this does not change anything when used as a dependency if you freeze your requirements, as the setup.py
requirement specifiers did not change.
v.0.3.0 - 22 may 2017
:::::::::::::::::::::
Features
--------
* ContextProcessors can now be implemented by getting the "yield" value (v = yield x), shortening the teardown-only
context processors by one line.
* File related writers (file, csv, json ...) now returns NOT_MODIFIED, making it easier to chain something after.
* More consistent console output, nodes are now sorted in a topological order before display.
* Graph.add_chain(...) now takes _input and _output parameters the same way, accepting indexes, instances or names
(subject to change).
* Graph.add_chain(...) now allows to "name" a chain, using _name keyword argument, to easily reference its output later
(subject to change).
* New settings module (bonobo.settings) read environment for some global configuration stuff (DEBUG and PROFILE, for
now).
* New Method subclass of Option allows to use Configurable objects as decorator (see bonobo.nodes.filter.Filter for a
simple example).
* New Filter transformation in standard library.
Internal features
-----------------
* Better ContextProcessor implementation, avoiding to use a decorator on the parent class. Now works with Configurable
instances like Option, Service and Method.
* ContextCurrifier replaces the logic that was in NodeExecutionContext, that setup and teardown the context stack. Maybe
the name is not ideal.
* All builtin transformations are of course updated to use the improved API, and should be 100% backward compatible.
* The "core" package has been dismantled, and its rare remaining members are now in "structs" and "util" packages.
* Standard transformation library has been moved under the bonobo.nodes package. It does not change anything if you used
bonobo.* (which you should).
* ValueHolder is now more restrictive, not allowing to use .value anymore.
Miscellaneous
-------------
* Code cleanup, dead code removal, more tests, etc.
* More documentation.
v.0.2.4 - 2 may 2017
::::::::::::::::::::
* Cosmetic release for PyPI package page formating. Same content as v.0.2.3.
v.0.2.3 - 1 may 2017
:::::::::::::::::::::
* Positional options now supported, backward compatible. All FileHandler subclasses supports their path argument as
positional.
* Better transformation lifecycle management (still work needed here).
* Windows continuous integration now works.
* Refactoring the "API" a lot to have a much cleaner first glance at it.
* More documentation, tutorials, and tuning project artifacts.
v.0.2.2 - 28 apr 2017
:::::::::::::::::::::
@ -29,11 +209,13 @@ Initial release
* Migration from rdc.etl.
* New cool name (ok, that's debatable).
* Only supports python 3.5+, aggressively (which means, we can use async, and we remove all things from python 2/six compat)
* Only supports python 3.5+, aggressively (which means, we can use async, and we remove all things from python 2/six
compat)
* Removes all thing deprecated and/or not really convincing from rdc.etl.
* We want transforms to be simple callables, so refactoring of the harness mess.
* We want to use plain python data structures, so hashes are removed. If you use python 3.6, you may even get sorted dicts.
* We want to use plain python data structures, so hashes are removed. If you use python 3.6, you may even get sorted
dicts.
* Input/output MUX DEMUX removed, maybe no need for that in the real world. May come back, but not in 1.0
* Change dependency policy. We need to include only the very basic requirements (and very required). Everything related
to transforms that we may not use (bs, sqla, ...) should be optional dependencies.
* Execution strategies, threaded by default.
* Execution strategies, threaded by default.

24
docs/contribute/index.rst
View File

@ -13,6 +13,26 @@ contributions have less value, all contributions are very important.
* You can enhance tests.
* etc.
tl;dr
:::::
1. Fork the github repository
.. code-block:: shell-session
$ git clone https://github.com/python-bonobo/bonobo.git # change this to use your fork.
$ cd bonobo
$ git remote add upstream https://github.com/python-bonobo/bonobo.git
$ git fetch upstream
$ git checkout upstream/develop -b feature/my_awesome_feature
$ # code, code, code, test, doc, code, test ...
$ git commit -m '[topic] .... blaaaah ....'
$ git push origin feature/my_awesome_feature
2. Open pull request
3. Rince, repeat
Code-related contributions (including tests and examples)
:::::::::::::::::::::::::::::::::::::::::::::::::::::::::
@ -58,7 +78,7 @@ Guidelines
License
:::::::
`Bonobo is released under the apache license <https://github.com/python-bonobo/bonobo/blob/0.3/LICENSE>`_.
`Bonobo is released under the apache license <https://www.bonobo-project.org/license>`_.
License for non lawyers
:::::::::::::::::::::::
@ -67,6 +87,6 @@ Use it, change it, hack it, brew it, eat it.
For pleasure, non-profit, profit or basically anything else, except stealing credit.
Provided without warranty.
Provided without any warranty.

38
docs/faq.rst
View File

@ -3,6 +3,7 @@ F.A.Q.
List of questions that went up about the project, in no particuliar order.
Too long; didn't read.
----------------------
@ -19,8 +20,22 @@ It's lean manufacturing for data.
.. note::
This is NOT a «big data» tool. We process around 5 millions database lines in around 1 hour with rdc.etl, bonobo
ancestor (algorithms are the same, we still need to run a bit of benchmarks).
This is NOT a «big data» tool. Neither a «data analysis» tool. We process around 5 millions database lines in around
1 hour with rdc.etl, bonobo ancestor (algorithms are the same, we still need to run a bit of benchmarks).
What versions of python does bonobo support? Why not more?
----------------------------------------------------------
Bonobo is battle-tested against the latest python 3.5 and python 3.6. It may work well using other patch releases of those
versions, but we cannot guarantee it.
The main reasons about why 3.5+:
* Creating a tool that works well under both python 2 and 3 is a lot more work.
* Python 3 is nearly 10 years old. Consider moving on.
* Python 3.5 contains syntaxic sugar that makes working with data a lot more convenient.
Can a graph contain another graph?
----------------------------------
@ -30,8 +45,14 @@ No, not for now. There are no tools today in bonobo to insert a graph as a subgr
It would be great to allow it, but there is a few design questions behind this, like what node you use as input and
output of the subgraph, etc.
On another hand, if you don't consider a graph as the container but by the nodes and edges it contains, its pretty
easy to add a set of nodes and edge to a subgraph, and thus simulate it. But there will be more threads, more copies
of the same nodes, so it's not really an acceptable answer for big graphs. If it was possible to use a Graph as a
node, then the problem would be correctly solved.
It is something to be seriously considered post 1.0 (probably way post 1.0).
How would one access contextual data from a transformation? Are there parameter injections like pytest's fixtures?
------------------------------------------------------------------------------------------------------------------
@ -41,7 +62,8 @@ context.
The API may evolve a bit though, because I feel it's a bit hackish, as it is. The concept will stay the same, but we need
to find a better way to apply it.
To understand how it works today, look at https://github.com/python-bonobo/bonobo/blob/0.3/bonobo/io/csv.py#L63 and class hierarchy.
To understand how it works today, look at https://github.com/python-bonobo/bonobo/blob/master/bonobo/nodes/io/csv.py#L31 and class hierarchy.
What is a plugin? Do I need to write one?
-----------------------------------------
@ -49,14 +71,19 @@ What is a plugin? Do I need to write one?
Plugins are special classes added to an execution context, used to enhance or change the actual behavior of an execution
in a generic way. You don't need to write plugins to code transformation graphs.
Is there a difference between a transformation node and a regular python function or generator?
-----------------------------------------------------------------------------------------------
No.
Short answer: no.
Transformation callables are just regular callables, and there is nothing that differentiate it from regular python callables.
You can even use some callables both in an imperative programming context and in a transformation graph, no problem.
Longer answer: yes, sometimes, but you should not care. The function-based transformations are plain old python callable. The
class-based transformations can be plain-old-python-objects, but can also subclass Configurable which brings a lot of
fancy features, like options, service injections, class factories as decorators...
Why did you include the word «marketing» in a commit message? Why is there a marketing-automation tag on the project? Isn't marketing evil?
-------------------------------------------------------------------------------------------------------------------------------------------
@ -83,6 +110,7 @@ See https://github.com/python-bonobo/bonobo/issues/1
Bonobo is not a replacement for pandas, nor dask, nor luigi, nor airflow... It may be a replacement for Pentaho, Talend
or other data integration suites but targets people more comfortable with code as an interface.
All those references to monkeys hurt my head. Bonobos are not monkeys.
----------------------------------------------------------------------
@ -96,6 +124,7 @@ known primate typing feature.»
See https://github.com/python-bonobo/bonobo/issues/24
Who is behind this?
-------------------
@ -104,6 +133,7 @@ Me (as an individual), and a few great people that helped me along the way. Not
The code, documentation, and surrounding material is created using spare time and may lack a bit velocity. Feel free
to jump in so we can go faster!
Documentation seriously lacks X, there is a problem in Y...
-----------------------------------------------------------

8
docs/guide/ext/docker.rst
View File

@ -4,11 +4,5 @@ Bonobo with Docker
.. todo:: The `bonobo-docker` package is at a very alpha stage, and things will change. This section is here to give a
brief overview but is neither complete nor definitive.
Installation
::::::::::::
Read the introduction: https://www.bonobo-project.org/with/docker
Overview
::::::::
Details
:::::::

7
docs/guide/ext/jupyter.rst
View File

@ -15,20 +15,23 @@ Install `bonobo` with the **jupyter** extra::
Install the jupyter extension::
jupyter nbextension enable --py --sys-prefix widgetsnbextension
jupyter nbextension enable --py --sys-prefix bonobo.ext.jupyter
Development
:::::::::::
You should favor yarn over npm to install node packages. If you prefer to use npm, it's up to you to adapt the code.
To install the widget for development, make sure you're using an editable install of bonobo (see install document)::
jupyter nbextension install --py --symlink --sys-prefix bonobo.ext.jupyter
jupyter nbextension enable --py --sys-prefix bonobo.ext.jupyter
If you wanna change the javascript, you should run webpack in watch mode in some terminal::
If you want to change the javascript, you should run webpack in watch mode in some terminal::
cd bonobo/ext/jupyter/js
npm install
yarn install
./node_modules/.bin/webpack --watch
To compile the widget into a distributable version (which gets packaged on PyPI when a release is made), just run

1
docs/guide/ext/sqlalchemy.rst
View File

@ -4,6 +4,7 @@ Bonobo with SQLAlchemy
.. todo:: The `bonobo-sqlalchemy` package is at a very alpha stage, and things will change. This section is here to
give a brief overview but is neither complete nor definitive.
Read the introduction: https://www.bonobo-project.org/with/sqlalchemy
Installation
::::::::::::

20
docs/guide/services.rst
View File

@ -81,6 +81,26 @@ A dictionary, or dictionary-like, "services" named argument can be passed to the
provided is pretty basic, and feature-less. But you can use much more evolved libraries instead of the provided
stub, and as long as it works the same (a.k.a implements a dictionary-like interface), the system will use it.
Solving concurrency problems
----------------------------
If a service cannot be used by more than one thread at a time, either because it's just not threadsafe, or because
it requires to carefully order the calls made (apis that includes nonces, or work on results returned by previous
calls are usually good candidates), you can use the :class:`bonobo.config.Exclusive` context processor to lock the
use of a dependency for a time period.
.. code-block:: python
from bonobo.config import Exclusive
def t1(api):
with Exclusive(api):
api.first_call()
api.second_call()
# ... etc
api.last_call()
Service configuration (to be decided and implemented)
:::::::::::::::::::::::::::::::::::::::::::::::::::::

8
docs/guide/transformations.rst
View File

@ -81,9 +81,13 @@ Services
.. autoclass:: bonobo.config.Service
Method
------
Methods
-------
.. autoclass:: bonobo.config.Method
ContextProcessors
-----------------
.. autoclass:: bonobo.config.ContextProcessor

2
docs/index.rst
View File

@ -8,8 +8,8 @@ Bonobo
tutorial/index
guide/index
reference/index
contribute/index
faq
contribute/index
genindex
modindex

53
docs/install.rst
View File

@ -1,34 +1,54 @@
Installation
============
Bonobo is `available on PyPI <https://pypi.python.org/pypi/bonobo>`_, and it's the easiest solution to get started.
Create an ETL project
:::::::::::::::::::::
Creating a project and starting to write code should take less than a minute:
.. code-block:: shell-session
$ pip install --upgrade bonobo cookiecutter
$ bonobo init my-etl-project
$ bonobo run my-etl-project
Once you bootstrapped a project, you can start editing the default example transformation by editing
`my-etl-project/main.py`. Now, you can head to :doc:`tutorial/index`.
Other installation options
::::::::::::::::::::::::::
Install from PyPI
-----------------
You can install it directly from the `Python Package Index <https://pypi.python.org/pypi/bonobo>`_ (like we did above).
.. code-block:: shell-session
$ pip install bonobo
Install from source
:::::::::::::::::::
-------------------
If you want to install an unreleased version, you can use git urls with pip. This is useful when using bonobo as a
dependency of your code and you want to try a forked version of bonobo with your software. You can use the git+http
string in your `requirements.txt` file. However, the best option for development on bonobo directly is not this one,
but editable installs (see below).
dependency of your code and you want to try a forked version of bonobo with your software. You can use a `git+http`
string in your `requirements.txt` file. However, the best option for development on bonobo is an editable install (see
below).
.. code-block:: shell-session
$ pip install git+https://github.com/python-bonobo/bonobo.git@0.3#egg=bonobo
$ pip install git+https://github.com/python-bonobo/bonobo.git@develop#egg=bonobo
Editable install
::::::::::::::::
----------------
If you plan on making patches to Bonobo, you should install it as an "editable" package, which is a really great pip feature.
Pip will clone your repository in a source directory and create a symlink for it in the site-package directory of your
python interpreter.
If you plan on making patches to Bonobo, you should install it as an "editable" package, which is a really great pip
feature. Pip will clone your repository in a source directory and create a symlink for it in the site-package directory
of your python interpreter.
.. code-block:: shell-session
$ pip install --editable git+https://github.com/python-bonobo/bonobo.git@0.3#egg=bonobo
$ pip install --editable git+https://github.com/python-bonobo/bonobo.git@master#egg=bonobo
.. note:: You can also use the `-e` flag instead of the long version.
@ -54,20 +74,17 @@ I usually name the git remote for the main bonobo repository "upstream", and my
$ git remote rename origin upstream
$ git remote add origin git@github.com:hartym/bonobo.git
$ git fetch --all
Of course, replace my github username by the one you used to fork bonobo. You should be good to go!
Windows support
:::::::::::::::
There are problems on the windows platform, mostly due to the fact bonobo was not developed by experienced windows users.
There are minor issues on the windows platform, mostly due to the fact bonobo was not developed by experienced windows
users.
We're trying to look into that but energy available to provide serious support on windows is very limited.
If you have experience in this domain and you're willing to help, you're more than welcome!
.. todo::
Better install docs, especially on how to use different forks or branches, etc.

12
docs/reference/examples.rst
View File

@ -1,10 +1,16 @@
Examples
========
There are a few examples bundled with **bonobo**. You'll find them under the :mod:`bonobo.examples` package, and
you can try them in a clone of bonobo by typing::
There are a few examples bundled with **bonobo**.
$ bonobo run bonobo/examples/.../file.py
You'll find them under the :mod:`bonobo.examples` package, and you can run them directly as modules:
$ bonobo run -m bonobo.examples...module
.. toctree::
:maxdepth: 4
examples/tutorials
Datasets

50
docs/reference/examples/tutorials.rst Normal file
View File

@ -0,0 +1,50 @@
Examples from the tutorial
==========================
Examples from :doc:`/tutorial/tut01`
::::::::::::::::::::::::::::::::::::
Example 1
---------
.. automodule:: bonobo.examples.tutorials.tut01e01
:members:
:undoc-members:
:show-inheritance:
Example 2
---------
.. automodule:: bonobo.examples.tutorials.tut01e02
:members:
:undoc-members:
:show-inheritance:
Examples from :doc:`/tutorial/tut02`
::::::::::::::::::::::::::::::::::::
Example 1: Read
---------------
.. automodule:: bonobo.examples.tutorials.tut02e01_read
:members:
:undoc-members:
:show-inheritance:
Example 2: Write
----------------
.. automodule:: bonobo.examples.tutorials.tut02e02_write
:members:
:undoc-members:
:show-inheritance:
Example 3: Write as map
-----------------------
.. automodule:: bonobo.examples.tutorials.tut02e02_writeasmap
:members:
:undoc-members:
:show-inheritance:

31
docs/roadmap.rst
View File

@ -1,12 +1,12 @@
Detailed roadmap
================
Internal roadmap notes
======================
initialize / finalize better than start / stop ?
Things that should be thought about and/or implemented, but that I don't know where to store.
Graph and node level plugins
::::::::::::::::::::::::::::
* Enhancers or nide-level plugins
* Enhancers or node-level plugins
* Graph level plugins
* Documentation
@ -15,21 +15,19 @@ Command line interface and environment
* How do we manage environment ? .env ?
* How do we configure plugins ?
* Console run should allow console plugin as a command line argument (or silence it).
Services and Processors
:::::::::::::::::::::::
* ContextProcessors not clean
* ContextProcessors not clean (a bit better, but still not in love with the api)
Next...
:::::::
* Release process specialised for bonobo. With changelog production, etc.
* Document how to upgrade version, like, minor need change badges, etc.
* PyPI page looks like crap: https://pypi.python.org/pypi/bonobo/0.2.1
* Windows break because of readme encoding. Fix in edgy.
* bonobo init --with sqlalchemy,docker
* Windows console looks crappy.
* bonobo init --with sqlalchemy,docker; cookiecutter?
* logger, vebosity level
@ -39,22 +37,15 @@ External libs that looks good
* dask.distributed
* mediator (event dispatcher)
Version 0.3
Version 0.4
:::::::::::
* Services !
* SQLAlchemy 101
Version 0.2
:::::::::::
Design decisions
::::::::::::::::
* Autodetect if within jupyter notebook context, and apply plugin if it's the case.
* New bonobo.structs package with simple data structures (bags, graphs, tokens).
Plugins API
:::::::::::
* Stabilize, find other things to do.
* initialize / finalize better than start / stop ?
Minor stuff
:::::::::::

23
docs/tutorial/index.rst
View File

@ -1,8 +1,6 @@
First steps
===========
Bonobo uses simple python and should be quick and easy to learn.
What is Bonobo?
:::::::::::::::
@ -11,17 +9,34 @@ python code in charge of handling similar shaped independant lines of data.
Bonobo *is not* a statistical or data-science tool. If you're looking for a data-analysis tool in python, use Pandas.
Bonobo is a lean manufacturing assembly line for data that let you focus on the actual work instead of the plumbery.
Bonobo is a lean manufacturing assembly line for data that let you focus on the actual work instead of the plumbery
(execution contexts, parallelism, error handling, console output, logging, ...).
Bonobo uses simple python and should be quick and easy to learn.
Tutorial
::::::::
.. note::
Good documentation is not easy to write. We do our best to make it better and better.
Although all content here should be accurate, you may feel a lack of completeness, for which we plaid guilty and
apologize.
If you're stuck, please come and ask on our `slack channel <https://bonobo-slack.herokuapp.com/>`_, we'll figure
something out.
If you're not stuck but had trouble understanding something, please consider contributing to the docs (via github
pull requests).
.. toctree::
:maxdepth: 2
tut01
tut02
tut03
tut04
What's next?
@ -43,6 +58,6 @@ Read about integrating external tools with bonobo
* :doc:`../guide/ext/docker`: run transformation graphs in isolated containers.
* :doc:`../guide/ext/jupyter`: run transformations within jupyter notebooks.
* :doc:`../guide/ext/selenium`: run
* :doc:`../guide/ext/selenium`: crawl the web using a real browser and work with the gathered data.
* :doc:`../guide/ext/sqlalchemy`: everything you need to interract with SQL databases.

160
docs/tutorial/tut01.rst
View File

@ -1,58 +1,91 @@
Basic concepts
==============
Let's get started!
==================
To begin with Bonobo, you need to install it in a working python 3.5+ environment:
To begin with Bonobo, you need to install it in a working python 3.5+ environment, and you'll also need cookiecutter
to bootstrap your project.
.. code-block:: shell-session
$ pip install bonobo
$ pip install bonobo cookiecutter
See :doc:`/install` for more options.
Let's write a first data transformation
:::::::::::::::::::::::::::::::::::::::
We'll start with the simplest transformation possible.
Create an empty project
:::::::::::::::::::::::
In **Bonobo**, a transformation is a plain old python callable, not more, not less. Let's write one that takes a string
and uppercases it.
Your ETL code will live in ETL projects, which are basically a bunch of files, including python code, that bonobo
can run.
.. code-block:: shell-session
$ bonobo init tutorial
This will create a `tutorial` directory (`content description here <https://www.bonobo-project.org/with/cookiecutter>`_).
To run this project, use:
.. code-block:: shell-session
$ bonobo run tutorial
Write a first transformation
::::::::::::::::::::::::::::
Open `tutorial/main.py`, and delete all the code here.
A transformation can be whatever python can call. Simplest transformations are functions and generators.
Let's write one:
.. code-block:: python
def uppercase(x: str):
def transform(x):
return x.upper()
Pretty straightforward.
Easy.
You could even use :func:`str.upper` directly instead of writing a wrapper, as a type's method (unbound) will take an
instance of this type as its first parameter (what you'd call `self` in your method).
.. note::
The type annotations written here are not used, but can make your code much more readable, and may very well be used as
validators in the future.
This function is very similar to :func:`str.upper`, which you can use directly.
Let's write two more transformations: a generator to produce the data to be transformed, and something that outputs it,
because, yeah, feedback is cool.
Let's write two more transformations for the "extract" and "load" steps. In this example, we'll generate the data from
scratch, and we'll use stdout to "simulate" data-persistence.
.. code-block:: python
def generate_data():
def extract():
yield 'foo'
yield 'bar'
yield 'baz'
def output(x: str):
def load(x):
print(x)
Once again, you could have skipped the pain of writing this and simply use an iterable to generate the data and the
builtin :func:`print` for the output, but we'll stick to writing our own transformations for now.
Bonobo makes no difference between generators (yielding functions) and regular functions. It will, in all cases, iterate
on things returned, and a normal function will just be seen as a generator that yields only once.
Let's chain the three transformations together and run the transformation graph:
.. note::
Once again, you should use the builtin :func:`print` directly instead of this `load()` function.
Create a transformation graph
:::::::::::::::::::::::::::::
Amongst other features, Bonobo will mostly help you there with the following:
* Execute the transformations in independant threads
* Pass the outputs of one thread to other(s) thread(s) inputs.
To do this, it needs to know what data-flow you want to achieve, and you'll use a :class:`bonobo.Graph` to describe it.
.. code-block:: python
import bonobo
graph = bonobo.Graph(generate_data, uppercase, output)
graph = bonobo.Graph(extract, transform, load)
if __name__ == '__main__':
bonobo.run(graph)
@ -64,14 +97,50 @@ Let's chain the three transformations together and run the transformation graph:
stylesheet = "../_static/graphs.css";
BEGIN [shape="point"];
BEGIN -> "generate_data" -> "uppercase" -> "output";
BEGIN -> "extract" -> "transform" -> "load";
}
We use the :func:`bonobo.run` helper that hides the underlying object composition necessary to actually run the
transformations in parallel, because it's simpler.
.. note::
Depending on what you're doing, you may use the shorthand helper method, or the verbose one. Always favor the shorter,
if you don't need to tune the graph or the execution strategy (see below).
The `if __name__ == '__main__':` section is not required, unless you want to run it directly using the python
interpreter.
Execute the job
:::::::::::::::
Save `tutorial/main.py` and execute your transformation again:
.. code-block:: shell-session
$ bonobo run tutorial
This example is available in :mod:`bonobo.examples.tutorials.tut01e01`, and you can also run it as a module:
.. code-block:: shell-session
$ bonobo run -m bonobo.examples.tutorials.tut01e01
Rewrite it using builtins
:::::::::::::::::::::::::
There is a much simpler way to describe an equivalent graph:
.. literalinclude:: ../../bonobo/examples/tutorials/tut01e02.py
:language: python
The `extract()` generator has been replaced by a list, as Bonobo will interpret non-callable iterables as a no-input
generator.
This example is also available in :mod:`bonobo.examples.tutorials.tut01e02`, and you can also run it as a module:
.. code-block:: shell-session
$ bonobo run -m bonobo.examples.tutorials.tut01e02
You can now jump to the next part (:doc:`tut02`), or read a small summary of concepts and definitions introduced here
below.
Takeaways
:::::::::
@ -79,7 +148,7 @@ Takeaways
① The :class:`bonobo.Graph` class is used to represent a data-processing pipeline.
It can represent simple list-like linear graphs, like here, but it can also represent much more complex graphs, with
branches and cycles.
forks and joins.
This is what the graph we defined looks like:
@ -97,10 +166,10 @@ either `return` or `yield` data to send it to the next step. Regular functions (
each call is guaranteed to return exactly one result, while generators (using `yield`) should be prefered if the
number of output lines for a given input varies.
③ The `Graph` instance, or `transformation graph` is then executed using an `ExecutionStrategy`. You did not use it
directly in this tutorial, but :func:`bonobo.run` created an instance of :class:`bonobo.ThreadPoolExecutorStrategy`
under the hood (which is the default strategy). Actual behavior of an execution will depend on the strategy chosen, but
the default should be fine in most of the basic cases.
③ The `Graph` instance, or `transformation graph` is executed using an `ExecutionStrategy`. You won't use it directly,
but :func:`bonobo.run` created an instance of :class:`bonobo.ThreadPoolExecutorStrategy` under the hood (the default
strategy). Actual behavior of an execution will depend on the strategy chosen, but the default should be fine for most
cases.
④ Before actually executing the `transformations`, the `ExecutorStrategy` instance will wrap each component in an
`execution context`, whose responsibility is to hold the state of the transformation. It enables to keep the
@ -109,23 +178,24 @@ the default should be fine in most of the basic cases.
Concepts and definitions
::::::::::::::::::::::::
* Transformation: a callable that takes input (as call parameters) and returns output(s), either as its return value or
* **Transformation**: a callable that takes input (as call parameters) and returns output(s), either as its return value or
by yielding values (a.k.a returning a generator).
* Transformation graph (or Graph): a set of transformations tied together in a :class:`bonobo.Graph` instance, which is a simple
directed acyclic graph (also refered as a DAG, sometimes).
* Node: a transformation within the context of a transformation graph. The node defines what to do with a
transformation's output, and especially what other nodes to feed with the output.
* Execution strategy (or strategy): a way to run a transformation graph. It's responsibility is mainly to parallelize
* **Transformation graph (or Graph)**: a set of transformations tied together in a :class:`bonobo.Graph` instance, which is
a directed acyclic graph (or DAG).
* **Node**: a graph element, most probably a transformation in a graph.
* **Execution strategy (or strategy)**: a way to run a transformation graph. It's responsibility is mainly to parallelize
(or not) the transformations, on one or more process and/or computer, and to setup the right queuing mechanism for
transformations' inputs and outputs.
* Execution context (or context): a wrapper around a node that holds the state for it. If the node needs state, there
are tools available in bonobo to feed it to the transformation using additional call parameters, and so every
transformation will be atomic.
* **Execution context (or context)**: a wrapper around a node that holds the state for it. If the node needs state, there
are tools available in bonobo to feed it to the transformation using additional call parameters, keeping
transformations stateless.
Next
::::
You now know all the basic concepts necessary to build (batch-like) data processors.
Time to jump to the second part: :doc:`tut02`
Time to jump to the second part: :doc:`tut02`.

106
docs/tutorial/tut02.rst
View File

@ -1,11 +1,14 @@
Working with files
==================
Bonobo would be a bit useless if the aim was just to uppercase small lists of strings.
Bonobo would be pointless if the aim was just to uppercase small lists of strings.
In fact, Bonobo should not be used if you don't expect any gain from parallelization/distribution of tasks.
Let's take the following graph as an example:
Some background...
::::::::::::::::::
Let's take the following graph:
.. graphviz::
@ -16,18 +19,22 @@ Let's take the following graph as an example:
"B" -> "D";
}
The execution strategy does a bit of under the scene work, wrapping every component in a thread (assuming you're using
the :class:`bonobo.strategies.ThreadPoolExecutorStrategy`).
When run, the execution strategy wraps every component in a thread (assuming you're using the default
:class:`bonobo.strategies.ThreadPoolExecutorStrategy`).
Bonobo will send each line of data in the input node's thread (here, `A`). Now, each time `A` *yields* or *returns*
something, it will be pushed on `B` input :class:`queue.Queue`, and will be consumed by `B`'s thread.
something, it will be pushed on `B` input :class:`queue.Queue`, and will be consumed by `B`'s thread. Meanwhile, `A`
will continue to run, if it's not done.
When there is more than one node linked as the output of a node (for example, with `B`, `C`, and `D`) , the same thing
When there is more than one node linked as the output of a node (for example, with `B`, `C`, and `D`), the same thing
happens except that each result coming out of `B` will be sent to both on `C` and `D` input :class:`queue.Queue`.
The great thing is that you generally don't have to think about it. Just be aware that your components will be run in
parallel (with the default strategy), and don't worry too much about blocking components, as they won't block their
siblings when run in bonobo.
One thing to keep in mind here is that as the objects are passed from thread to thread, you need to write "pure"
transformations (see :doc:`/guide/purity`).
You generally don't have to think about it. Just be aware that your nodes will run in parallel, and don't worry
too much about nodes running blocking operations, as they will run in parallel. As soon as a line of output is ready,
the next nodes will start consuming it.
That being said, let's manipulate some files.
@ -38,46 +45,41 @@ There are a few component builders available in **Bonobo** that let you read fro
All readers work the same way. They need a filesystem to work with, and open a "path" they will read from.
* :class:`bonobo.io.FileReader`
* :class:`bonobo.io.JsonReader`
* :class:`bonobo.io.CsvReader`
* :class:`bonobo.CsvReader`
* :class:`bonobo.FileReader`
* :class:`bonobo.JsonReader`
* :class:`bonobo.PickleReader`
We'll use a text file that was generated using Bonobo from the "liste-des-cafes-a-un-euro" dataset made available by
Mairie de Paris under the Open Database License (ODbL). You can `explore the original dataset
<https://opendata.paris.fr/explore/dataset/liste-des-cafes-a-un-euro/information/>`_.
You'll need the `example dataset <https://github.com/python-bonobo/bonobo/blob/0.3/bonobo/examples/datasets/coffeeshops.txt>`_,
available in **Bonobo**'s repository.
You'll need the `"coffeeshops.txt" example dataset <https://github.com/python-bonobo/bonobo/blob/master/bonobo/examples/datasets/coffeeshops.txt>`_,
available in **Bonobo**'s repository:
.. literalinclude:: ../../bonobo/examples/tutorials/tut02_01_read.py
.. code-block:: shell-session
$ curl https://raw.githubusercontent.com/python-bonobo/bonobo/master/bonobo/examples/datasets/coffeeshops.txt > `python -c 'import bonobo; print(bonobo.get_examples_path("datasets/coffeeshops.txt"))'`
.. note::
The "example dataset download" step will be easier in the future.
https://github.com/python-bonobo/bonobo/issues/134
.. literalinclude:: ../../bonobo/examples/tutorials/tut02e01_read.py
:language: python
You can run this script directly using the python interpreter:
You can also run this example as a module (but you'll still need the dataset...):
.. code-block:: shell-session
$ python bonobo/examples/tutorials/tut02_01_read.py
$ bonobo run -m bonobo.examples.tutorials.tut02e01_read
Another option is to use the bonobo cli, which allows more flexibility:
.. note::
.. code-block:: shell-session
$ bonobo run bonobo/examples/tutorials/tut02_01_read.py
Using bonobo command line has a few advantages.
It will look for one and only one :class:`bonobo.Graph` instance in the file given as argument, configure an execution
strategy, eventually plugins, and execute it. It has the benefit of allowing to tune the "artifacts" surrounding the
transformation graph on command line (verbosity, plugins ...), and it will also ease the transition to run
transformation graphs in containers, as the syntax will be the same. Of course, it is not required, and the
containerization capabilities are provided by an optional and separate python package.
It also change a bit the way you can configure service dependencies. The CLI won't run the `if __name__ == '__main__'`
block, and thus it won't get the configured services passed to :func:`bonobo.run`. Instead, one option to configure
services is to define a `get_services()` function in a
`_services.py <https://github.com/python-bonobo/bonobo/blob/0.3/bonobo/examples/tutorials/_services.py>`_ file.
There will be more options using the CLI or environment to override things soon.
Don't focus too much on the `get_services()` function for now. It is required, with this exact name, but we'll get
into that in a few minutes.
Writing to files
::::::::::::::::
@ -86,22 +88,34 @@ Let's split this file's each lines on the first comma and store a json file mapp
Here are, like the readers, the classes available to write files
* :class:`bonobo.io.FileWriter`
* :class:`bonobo.io.JsonWriter`
* :class:`bonobo.io.CsvWriter`
* :class:`bonobo.CsvWriter`
* :class:`bonobo.FileWriter`
* :class:`bonobo.JsonWriter`
* :class:`bonobo.PickleWriter`
Let's write a first implementation:
.. literalinclude:: ../../bonobo/examples/tutorials/tut02_02_write.py
.. literalinclude:: ../../bonobo/examples/tutorials/tut02e02_write.py
:language: python
You can run it and read the output file, you'll see it misses the "map" part of the question. Let's extend
:class:`bonobo.io.JsonWriter` to finish the job:
(run it with :code:`bonobo run -m bonobo.examples.tutorials.tut02e02_write` or :code:`bonobo run myfile.py`)
.. literalinclude:: ../../bonobo/examples/tutorials/tut02_03_writeasmap.py
If you read the output file, you'll see it misses the "map" part of the problem.
Let's extend :class:`bonobo.io.JsonWriter` to finish the job:
.. literalinclude:: ../../bonobo/examples/tutorials/tut02e03_writeasmap.py
:language: python
You can now run it again, it should produce a nice map. We favored a bit hackish solution here instead of constructing a
map in python then passing the whole to :func:`json.dumps` because we want to work with streams, if you have to
construct the whole data structure in python, you'll loose a lot of bonobo's benefits.
(run it with :code:`bonobo run -m bonobo.examples.tutorials.tut02e03_writeasmap` or :code:`bonobo run myfile.py`)
It should produce a nice map.
We favored a bit hackish solution here instead of constructing a map in python then passing the whole to
:func:`json.dumps` because we want to work with streams, if you have to construct the whole data structure in python,
you'll loose a lot of bonobo's benefits.
Next
::::
Time to write some more advanced transformations, with service dependencies: :doc:`tut03`.

200
docs/tutorial/tut03.rst Normal file
View File

@ -0,0 +1,200 @@
Configurables and Services
==========================
.. note::
This section lacks completeness, sorry for that (but you can still read it!).
In the last section, we used a few new tools.
Class-based transformations and configurables
:::::::::::::::::::::::::::::::::::::::::::::
Bonobo is a bit dumb. If something is callable, it considers it can be used as a transformation, and it's up to the
user to provide callables that logically fits in a graph.
You can use plain python objects with a `__call__()` method, and it ill just work.
As a lot of transformations needs common machinery, there is a few tools to quickly build transformations, most of
them requiring your class to subclass :class:`bonobo.config.Configurable`.
Configurables allows to use the following features:
* You can add **Options** (using the :class:`bonobo.config.Option` descriptor). Options can be positional, or keyword
based, can have a default value and will be consumed from the constructor arguments.
.. code-block:: python
from bonobo.config import Configurable, Option
class PrefixIt(Configurable):
prefix = Option(str, positional=True, default='>>>')
def call(self, row):
return self.prefix + ' ' + row
prefixer = PrefixIt('$')
* You can add **Services** (using the :class:`bonobo.config.Service` descriptor). Services are a subclass of
:class:`bonobo.config.Option`, sharing the same basics, but specialized in the definition of "named services" that
will be resolved at runtime (a.k.a for which we will provide an implementation at runtime). We'll dive more into that
in the next section
.. code-block:: python
from bonobo.config import Configurable, Option, Service
class HttpGet(Configurable):
url = Option(default='https://jsonplaceholder.typicode.com/users')
http = Service('http.client')
def call(self, http):
resp = http.get(self.url)
for row in resp.json():
yield row
http_get = HttpGet()
* You can add **Methods** (using the :class:`bonobo.config.Method` descriptor). :class:`bonobo.config.Method` is a
subclass of :class:`bonobo.config.Option` that allows to pass callable parameters, either to the class constructor,
or using the class as a decorator.
.. code-block:: python
from bonobo.config import Configurable, Method
class Applier(Configurable):
apply = Method()
def call(self, row):
return self.apply(row)
@Applier
def Prefixer(self, row):
return 'Hello, ' + row
prefixer = Prefixer()
* You can add **ContextProcessors**, which are an advanced feature we won't introduce here. If you're familiar with
pytest, you can think of them as pytest fixtures, execution wise.
Services
::::::::
The motivation behind services is mostly separation of concerns, testability and deployability.
Usually, your transformations will depend on services (like a filesystem, an http client, a database, a rest api, ...).
Those services can very well be hardcoded in the transformations, but there is two main drawbacks:
* You won't be able to change the implementation depending on the current environment (development laptop versus
production servers, bug-hunting session versus execution, etc.)
* You won't be able to test your transformations without testing the associated services.
To overcome those caveats of hardcoding things, we define Services in the configurable, which are basically
string-options of the service names, and we provide an implementation at the last moment possible.
There are two ways of providing implementations:
* Either file-wide, by providing a `get_services()` function that returns a dict of named implementations (we did so
with filesystems in the previous step, :doc:`tut02.rst`)
* Either directory-wide, by providing a `get_services()` function in a specially named `_services.py` file.
The first is simpler if you only have one transformation graph in one file, the second allows to group coherent
transformations together in a directory and share the implementations.
Let's see how to use it, starting from the previous service example:
.. code-block:: python
from bonobo.config import Configurable, Option, Service
class HttpGet(Configurable):
url = Option(default='https://jsonplaceholder.typicode.com/users')
http = Service('http.client')
def call(self, http):
resp = http.get(self.url)
for row in resp.json():
yield row
We defined an "http.client" service, that obviously should have a `get()` method, returning responses that have a
`json()` method.
Let's provide two implementations for that. The first one will be using `requests <http://docs.python-requests.org/>`_,
that coincidally satisfies the described interface:
.. code-block:: python
import bonobo
import requests
def get_services():
return {
'http.client': requests
}
graph = bonobo.Graph(
HttpGet(),
print,
)
If you run this code, you should see some mock data returned by the webservice we called (assuming it's up and you can
reach it).
Now, the second implementation will replace that with a mock, used for testing purposes:
.. code-block:: python
class HttpResponseStub:
def json(self):
return [
{'id': 1, 'name': 'Leanne Graham', 'username': 'Bret', 'email': 'Sincere@april.biz', 'address': {'street': 'Kulas Light', 'suite': 'Apt. 556', 'city': 'Gwenborough', 'zipcode': '92998-3874', 'geo': {'lat': '-37.3159', 'lng': '81.1496'}}, 'phone': '1-770-736-8031 x56442', 'website': 'hildegard.org', 'company': {'name': 'Romaguera-Crona', 'catchPhrase': 'Multi-layered client-server neural-net', 'bs': 'harness real-time e-markets'}},
{'id': 2, 'name': 'Ervin Howell', 'username': 'Antonette', 'email': 'Shanna@melissa.tv', 'address': {'street': 'Victor Plains', 'suite': 'Suite 879', 'city': 'Wisokyburgh', 'zipcode': '90566-7771', 'geo': {'lat': '-43.9509', 'lng': '-34.4618'}}, 'phone': '010-692-6593 x09125', 'website': 'anastasia.net', 'company': {'name': 'Deckow-Crist', 'catchPhrase': 'Proactive didactic contingency', 'bs': 'synergize scalable supply-chains'}},
]
class HttpStub:
def get(self, url):
return HttpResponseStub()
def get_services():
return {
'http.client': HttpStub()
}
graph = bonobo.Graph(
HttpGet(),
print,
)
The `Graph` definition staying the exact same, you can easily substitute the `_services.py` file depending on your
environment (the way you're doing this is out of bonobo scope and heavily depends on your usual way of managing
configuration files on different platforms).
Starting with bonobo 0.5 (not yet released), you will be able to use service injections with function-based
transformations too, using the `bonobo.config.requires` decorator to mark a dependency.
.. code-block:: python
from bonobo.config import requires
@requires('http.client')
def http_get(http):
resp = http.get('https://jsonplaceholder.typicode.com/users')
for row in resp.json():
yield row
Read more
:::::::::
* :doc:`/guide/services`
* :doc:`/reference/api_config`
Next
::::
:doc:`tut04`.

214
docs/tutorial/tut04.rst Normal file
View File

@ -0,0 +1,214 @@
Working with databases
======================
Databases (and especially SQL databases here) are not the focus of Bonobo, thus support for it is not (and will never
be) included in the main package. Instead, working with databases is done using third party, well maintained and
specialized packages, like SQLAlchemy, or other database access libraries from the python cheese shop.
.. note::
SQLAlchemy extension is not yet complete. Things may be not optimal, and some APIs will change. You can still try,
of course.
Consider the following document as a "preview" (yes, it should work, yes it may break in the future).
Also, note that for early development stages, we explicitely support only PostreSQL, although it may work well
with `any other database supported by SQLAlchemy <http://docs.sqlalchemy.org/en/latest/core/engines.html#supported-databases>`_.
First, read https://www.bonobo-project.org/with/sqlalchemy for instructions on how to install. You **do need** the
bleeding edge version of `bonobo` and `bonobo-sqlalchemy` to make this work.
Requirements
::::::::::::
Once you installed `bonobo_sqlalchemy` (read https://www.bonobo-project.org/with/sqlalchemy to use bleeding edge
version), install the following additional packages:
.. code-block:: shell-session
$ pip install -U python-dotenv psycopg2 awesome-slugify
Those packages are not required by the extension, but `python-dotenv` will help us configure the database DSN, and
`psycopg2` is required by SQLAlchemy to connect to PostgreSQL databases. Also, we'll use a slugifier to create unique
identifiers for the database (maybe not what you'd do in the real world, but very much sufficient for example purpose).
Configure a database engine
:::::::::::::::::::::::::::
Open your `_services.py` file and replace the code:
.. code-block:: python
import bonobo, dotenv, logging, os
from bonobo_sqlalchemy.util import create_postgresql_engine
dotenv.load_dotenv(dotenv.find_dotenv())
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
def get_services():
return {
'fs': bonobo.open_examples_fs('datasets'),
'fs.output': bonobo.open_fs(),
'sqlalchemy.engine': create_postgresql_engine(**{
'name': 'tutorial',
'user': 'tutorial',
'pass': 'tutorial',
})
}
The `create_postgresql_engine` is a tiny function building the DSN from reasonable defaults, that you can override
either by providing kwargs, or with system environment variables. If you want to override something, open the `.env`
file and add values for one or more of `POSTGRES_NAME`, `POSTGRES_USER`, 'POSTGRES_PASS`, `POSTGRES_HOST`,
`POSTGRES_PORT`. Please note that kwargs always have precedence on environment, but that you should prefer using
environment variables for anything that is not immutable from one platform to another.
Add database operation to the graph
:::::::::::::::::::::::::::::::::::
Let's create a `tutorial/pgdb.py` job:
.. code-block:: python
import bonobo
import bonobo_sqlalchemy
from bonobo.examples.tutorials.tut02e03_writeasmap import graph, split_one_to_map
graph = graph.copy()
graph.add_chain(
bonobo_sqlalchemy.InsertOrUpdate('coffeeshops'),
_input=split_one_to_map
)
Notes here:
* We use the code from :doc:`tut02`, which is bundled with bonobo in the `bonobo.examples.tutorials` package.
* We "fork" the graph, by creating a copy and appending a new "chain", starting at a point that exists in the other
graph.
* We use :class:`bonobo_sqlalchemy.InsertOrUpdate` (which role, in case it is not obvious, is to create database rows if
they do not exist yet, or update the existing row, based on a "discriminant" criteria (by default, "id")).
If we run this transformation (with `bonobo run tutorial/pgdb.py`), we should get an error:
.. code-block:: text
| File ".../lib/python3.6/site-packages/psycopg2/__init__.py", line 130, in connect
| conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
| sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) FATAL: database "tutorial" does not exist
|
|
| The above exception was the direct cause of the following exception:
|
| Traceback (most recent call last):
| File ".../bonobo-devkit/bonobo/bonobo/strategies/executor.py", line 45, in _runner
| node_context.start()
| File ".../bonobo-devkit/bonobo/bonobo/execution/base.py", line 75, in start
| self._stack.setup(self)
| File ".../bonobo-devkit/bonobo/bonobo/config/processors.py", line 94, in setup
| _append_to_context = next(_processed)
| File ".../bonobo-devkit/bonobo-sqlalchemy/bonobo_sqlalchemy/writers.py", line 43, in create_connection
| raise UnrecoverableError('Could not create SQLAlchemy connection: {}.'.format(str(exc).replace('\n', ''))) from exc
| bonobo.errors.UnrecoverableError: Could not create SQLAlchemy connection: (psycopg2.OperationalError) FATAL: database "tutorial" does not exist.
The database we requested do not exist. It is not the role of bonobo to do database administration, and thus there is
no tool here to create neither the database, nor the tables we want to use.
Create database and table
:::::::::::::::::::::::::
There are however tools in `sqlalchemy` to manage tables, so we'll create the database by ourselves, and ask sqlalchemy
to create the table:
.. code-block:: shell-session
$ psql -U postgres -h localhost
psql (9.6.1, server 9.6.3)
Type "help" for help.
postgres=# CREATE ROLE tutorial WITH LOGIN PASSWORD 'tutorial';
CREATE ROLE
postgres=# CREATE DATABASE tutorial WITH OWNER=tutorial TEMPLATE=template0 ENCODING='utf-8';
CREATE DATABASE
Now, let's use a little trick and add this section to `pgdb.py`:
.. code-block:: python
import sys
from sqlalchemy import Table, Column, String, Integer, MetaData
def main():
from bonobo.commands.run import get_default_services
services = get_default_services(__file__)
if len(sys.argv) == 1:
return bonobo.run(graph, services=services)
elif len(sys.argv) == 2 and sys.argv[1] == 'reset':
engine = services.get('sqlalchemy.engine')
metadata = MetaData()
coffee_table = Table(
'coffeeshops',
metadata,
Column('id', String(255), primary_key=True),
Column('name', String(255)),
Column('address', String(255)),
)
metadata.drop_all(engine)
metadata.create_all(engine)
else:
raise NotImplementedError('I do not understand.')
if __name__ == '__main__':
main()
.. note::
We're using private API of bonobo here, which is unsatisfactory, discouraged and may change. Some way to get the
service dictionnary will be added to the public api in a future release of bonobo.
Now run:
.. code-block:: python
$ python tutorial/pgdb.py reset
Database and table should now exist.
Format the data
:::::::::::::::
Let's prepare our data for database, and change the `.add_chain(..)` call to do it prior to `InsertOrUpdate(...)`
.. code-block:: python
from slugify import slugify_url
def format_for_db(row):
name, address = list(row.items())[0]
return {
'id': slugify_url(name),
'name': name,
'address': address,
}
# ...
graph = graph.copy()
graph.add_chain(
format_for_db,
bonobo_sqlalchemy.InsertOrUpdate('coffeeshops'),
_input=split_one_to_map
)
Run!
::::
You can now run the script (either with `bonobo run tutorial/pgdb.py` or directly with the python interpreter, as we
added a "main" section) and the dataset should be inserted in your database. If you run it again, no new rows are
created.
Note that as we forked the graph from :doc:`tut02`, the transformation also writes the data to `coffeeshops.json`, as
before.

37
requirements-dev.txt
View File

@ -1,27 +1,34 @@
-e .[dev]
alabaster==0.7.10
astroid==1.5.2
arrow==0.10.0
babel==2.4.0
coverage==4.3.4
binaryornot==0.4.3
certifi==2017.4.17
chardet==3.0.4
click==6.7
cookiecutter==1.5.1
coverage==4.4.1
docutils==0.13.1
future==0.16.0
idna==2.5
imagesize==0.7.1
isort==4.2.5
jinja2-time==0.2.0
jinja2==2.9.6
lazy-object-proxy==1.2.2
markupsafe==1.0
mccabe==0.6.1
py==1.4.33
poyo==0.4.1
py==1.4.34
pygments==2.2.0
pylint==1.7.1
pytest-cov==2.4.0
pytest-cov==2.5.1
pytest-sugar==0.8.0
pytest-timeout==1.2.0
pytest==3.0.7
pytest==3.1.2
python-dateutil==2.6.0
pytz==2017.2
requests==2.13.0
requests==2.18.1
six==1.10.0
snowballstemmer==1.2.1
sphinx-rtd-theme==0.2.4
sphinx==1.5.5
wrapt==1.10.10
yapf==0.16.1
sphinx==1.6.3
sphinxcontrib-websupport==1.0.1
termcolor==1.1.0
urllib3==1.21.1
whichcraft==0.4.1

20
requirements-docker.txt Normal file
View File

@ -0,0 +1,20 @@
-e .[docker]
appdirs==1.4.3
bonobo-docker==0.2.9
certifi==2017.4.17
chardet==3.0.4
colorama==0.3.9
docker-pycreds==0.2.1
docker==2.3.0
fs==2.0.4
idna==2.5
packaging==16.8
pbr==3.1.1
psutil==5.2.2
pyparsing==2.2.0
pytz==2017.2
requests==2.18.1
six==1.10.0
stevedore==1.23.0
urllib3==1.21.1
websocket-client==0.44.0

13
requirements-jupyter.txt
View File

@ -1,31 +1,30 @@
-e .[jupyter]
appnope==0.1.0
bleach==2.0.0
decorator==4.0.11
entrypoints==0.2.2
entrypoints==0.2.3
html5lib==0.999999999
ipykernel==4.6.1
ipython-genutils==0.2.0
ipython==6.0.0
ipython==6.1.0
ipywidgets==6.0.0
jedi==0.10.2
jinja2==2.9.6
jsonschema==2.6.0
jupyter-client==5.0.1
jupyter-client==5.1.0
jupyter-console==5.1.0
jupyter-core==4.3.0
jupyter==1.0.0
markupsafe==1.0
mistune==0.7.4
nbconvert==5.1.1
nbconvert==5.2.1
nbformat==4.3.0
notebook==5.0.0
pandocfilters==1.4.1
pexpect==4.2.1
pickleshare==0.7.4
prompt-toolkit==1.0.14
ptyprocess==0.5.1
ptyprocess==0.5.2
pygments==2.2.0
python-dateutil==2.6.0
pyzmq==16.0.2
@ -33,7 +32,7 @@ qtconsole==4.3.0
simplegeneric==0.8.1
six==1.10.0
terminado==0.6
testpath==0.3
testpath==0.3.1
tornado==4.5.1
traitlets==4.3.2
wcwidth==0.1.7

16
requirements.txt
View File

@ -1,12 +1,16 @@
-e .
appdirs==1.4.3
certifi==2017.4.17
chardet==3.0.4
colorama==0.3.9
enum34==1.1.6
fs==2.0.3
pbr==3.0.0
fs==2.0.4
idna==2.5
packaging==16.8
pbr==3.1.1
psutil==5.2.2
pyparsing==2.2.0
pytz==2017.2
requests==2.13.0
requests==2.18.1
six==1.10.0
stevedore==1.21.0
stevedore==1.23.0
urllib3==1.21.1

43
setup.py
View File

@ -18,13 +18,19 @@ except NameError:
# Get the long description from the README file
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
long_description = f.read()
try:
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
long_description = f.read()
except:
long_description = ''
# Get the classifiers from the classifiers file
tolines = lambda c: list(filter(None, map(lambda s: s.strip(), c.split('\n'))))
with open(path.join(here, 'classifiers.txt'), encoding='utf-8') as f:
classifiers = tolines(f.read())
try:
with open(path.join(here, 'classifiers.txt'), encoding='utf-8') as f:
classifiers = tolines(f.read())
except:
classifiers = []
version_ns = {}
try:
@ -35,41 +41,36 @@ else:
version = version_ns.get('__version__', 'dev')
setup(
name='bonobo',
author='Romain Dorgueil',
author_email='romain@dorgueil.net',
description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for '
'python 3.5+.'),
license='Apache License, Version 2.0',
install_requires=[
'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0'
],
name='bonobo',
version=version,
long_description=long_description,
classifiers=classifiers,
packages=find_packages(exclude=['ez_setup', 'example', 'test']),
include_package_data=True,
data_files=[
(
'share/jupyter/nbextensions/bonobo-jupyter', [
'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js',
'bonobo/ext/jupyter/static/index.js.map'
]
)
install_requires=[
'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'packaging (>= 16, < 17)', 'psutil (>= 5.2, < 6.0)',
'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)'
],
extras_require={
'dev': [
'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx',
'sphinx_rtd_theme', 'yapf'
'cookiecutter (>= 1.5, < 1.6)', 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)',
'pytest-cov (>= 2.5, < 3.0)', 'pytest-sugar (>= 0.8, < 0.9)', 'pytest-timeout (>= 1, < 2)',
'sphinx (>= 1.6, < 2.0)'
],
'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5']
'docker': ['bonobo-docker'],
'jupyter': ['ipywidgets (>= 6.0.0, < 7)', 'jupyter (>= 1.0, < 1.1)']
},
entry_points={
'bonobo.commands': [
'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register',
'version = bonobo.commands.version:register'
],
'console_scripts': ['bonobo = bonobo.commands:entrypoint'],
'edgy.project.features': ['bonobo = '
'bonobo.ext.edgy.project.feature:BonoboFeature']
'console_scripts': ['bonobo = bonobo.commands:entrypoint']
},
url='https://www.bonobo-project.org/',
download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version),

0
bonobo/ext/edgy/project/__init__.py → tests/__init__.py
View File

57
tests/test_config.py → tests/config/test_configurables.py
View File

@ -2,7 +2,6 @@ import pytest
from bonobo.config.configurables import Configurable
from bonobo.config.options import Option
from bonobo.config.services import Container, Service, validate_service_name
class MyConfigurable(Configurable):
@ -25,28 +24,6 @@ class MyConfigurableUsingPositionalOptions(MyConfigurable):
third = Option(str, required=False, positional=True)
class PrinterInterface():
def print(self, *args):
raise NotImplementedError()
class ConcretePrinter(PrinterInterface):
def __init__(self, prefix):
self.prefix = prefix
def print(self, *args):
return ';'.join((self.prefix, *args))
class MyServiceDependantConfigurable(Configurable):
printer = Service(
PrinterInterface,
)
def __call__(self, printer: PrinterInterface, *args):
return printer.print(*args)
def test_missing_required_option_error():
with pytest.raises(TypeError) as exc:
MyConfigurable()
@ -107,39 +84,5 @@ def test_option_resolution_order():
assert o.integer == None
def test_service_name_validator():
assert validate_service_name('foo') == 'foo'
assert validate_service_name('foo.bar') == 'foo.bar'
assert validate_service_name('Foo') == 'Foo'
assert validate_service_name('Foo.Bar') == 'Foo.Bar'
assert validate_service_name('Foo.a0') == 'Foo.a0'
with pytest.raises(ValueError):
validate_service_name('foo.0')
with pytest.raises(ValueError):
validate_service_name('0.foo')
SERVICES = Container(
printer0=ConcretePrinter(prefix='0'),
printer1=ConcretePrinter(prefix='1'),
)
def test_service_dependency():
o = MyServiceDependantConfigurable(printer='printer0')
assert o(SERVICES.get('printer0'), 'foo', 'bar') == '0;foo;bar'
assert o(SERVICES.get('printer1'), 'bar', 'baz') == '1;bar;baz'
assert o(*SERVICES.args_for(o), 'foo', 'bar') == '0;foo;bar'
def test_service_dependency_unavailable():
o = MyServiceDependantConfigurable(printer='printer2')
with pytest.raises(KeyError):
SERVICES.args_for(o)
def test_option_positional():
o = MyConfigurableUsingPositionalOptions('1', '2', '3', required_str='hello')

7
tests/test_config_method.py → tests/config/test_methods.py
View File

@ -28,7 +28,10 @@ def test_define_with_decorator():
def Concrete(self, *args, **kwargs):
calls.append((args, kwargs, ))
assert callable(Concrete.handler)
t = Concrete('foo', bar='baz')
assert callable(t.handler)
assert len(calls) == 0
t()
assert len(calls) == 1
@ -41,6 +44,7 @@ def test_define_with_argument():
calls.append((args, kwargs, ))
t = MethodBasedConfigurable('foo', bar='baz', handler=concrete_handler)
assert callable(t.handler)
assert len(calls) == 0
t()
assert len(calls) == 1
@ -54,6 +58,7 @@ def test_define_with_inheritance():
calls.append((args, kwargs, ))
t = Inheriting('foo', bar='baz')
assert callable(t.handler)
assert len(calls) == 0
t()
assert len(calls) == 1
@ -69,7 +74,9 @@ def test_inheritance_then_decorate():
def Concrete(self, *args, **kwargs):
calls.append((args, kwargs, ))
assert callable(Concrete.handler)
t = Concrete('foo', bar='baz')
assert callable(t.handler)
assert len(calls) == 0
t()
assert len(calls) == 1

0
tests/test_config_processors.py → tests/config/test_processors.py
View File

96
tests/config/test_services.py Normal file
View File

@ -0,0 +1,96 @@
import threading
import time
import pytest
from bonobo.config import Configurable, Container, Exclusive, Service
from bonobo.config.services import validate_service_name
class PrinterInterface():
def print(self, *args):
raise NotImplementedError()
class ConcretePrinter(PrinterInterface):
def __init__(self, prefix):
self.prefix = prefix
def print(self, *args):
return ';'.join((self.prefix, *args))
SERVICES = Container(
printer0=ConcretePrinter(prefix='0'),
printer1=ConcretePrinter(prefix='1'),
)
class MyServiceDependantConfigurable(Configurable):
printer = Service(
PrinterInterface,
)
def __call__(self, printer: PrinterInterface, *args):
return printer.print(*args)
def test_service_name_validator():
assert validate_service_name('foo') == 'foo'
assert validate_service_name('foo.bar') == 'foo.bar'
assert validate_service_name('Foo') == 'Foo'
assert validate_service_name('Foo.Bar') == 'Foo.Bar'
assert validate_service_name('Foo.a0') == 'Foo.a0'
with pytest.raises(ValueError):
validate_service_name('foo.0')
with pytest.raises(ValueError):
validate_service_name('0.foo')
def test_service_dependency():
o = MyServiceDependantConfigurable(printer='printer0')
assert o(SERVICES.get('printer0'), 'foo', 'bar') == '0;foo;bar'
assert o(SERVICES.get('printer1'), 'bar', 'baz') == '1;bar;baz'
assert o(*SERVICES.args_for(o), 'foo', 'bar') == '0;foo;bar'
def test_service_dependency_unavailable():
o = MyServiceDependantConfigurable(printer='printer2')
with pytest.raises(KeyError):
SERVICES.args_for(o)
class VCR:
def __init__(self):
self.tape = []
def append(self, x):
return self.tape.append(x)
def test_exclusive():
vcr = VCR()
vcr.append('hello')
def record(prefix, vcr=vcr):
with Exclusive(vcr):
for i in range(5):
vcr.append(' '.join((prefix, str(i))))
time.sleep(0.05)
threads = [threading.Thread(target=record, args=(str(i), )) for i in range(5)]
for thread in threads:
thread.start()
time.sleep(0.01) # this is not good practice, how to test this without sleeping ?? XXX
for thread in threads:
thread.join()
assert vcr.tape == [
'hello', '0 0', '0 1', '0 2', '0 3', '0 4', '1 0', '1 1', '1 2', '1 3', '1 4', '2 0', '2 1', '2 2', '2 3',
'2 4', '3 0', '3 1', '3 2', '3 3', '3 4', '4 0', '4 1', '4 2', '4 3', '4 4'
]

92
tests/io/test_csv.py
View File

@ -1,42 +1,56 @@
import pytest
from bonobo import Bag, CsvReader, CsvWriter, open_fs
from bonobo import Bag, CsvReader, CsvWriter, settings
from bonobo.constants import BEGIN, END
from bonobo.execution.node import NodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
csv_tester = FilesystemTester('csv')
csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar'
def test_write_csv_to_file(tmpdir):
fs, filename = open_fs(tmpdir), 'output.csv'
def test_write_csv_to_file_arg0(tmpdir):
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
writer = CsvWriter(path=filename)
context = NodeExecutionContext(writer, services={'fs': fs})
with NodeExecutionContext(CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
context.step()
context.step()
context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
context.start()
context.step()
context.step()
context.stop()
assert fs.open(filename).read() == 'foo\nbar\nbaz\n'
with fs.open(filename) as fp:
assert fp.read() == 'foo\nbar\nbaz\n'
with pytest.raises(AttributeError):
getattr(context, 'file')
def test_read_csv_from_file(tmpdir):
fs, filename = open_fs(tmpdir), 'input.csv'
fs.open(filename, 'w').write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar')
@pytest.mark.parametrize('add_kwargs', ({}, {
'ioformat': settings.IOFORMAT_KWARGS,
}, ))
def test_write_csv_to_file_kwargs(tmpdir, add_kwargs):
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
reader = CsvReader(path=filename, delimiter=',')
with NodeExecutionContext(CsvWriter(path=filename, **add_kwargs), services=services) as context:
context.write(BEGIN, Bag(**{'foo': 'bar'}), Bag(**{'foo': 'baz', 'ignore': 'this'}), END)
context.step()
context.step()
context = CapturingNodeExecutionContext(reader, services={'fs': fs})
with fs.open(filename) as fp:
assert fp.read() == 'foo\nbar\nbaz\n'
context.start()
context.write(BEGIN, Bag(), END)
context.step()
context.stop()
with pytest.raises(AttributeError):
getattr(context, 'file')
def test_read_csv_from_file_arg0(tmpdir):
fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
with CapturingNodeExecutionContext(
CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0),
services=services,
) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2
@ -55,3 +69,35 @@ def test_read_csv_from_file(tmpdir):
'b': 'b bar',
'c': 'c bar',
}
def test_read_csv_from_file_kwargs(tmpdir):
fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
with CapturingNodeExecutionContext(
CsvReader(path=filename, delimiter=','),
services=services,
) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2
args0, kwargs0 = context.send.call_args_list[0]
assert len(args0) == 1 and not len(kwargs0)
args1, kwargs1 = context.send.call_args_list[1]
assert len(args1) == 1 and not len(kwargs1)
_args, _kwargs = args0[0].get()
assert not len(_args) and _kwargs == {
'a': 'a foo',
'b': 'b foo',
'c': 'c foo',
}
_args, _kwargs = args1[0].get()
assert not len(_args) and _kwargs == {
'a': 'a bar',
'b': 'b bar',
'c': 'c bar',
}

65
tests/io/test_file.py
View File

@ -1,9 +1,22 @@
import pytest
from bonobo import Bag, FileReader, FileWriter, open_fs
from bonobo import Bag, FileReader, FileWriter
from bonobo.constants import BEGIN, END
from bonobo.execution.node import NodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
txt_tester = FilesystemTester('txt')
txt_tester.input_data = 'Hello\nWorld\n'
def test_file_writer_contextless(tmpdir):
fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
with FileWriter(path=filename).open(fs) as fp:
fp.write('Yosh!')
with fs.open(filename) as fp:
assert fp.read() == 'Yosh!'
@pytest.mark.parametrize(
@ -14,43 +27,23 @@ from bonobo.util.testing import CapturingNodeExecutionContext
]
)
def test_file_writer_in_context(tmpdir, lines, output):
fs, filename = open_fs(tmpdir), 'output.txt'
fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
writer = FileWriter(path=filename)
context = NodeExecutionContext(writer, services={'fs': fs})
with NodeExecutionContext(FileWriter(path=filename), services=services) as context:
context.write(BEGIN, *map(Bag, lines), END)
for _ in range(len(lines)):
context.step()
context.start()
context.write(BEGIN, *map(Bag, lines), END)
for _ in range(len(lines)):
with fs.open(filename) as fp:
assert fp.read() == output
def test_file_reader(tmpdir):
fs, filename, services = txt_tester.get_services_for_reader(tmpdir)
with CapturingNodeExecutionContext(FileReader(path=filename), services=services) as context:
context.write(BEGIN, Bag(), END)
context.step()
context.stop()
assert fs.open(filename).read() == output
def test_file_writer_out_of_context(tmpdir):
fs, filename = open_fs(tmpdir), 'output.txt'
writer = FileWriter(path=filename)
with writer.open(fs) as fp:
fp.write('Yosh!')
assert fs.open(filename).read() == 'Yosh!'
def test_file_reader_in_context(tmpdir):
fs, filename = open_fs(tmpdir), 'input.txt'
fs.open(filename, 'w').write('Hello\nWorld\n')
reader = FileReader(path=filename)
context = CapturingNodeExecutionContext(reader, services={'fs': fs})
context.start()
context.write(BEGIN, Bag(), END)
context.step()
context.stop()
assert len(context.send.mock_calls) == 2

60
tests/io/test_json.py
View File

@ -1,42 +1,48 @@
import pytest
from bonobo import Bag, JsonReader, JsonWriter, open_fs
from bonobo import Bag, JsonReader, JsonWriter, settings
from bonobo.constants import BEGIN, END
from bonobo.execution.node import NodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
json_tester = FilesystemTester('json')
json_tester.input_data = '''[{"x": "foo"},{"x": "bar"}]'''
def test_write_json_to_file(tmpdir):
fs, filename = open_fs(tmpdir), 'output.json'
def test_write_json_arg0(tmpdir):
fs, filename, services = json_tester.get_services_for_writer(tmpdir)
writer = JsonWriter(path=filename)
context = NodeExecutionContext(writer, services={'fs': fs})
with NodeExecutionContext(JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
context.write(BEGIN, Bag({'foo': 'bar'}), END)
context.step()
context.start()
context.write(BEGIN, Bag({'foo': 'bar'}), END)
context.step()
context.stop()
assert fs.open(filename).read() == '[{"foo": "bar"}]'
with pytest.raises(AttributeError):
getattr(context, 'file')
with pytest.raises(AttributeError):
getattr(context, 'first')
with fs.open(filename) as fp:
assert fp.read() == '[{"foo": "bar"}]'
def test_read_json_from_file(tmpdir):
fs, filename = open_fs(tmpdir), 'input.json'
fs.open(filename, 'w').write('[{"x": "foo"},{"x": "bar"}]')
reader = JsonReader(path=filename)
@pytest.mark.parametrize('add_kwargs', ({}, {
'ioformat': settings.IOFORMAT_KWARGS,
}, ))
def test_write_json_kwargs(tmpdir, add_kwargs):
fs, filename, services = json_tester.get_services_for_writer(tmpdir)
context = CapturingNodeExecutionContext(reader, services={'fs': fs})
with NodeExecutionContext(JsonWriter(filename, **add_kwargs), services=services) as context:
context.write(BEGIN, Bag(**{'foo': 'bar'}), END)
context.step()
context.start()
context.write(BEGIN, Bag(), END)
context.step()
context.stop()
with fs.open(filename) as fp:
assert fp.read() == '[{"foo": "bar"}]'
def test_read_json_arg0(tmpdir):
fs, filename, services = json_tester.get_services_for_reader(tmpdir)
with CapturingNodeExecutionContext(
JsonReader(filename, ioformat=settings.IOFORMAT_ARG0),
services=services,
) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2

54
tests/io/test_pickle.py Normal file
View File

@ -0,0 +1,54 @@
import pickle
import pytest
from bonobo import Bag, PickleReader, PickleWriter, settings
from bonobo.constants import BEGIN, END
from bonobo.execution.node import NodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
pickle_tester = FilesystemTester('pkl', mode='wb')
pickle_tester.input_data = pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']])
def test_write_pickled_dict_to_file(tmpdir):
fs, filename, services = pickle_tester.get_services_for_writer(tmpdir)
with NodeExecutionContext(PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
context.step()
context.step()
with fs.open(filename, 'rb') as fp:
assert pickle.loads(fp.read()) == {'foo': 'bar'}
with pytest.raises(AttributeError):
getattr(context, 'file')
def test_read_pickled_list_from_file(tmpdir):
fs, filename, services = pickle_tester.get_services_for_reader(tmpdir)
with CapturingNodeExecutionContext(
PickleReader(filename, ioformat=settings.IOFORMAT_ARG0), services=services
) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2
args0, kwargs0 = context.send.call_args_list[0]
assert len(args0) == 1 and not len(kwargs0)
args1, kwargs1 = context.send.call_args_list[1]
assert len(args1) == 1 and not len(kwargs1)
assert args0[0].args[0] == {
'a': 'a foo',
'b': 'b foo',
'c': 'c foo',
}
assert args1[0].args[0] == {
'a': 'a bar',
'b': 'b bar',
'c': 'c bar',
}

62
tests/test_commands.py
View File

@ -1,10 +1,26 @@
import runpy
import sys
from unittest.mock import patch
import pkg_resources
import pytest
from bonobo import get_examples_path
from bonobo import __main__, __version__, get_examples_path
from bonobo.commands import entrypoint
def runner_entrypoint(*args):
return entrypoint(list(args))
def runner_module(*args):
with patch.object(sys, 'argv', ['bonobo', *args]):
return runpy.run_path(__main__.__file__, run_name='__main__')
all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module])
def test_entrypoint():
commands = {}
@ -13,23 +29,51 @@ def test_entrypoint():
assert 'init' in commands
assert 'run' in commands
assert 'version' in commands
def test_no_command(capsys):
@all_runners
def test_no_command(runner, capsys):
with pytest.raises(SystemExit):
entrypoint([])
runner()
_, err = capsys.readouterr()
assert 'error: the following arguments are required: command' in err
def test_init():
pass # need ext dir
def test_run(capsys):
entrypoint(['run', '--quiet', get_examples_path('types/strings.py')])
@all_runners
def test_run(runner, capsys):
runner('run', '--quiet', get_examples_path('types/strings.py'))
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0].startswith('Foo ')
assert out[1].startswith('Bar ')
assert out[2].startswith('Baz ')
@all_runners
def test_run_module(runner, capsys):
runner('run', '--quiet', '-m', 'bonobo.examples.types.strings')
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0].startswith('Foo ')
assert out[1].startswith('Bar ')
assert out[2].startswith('Baz ')
@all_runners
def test_run_path(runner, capsys):
runner('run', '--quiet', get_examples_path('types'))
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0].startswith('Foo ')
assert out[1].startswith('Bar ')
assert out[2].startswith('Baz ')
@all_runners
def test_version(runner, capsys):
runner('version')
out, err = capsys.readouterr()
out = out.strip()
assert out.startswith('bonobo ')
assert __version__ in out

4
tests/test_publicapi.py
View File

@ -1,4 +1,4 @@
import types
import inspect
def test_wildcard_import():
@ -10,7 +10,7 @@ def test_wildcard_import():
if name.startswith('_'):
continue
attr = getattr(bonobo, name)
if isinstance(attr, types.ModuleType):
if inspect.ismodule(attr):
continue
assert name in bonobo.__all__

58
tests/util/test_objects.py
View File

@ -1,4 +1,9 @@
import operator
import pytest
from bonobo.util.objects import Wrapper, get_name, ValueHolder
from bonobo.util.testing import optional_contextmanager
class foo:
@ -52,3 +57,56 @@ def test_valueholder():
assert y == x
assert y is not x
assert repr(x) == repr(y) == repr(43)
unsupported_operations = {
int: {operator.matmul},
str: {
operator.sub, operator.mul, operator.matmul, operator.floordiv, operator.truediv, operator.mod, divmod,
operator.pow, operator.lshift, operator.rshift, operator.and_, operator.xor, operator.or_
},
}
@pytest.mark.parametrize('x,y', [(5, 3), (0, 10), (0, 0), (1, 1), ('foo', 'bar'), ('', 'baz!')])
@pytest.mark.parametrize(
'operation,inplace_operation', [
(operator.add, operator.iadd),
(operator.sub, operator.isub),
(operator.mul, operator.imul),
(operator.matmul, operator.imatmul),
(operator.truediv, operator.itruediv),
(operator.floordiv, operator.ifloordiv),
(operator.mod, operator.imod),
(divmod, None),
(operator.pow, operator.ipow),
(operator.lshift, operator.ilshift),
(operator.rshift, operator.irshift),
(operator.and_, operator.iand),
(operator.xor, operator.ixor),
(operator.or_, operator.ior),
]
)
def test_valueholder_integer_operations(x, y, operation, inplace_operation):
v = ValueHolder(x)
is_supported = operation not in unsupported_operations.get(type(x), set())
isdiv = ('div' in operation.__name__) or ('mod' in operation.__name__)
# forward...
with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported):
with optional_contextmanager(pytest.raises(ZeroDivisionError), ignore=y or not isdiv):
assert operation(x, y) == operation(v, y)
# backward...
with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported):
with optional_contextmanager(pytest.raises(ZeroDivisionError), ignore=x or not isdiv):
assert operation(y, x) == operation(y, v)
# in place...
if inplace_operation is not None:
with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported):
with optional_contextmanager(pytest.raises(ZeroDivisionError), ignore=y or not isdiv):
inplace_operation(v, y)
assert v == operation(x, y)