paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #132 from hartym/develop

Browse Source 
Refactoring of configurables.
This commit is contained in:
Romain Dorgueil
2017-07-05 13:13:39 +02:00
committed by GitHub
parent 285f8e230c 4a2c7280d6
commit c39558be5d
54 changed files with 1305 additions and 689 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@ -2,6 +2,7 @@
*,cover
*.egg
*.egg-info/
*.iml
*.log
*.manifest
*.mo
@ -20,25 +21,17 @@
.installed.cfg
.ipynb_checkpoints
.python-version
.tox/
.webassets-cache
/.idea
/.release
/bonobo.iml
/bonobo/examples/work_in_progress/
/bonobo/ext/jupyter/js/node_modules/
/build/
/coverage.xml
/develop-eggs/
/dist/
/docs/_build/
/downloads/
/eggs/
/examples/private
/htmlcov/
/sdist/
/tags
celerybeat-schedule
parts/
pip-delete-this-directory.txt
pip-log.txt

4
Makefile
View File

@ -1,7 +1,7 @@
# This file has been auto-generated.
# All changes will be lost, see Projectfile.
#
# Updated at 2017-06-10 15:15:34.093885
# Updated at 2017-07-04 10:50:55.775681
PACKAGE ?= bonobo
PYTHON ?= $(shell which python)
@ -18,7 +18,7 @@ SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build
SPHINX_OPTIONS ?=
SPHINX_SOURCEDIR ?= docs
SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
YAPF ?= $(PYTHON_DIRNAME)/yapf
YAPF ?= $(PYTHON) -m yapf
YAPF_OPTIONS ?= -rip
VERSION ?= $(shell git describe 2>/dev/null || echo dev)

12
bonobo/_api.py
View File

@ -1,6 +1,6 @@
from bonobo.structs import Bag, Graph, Token
from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
PrettyPrinter, PickleWriter, PickleReader, Tee, count, identity, noop, pprint
PrettyPrinter, PickleWriter, PickleReader, RateLimited, Tee, count, identity, noop
from bonobo.strategies import create_strategy
from bonobo.util.objects import get_name
@ -45,7 +45,7 @@ def run(graph, strategy=None, plugins=None, services=None):
from bonobo import settings
settings.check()
if not settings.QUIET: # pragma: no cover
if not settings.QUIET.get(): # pragma: no cover
if _is_interactive_console():
from bonobo.ext.console import ConsoleOutputPlugin
if ConsoleOutputPlugin not in plugins:
@ -68,7 +68,7 @@ register_api(create_strategy)
# Shortcut to filesystem2's open_fs, that we make available there for convenience.
@register_api
def open_fs(fs_url, *args, **kwargs):
def open_fs(fs_url=None, *args, **kwargs):
"""
Wraps :func:`fs.open_fs` function with a few candies.
@ -83,6 +83,10 @@ def open_fs(fs_url, *args, **kwargs):
"""
from fs import open_fs as _open_fs
from os.path import expanduser
from os import getcwd
if fs_url is None:
fs_url = getcwd()
return _open_fs(expanduser(str(fs_url)), *args, **kwargs)
@ -100,11 +104,11 @@ register_api_group(
PrettyPrinter,
PickleReader,
PickleWriter,
RateLimited,
Tee,
count,
identity,
noop,
pprint,
)

2
bonobo/_version.py
View File

@ -1 +1 @@
__version__ = '0.4.0'
__version__ = '0.4.3'

6
bonobo/commands/__init__.py
View File

@ -27,9 +27,9 @@ def entrypoint(args=None):
args = parser.parse_args(args).__dict__
if args.pop('debug', False):
settings.DEBUG = True
settings.LOGGING_LEVEL = logging.DEBUG
logging.set_level(settings.LOGGING_LEVEL)
settings.DEBUG.set(True)
settings.LOGGING_LEVEL.set(logging.DEBUG)
logging.set_level(settings.LOGGING_LEVEL.get())
logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args))
commands[args.pop('command')](**args)

4
bonobo/commands/run.py
View File

@ -31,10 +31,10 @@ def execute(filename, module, install=False, quiet=False, verbose=False):
from bonobo import Graph, run, settings
if quiet:
settings.QUIET = True
settings.QUIET.set(True)
if verbose:
settings.DEBUG = True
settings.DEBUG.set(True)
if filename:
if os.path.isdir(filename):

7
bonobo/config/__init__.py
View File

@ -1,9 +1,9 @@
from bonobo.config.configurables import Configurable
from bonobo.config.options import Option, Method
from bonobo.config.options import Method, Option
from bonobo.config.processors import ContextProcessor
from bonobo.config.services import Container, Service, Exclusive
from bonobo.config.services import Container, Exclusive, Service, requires
# bonobo.config public programming interface
# Bonobo's Config API
__all__ = [
'Configurable',
'Container',
@ -12,4 +12,5 @@ __all__ = [
'Method',
'Option',
'Service',
'requires',
]

199
bonobo/config/configurables.py
View File

@ -1,12 +1,14 @@
from bonobo.config.options import Method, Option
from bonobo.config.processors import ContextProcessor
from bonobo.errors import ConfigurationError, AbstractError
from bonobo.util.inspect import isoption, iscontextprocessor
from bonobo.errors import AbstractError
from bonobo.util.collections import sortedlist
__all__ = [
'Configurable',
'Option',
]
get_creation_counter = lambda v: v._creation_counter
class ConfigurableMeta(type):
"""
@ -15,36 +17,78 @@ class ConfigurableMeta(type):
def __init__(cls, what, bases=None, dict=None):
super().__init__(what, bases, dict)
cls.__options__ = {}
cls.__positional_options__ = []
cls.__processors__ = []
cls.__wrappable__ = None
cls.__processors = sortedlist()
cls.__methods = sortedlist()
cls.__options = sortedlist()
cls.__names = set()
# cls.__kwoptions = []
for typ in cls.__mro__:
for name, value in typ.__dict__.items():
if isinstance(value, Option):
if isinstance(value, ContextProcessor):
cls.__processors__.append(value)
else:
if not value.name:
value.name = name
for name, value in filter(lambda x: isoption(x[1]), typ.__dict__.items()):
if iscontextprocessor(value):
cls.__processors.insort((value._creation_counter, value))
continue
if isinstance(value, Method):
if cls.__wrappable__:
raise ConfigurationError(
'Cannot define more than one "Method" option in a configurable. That may change in the future.'
)
cls.__wrappable__ = name
if not value.name:
value.name = name
if not name in cls.__options__:
cls.__options__[name] = value
if not name in cls.__names:
cls.__names.add(name)
cls.__options.insort((not value.positional, value._creation_counter, name, value))
if value.positional:
cls.__positional_options__.append(name)
@property
def __options__(cls):
return ((name, option) for _, _, name, option in cls.__options)
# This can be done before, more efficiently. Not so bad neither as this is only done at type() creation time
# (aka class Xxx(...) time) and there should not be hundreds of processors. Still not very elegant.
cls.__processors__ = sorted(cls.__processors__, key=lambda v: v._creation_counter)
@property
def __options_dict__(cls):
return dict(cls.__options__)
@property
def __processors__(cls):
return (processor for _, processor in cls.__processors)
def __repr__(self):
return ' '.join(('<Configurable', super(ConfigurableMeta, self).__repr__().split(' ', 1)[1], ))
try:
import _functools
except:
import functools
PartiallyConfigured = functools.partial
else:
class PartiallyConfigured(_functools.partial):
@property # TODO XXX cache this shit
def _options_values(self):
""" Simulate option values for partially configured objects. """
try:
return self.__options_values
except AttributeError:
self.__options_values = {**self.keywords}
position = 0
for name, option in self.func.__options__:
if not option.positional:
break # no positional left
if name in self.keywords:
continue # already fulfilled
self.__options_values[name] = self.args[position] if len(self.args) >= position + 1 else None
position += 1
return self.__options_values
def __getattr__(self, item):
_dict = self.func.__options_dict__
if item in _dict:
return _dict[item].__get__(self, self.func)
return getattr(self.func, item)
class Configurable(metaclass=ConfigurableMeta):
@ -54,61 +98,106 @@ class Configurable(metaclass=ConfigurableMeta):
"""
def __new__(cls, *args, **kwargs):
if cls.__wrappable__ and len(args) == 1 and hasattr(args[0], '__call__'):
return type(args[0].__name__, (cls, ), {cls.__wrappable__: args[0]})
def __new__(cls, *args, _final=False, **kwargs):
"""
Custom instance builder. If not all options are fulfilled, will return a :class:`PartiallyConfigured` instance
which is just a :class:`functools.partial` object that behaves like a :class:`Configurable` instance.
return super(Configurable, cls).__new__(cls)
def __init__(self, *args, **kwargs):
super().__init__()
# initialize option's value dictionary, used by descriptor implementation (see Option).
self.__options_values__ = {}
The special `_final` argument can be used to force final instance to be created, or an error raised if options
are missing.
:param args:
:param _final: bool
:param kwargs:
:return: Configurable or PartiallyConfigured
"""
options = tuple(cls.__options__)
# compute missing options, given the kwargs.
missing = set()
for name, option in type(self).__options__.items():
for name, option in options:
if option.required and not option.name in kwargs:
missing.add(name)
# transform positional arguments in keyword arguments if possible.
position = 0
for positional_option in self.__positional_options__:
if len(args) <= position:
break
kwargs[positional_option] = args[position]
position += 1
if positional_option in missing:
missing.remove(positional_option)
for name, option in options:
if not option.positional:
break # option orders make all positional options first, job done.
# complain if there are still missing options.
if len(missing):
raise TypeError(
'{}() missing {} required option{}: {}.'.format(
type(self).__name__,
len(missing), 's' if len(missing) > 1 else '', ', '.join(map(repr, sorted(missing)))
)
)
if not isoption(getattr(cls, name)):
missing.remove(name)
continue
if len(args) <= position:
break # no more positional arguments given.
position += 1
if name in missing:
missing.remove(name)
# complain if there is more options than possible.
extraneous = set(kwargs.keys()) - set(type(self).__options__.keys())
extraneous = set(kwargs.keys()) - (set(next(zip(*options))) if len(options) else set())
if len(extraneous):
raise TypeError(
'{}() got {} unexpected option{}: {}.'.format(
type(self).__name__,
cls.__name__,
len(extraneous), 's' if len(extraneous) > 1 else '', ', '.join(map(repr, sorted(extraneous)))
)
)
# missing options? we'll return a partial instance to finish the work later, unless we're required to be
# "final".
if len(missing):
if _final:
raise TypeError(
'{}() missing {} required option{}: {}.'.format(
cls.__name__,
len(missing), 's' if len(missing) > 1 else '', ', '.join(map(repr, sorted(missing)))
)
)
return PartiallyConfigured(cls, *args, **kwargs)
return super(Configurable, cls).__new__(cls)
def __init__(self, *args, **kwargs):
# initialize option's value dictionary, used by descriptor implementation (see Option).
self._options_values = {**kwargs}
# set option values.
for name, value in kwargs.items():
setattr(self, name, value)
position = 0
for name, option in self.__options__:
if not option.positional:
break # option orders make all positional options first
# value was overriden? Skip.
maybe_value = getattr(type(self), name)
if not isoption(maybe_value):
continue
if len(args) <= position:
break
if name in self._options_values:
raise ValueError('Already got a value for option {}'.format(name))
setattr(self, name, args[position])
position += 1
def __call__(self, *args, **kwargs):
""" You can implement a configurable callable behaviour by implemenenting the call(...) method. Of course, it is also backward compatible with legacy __call__ override.
"""
return self.call(*args, **kwargs)
@property
def __options__(self):
return type(self).__options__
@property
def __processors__(self):
return type(self).__processors__
def call(self, *args, **kwargs):
raise AbstractError('Not implemented.')

59
bonobo/config/options.py
View File

@ -1,3 +1,6 @@
from bonobo.util.inspect import istype
class Option:
"""
An Option is a descriptor for Configurable's parameters.
@ -14,7 +17,9 @@ class Option:
If an option is required, an error will be raised if no value is provided (at runtime). If it is not, option
will have the default value if user does not override it at runtime.
(default: False)
Ignored if a default is provided, meaning that the option cannot be required.
(default: True)
.. attribute:: positional
@ -48,10 +53,10 @@ class Option:
_creation_counter = 0
def __init__(self, type=None, *, required=False, positional=False, default=None):
def __init__(self, type=None, *, required=True, positional=False, default=None):
self.name = None
self.type = type
self.required = required
self.required = required if default is None else False
self.positional = positional
self.default = default
@ -60,12 +65,27 @@ class Option:
Option._creation_counter += 1
def __get__(self, inst, typ):
if not self.name in inst.__options_values__:
inst.__options_values__[self.name] = self.get_default()
return inst.__options_values__[self.name]
# XXX If we call this on the type, then either return overriden value or ... ???
if inst is None:
return vars(type).get(self.name, self)
if not self.name in inst._options_values:
inst._options_values[self.name] = self.get_default()
return inst._options_values[self.name]
def __set__(self, inst, value):
inst.__options_values__[self.name] = self.clean(value)
inst._options_values[self.name] = self.clean(value)
def __repr__(self):
return '<{positional}{typename} {type}{name} default={default!r}{required}>'.format(
typename=type(self).__name__,
type='({})'.format(self.type) if istype(self.type) else '',
name=self.name,
positional='*' if self.positional else '**',
default=self.default,
required=' (required)' if self.required else '',
)
def clean(self, value):
return self.type(value) if self.type else value
@ -105,20 +125,17 @@ class Method(Option):
"""
def __init__(self):
super().__init__(None, required=False)
def __get__(self, inst, typ):
if not self.name in inst.__options_values__:
inst.__options_values__[self.name] = getattr(inst, self.name)
return inst.__options_values__[self.name]
def __init__(self, *, required=True, positional=True):
super().__init__(None, required=required, positional=positional)
def __set__(self, inst, value):
if isinstance(value, str):
raise ValueError('should be callable')
inst.__options_values__[self.name] = self.type(value) if self.type else value
def clean(self, value):
if not hasattr(value, '__call__'):
raise ValueError('{} value must be callable.'.format(type(self).__name__))
return value
raise TypeError(
'Option of type {!r} is expecting a callable value, got {!r} object (which is not).'.
format(type(self).__name__, type(value).__name__)
)
inst._options_values[self.name] = self.type(value) if self.type else value
def __call__(self, *args, **kwargs):
# only here to trick IDEs into thinking this is callable.
raise NotImplementedError('You cannot call the descriptor')

10
bonobo/config/processors.py
View File

@ -74,8 +74,7 @@ class ContextCurrifier:
def __init__(self, wrapped, *initial_context):
self.wrapped = wrapped
self.context = tuple(initial_context)
self._stack = []
self._stack_values = []
self._stack, self._stack_values = None, None
def __iter__(self):
yield from self.wrapped
@ -86,8 +85,10 @@ class ContextCurrifier:
return self.wrapped(*self.context, *args, **kwargs)
def setup(self, *context):
if len(self._stack):
if self._stack is not None:
raise RuntimeError('Cannot setup context currification twice.')
self._stack, self._stack_values = list(), list()
for processor in resolve_processors(self.wrapped):
_processed = processor(self.wrapped, *context, *self.context)
_append_to_context = next(_processed)
@ -97,7 +98,7 @@ class ContextCurrifier:
self._stack.append(_processed)
def teardown(self):
while len(self._stack):
while self._stack:
processor = self._stack.pop()
try:
# todo yield from ? how to ?
@ -108,6 +109,7 @@ class ContextCurrifier:
else:
# No error ? We should have had StopIteration ...
raise RuntimeError('Context processors should not yield more than once.')
self._stack, self._stack_values = None, None
@contextmanager
def as_contextmanager(self, *context):

26
bonobo/config/services.py
View File

@ -53,10 +53,14 @@ class Service(Option):
super().__init__(str, required=False, default=name)
def __set__(self, inst, value):
inst.__options_values__[self.name] = validate_service_name(value)
inst._options_values[self.name] = validate_service_name(value)
def resolve(self, inst, services):
return services.get(getattr(inst, self.name))
try:
name = getattr(inst, self.name)
except AttributeError:
name = self.name
return services.get(name)
class Container(dict):
@ -71,7 +75,7 @@ class Container(dict):
def args_for(self, mixed):
try:
options = mixed.__options__
options = dict(mixed.__options__)
except AttributeError:
options = {}
@ -126,3 +130,19 @@ class Exclusive(ContextDecorator):
def __exit__(self, *exc):
self.get_lock().release()
def requires(*service_names):
def decorate(mixed):
try:
options = mixed.__options__
except AttributeError:
mixed.__options__ = options = {}
for service_name in service_names:
service = Service(service_name)
service.name = service_name
options[service_name] = service
return mixed
return decorate

19
bonobo/errors.py
View File

@ -58,5 +58,22 @@ class ConfigurationError(Exception):
pass
class MissingServiceImplementationError(KeyError):
class UnrecoverableError(Exception):
"""Flag for errors that must interrupt the workflow, either because they will happen for sure on each node run, or
because you know that your transformation has no point continuing runnning after a bad event."""
class UnrecoverableValueError(UnrecoverableError, ValueError):
pass
class UnrecoverableRuntimeError(UnrecoverableError, RuntimeError):
pass
class UnrecoverableNotImplementedError(UnrecoverableError, NotImplementedError):
pass
class MissingServiceImplementationError(UnrecoverableError, KeyError):
pass

250
bonobo/examples/datasets/coffeeshops.json
View File

@ -1,182 +1,182 @@
{"Le Reynou": "2 bis quai de la m\u00e9gisserie, 75001 Paris, France",
"les montparnos": "65 boulevard Pasteur, 75015 Paris, France",
"Le Saint Jean": "23 rue des abbesses, 75018 Paris, France",
"Le Felteu": "1 rue Pecquay, 75004 Paris, France",
{"les montparnos": "65 boulevard Pasteur, 75015 Paris, France",
"Coffee Chope": "344Vrue Vaugirard, 75015 Paris, France",
"Caf\u00e9 Lea": "5 rue Claude Bernard, 75005 Paris, France",
"Le Bellerive": "71 quai de Seine, 75019 Paris, France",
"Le drapeau de la fidelit\u00e9": "21 rue Copreaux, 75015 Paris, France",
"O q de poule": "53 rue du ruisseau, 75018 Paris, France",
"Le caf\u00e9 des amis": "125 rue Blomet, 75015 Paris, France",
"Le chantereine": "51 Rue Victoire, 75009 Paris, France",
"Le M\u00fcller": "11 rue Feutrier, 75018 Paris, France",
"La Caravane": "Rue de la Fontaine au Roi, 75011 Paris, France",
"Le Pas Sage": "1 Passage du Grand Cerf, 75002 Paris, France",
"La Renaissance": "112 Rue Championnet, 75018 Paris, France",
"Ext\u00e9rieur Quai": "5, rue d'Alsace, 75010 Paris, France",
"Le Sully": "6 Bd henri IV, 75004 Paris, France",
"Le drapeau de la fidelit\u00e9": "21 rue Copreaux, 75015 Paris, France",
"Le caf\u00e9 des amis": "125 rue Blomet, 75015 Paris, France",
"Le Kleemend's": "34 avenue Pierre Mend\u00e8s-France, 75013 Paris, France",
"Assaporare Dix sur Dix": "75, avenue Ledru-Rollin, 75012 Paris, France",
"Caf\u00e9 Pierre": "202 rue du faubourg st antoine, 75012 Paris, France",
"Le Caf\u00e9 Livres": "10 rue Saint Martin, 75004 Paris, France",
"Le Chaumontois": "12 rue Armand Carrel, 75018 Paris, France",
"Le Square": "31 rue Saint-Dominique, 75007 Paris, France",
"Les Arcades": "61 rue de Ponthieu, 75008 Paris, France",
"La Bauloise": "36 rue du hameau, 75015 Paris, France",
"Le Dellac": "14 rue Rougemont, 75009 Paris, France",
"Le Bosquet": "46 avenue Bosquet, 75007 Paris, France",
"Le Sully": "6 Bd henri IV, 75004 Paris, France",
"Le Felteu": "1 rue Pecquay, 75004 Paris, France",
"Le bistrot de Ma\u00eblle et Augustin": "42 rue coquill\u00e8re, 75001 Paris, France",
"D\u00e9d\u00e9 la frite": "52 rue Notre-Dame des Victoires, 75002 Paris, France",
"Cardinal Saint-Germain": "11 boulevard Saint-Germain, 75005 Paris, France",
"Le Reynou": "2 bis quai de la m\u00e9gisserie, 75001 Paris, France",
"Aux cadrans": "21 ter boulevard Diderot, 75012 Paris, France",
"Le Saint Jean": "23 rue des abbesses, 75018 Paris, France",
"La Renaissance": "112 Rue Championnet, 75018 Paris, France",
"Le Square": "31 rue Saint-Dominique, 75007 Paris, France",
"Les Arcades": "61 rue de Ponthieu, 75008 Paris, France",
"Le Kleemend's": "34 avenue Pierre Mend\u00e8s-France, 75013 Paris, France",
"Assaporare Dix sur Dix": "75, avenue Ledru-Rollin, 75012 Paris, France",
"Caf\u00e9 Pierre": "202 rue du faubourg st antoine, 75012 Paris, France",
"Caf\u00e9 antoine": "17 rue Jean de la Fontaine, 75016 Paris, France",
"Au cerceau d'or": "129 boulevard sebastopol, 75002 Paris, France",
"Aux cadrans": "21 ter boulevard Diderot, 75012 Paris, France",
"Caf\u00e9 Lea": "5 rue Claude Bernard, 75005 Paris, France",
"Le Bellerive": "71 quai de Seine, 75019 Paris, France",
"La Bauloise": "36 rue du hameau, 75015 Paris, France",
"Le Dellac": "14 rue Rougemont, 75009 Paris, France",
"La Caravane": "Rue de la Fontaine au Roi, 75011 Paris, France",
"Le Pas Sage": "1 Passage du Grand Cerf, 75002 Paris, France",
"Le Caf\u00e9 Livres": "10 rue Saint Martin, 75004 Paris, France",
"Le Chaumontois": "12 rue Armand Carrel, 75018 Paris, France",
"Drole d'endroit pour une rencontre": "58 rue de Montorgueil, 75002 Paris, France",
"Le pari's caf\u00e9": "104 rue caulaincourt, 75018 Paris, France",
"Le Poulailler": "60 rue saint-sabin, 75011 Paris, France",
"Chai 33": "33 Cour Saint Emilion, 75012 Paris, France",
"L'Assassin": "99 rue Jean-Pierre Timbaud, 75011 Paris, France",
"l'Usine": "1 rue d'Avron, 75020 Paris, France",
"La Bricole": "52 rue Liebniz, 75018 Paris, France",
"le ronsard": "place maubert, 75005 Paris, France",
"Face Bar": "82 rue des archives, 75003 Paris, France",
"American Kitchen": "49 rue bichat, 75010 Paris, France",
"La Marine": "55 bis quai de valmy, 75010 Paris, France",
"Le Bloc": "21 avenue Brochant, 75017 Paris, France",
"La Recoleta au Manoir": "229 avenue Gambetta, 75020 Paris, France",
"Le Pareloup": "80 Rue Saint-Charles, 75015 Paris, France",
"La Brasserie Gait\u00e9": "3 rue de la Gait\u00e9, 75014 Paris, France",
"Caf\u00e9 Zen": "46 rue Victoire, 75009 Paris, France",
"O'Breizh": "27 rue de Penthi\u00e8vre, 75008 Paris, France",
"Le Petit Choiseul": "23 rue saint augustin, 75002 Paris, France",
"Invitez vous chez nous": "7 rue Ep\u00e9e de Bois, 75005 Paris, France",
"La Cordonnerie": "142 Rue Saint-Denis 75002 Paris, 75002 Paris, France",
"Le Supercoin": "3, rue Baudelique, 75018 Paris, France",
"Populettes": "86 bis rue Riquet, 75018 Paris, France",
"Au bon coin": "49 rue des Cloys, 75018 Paris, France",
"Le Couvent": "69 rue Broca, 75013 Paris, France",
"La Br\u00fblerie des Ternes": "111 rue mouffetard, 75005 Paris, France",
"L'\u00c9cir": "59 Boulevard Saint-Jacques, 75014 Paris, France",
"Le Chat bossu": "126, rue du Faubourg Saint Antoine, 75012 Paris, France",
"Denfert caf\u00e9": "58 boulvevard Saint Jacques, 75014 Paris, France",
"Le Caf\u00e9 frapp\u00e9": "95 rue Montmartre, 75002 Paris, France",
"La Perle": "78 rue vieille du temple, 75003 Paris, France",
"Le Descartes": "1 rue Thouin, 75005 Paris, France",
"Bagels & Coffee Corner": "Place de Clichy, 75017 Paris, France",
"Le petit club": "55 rue de la tombe Issoire, 75014 Paris, France",
"Le Plein soleil": "90 avenue Parmentier, 75011 Paris, France",
"Le Relais Haussmann": "146, boulevard Haussmann, 75008 Paris, France",
"Le Malar": "88 rue Saint-Dominique, 75007 Paris, France",
"Au panini de la place": "47 rue Belgrand, 75020 Paris, France",
"Le Village": "182 rue de Courcelles, 75017 Paris, France",
"Pause Caf\u00e9": "41 rue de Charonne, 75011 Paris, France",
"Le Pure caf\u00e9": "14 rue Jean Mac\u00e9, 75011 Paris, France",
"Extra old caf\u00e9": "307 fg saint Antoine, 75011 Paris, France",
"Chez Fafa": "44 rue Vinaigriers, 75010 Paris, France",
"En attendant l'or": "3 rue Faidherbe, 75011 Paris, France",
"Br\u00fblerie San Jos\u00e9": "30 rue des Petits-Champs, 75002 Paris, France",
"Caf\u00e9 de la Mairie (du VIII)": "rue de Lisbonne, 75008 Paris, France",
"Le General Beuret": "9 Place du General Beuret, 75015 Paris, France",
"Le Cap Bourbon": "1 rue Louis le Grand, 75002 Paris, France",
"En attendant l'or": "3 rue Faidherbe, 75011 Paris, France",
"Caf\u00e9 Martin": "2 place Martin Nadaud, 75001 Paris, France",
"Etienne": "14 rue Turbigo, Paris, 75001 Paris, France",
"L'ing\u00e9nu": "184 bd Voltaire, 75011 Paris, France",
"Le Biz": "18 rue Favart, 75002 Paris, France",
"L'Olive": "8 rue L'Olive, 75018 Paris, France",
"Le pari's caf\u00e9": "104 rue caulaincourt, 75018 Paris, France",
"Le Poulailler": "60 rue saint-sabin, 75011 Paris, France",
"La Marine": "55 bis quai de valmy, 75010 Paris, France",
"American Kitchen": "49 rue bichat, 75010 Paris, France",
"Chai 33": "33 Cour Saint Emilion, 75012 Paris, France",
"Face Bar": "82 rue des archives, 75003 Paris, France",
"Le Bloc": "21 avenue Brochant, 75017 Paris, France",
"La Bricole": "52 rue Liebniz, 75018 Paris, France",
"le ronsard": "place maubert, 75005 Paris, France",
"l'Usine": "1 rue d'Avron, 75020 Paris, France",
"La Cordonnerie": "142 Rue Saint-Denis 75002 Paris, 75002 Paris, France",
"Invitez vous chez nous": "7 rue Ep\u00e9e de Bois, 75005 Paris, France",
"Le sully": "13 rue du Faubourg Saint Denis, 75010 Paris, France",
"Le Ragueneau": "202 rue Saint-Honor\u00e9, 75001 Paris, France",
"Le Biz": "18 rue Favart, 75002 Paris, France",
"Le Cap Bourbon": "1 rue Louis le Grand, 75002 Paris, France",
"Le General Beuret": "9 Place du General Beuret, 75015 Paris, France",
"Le Germinal": "95 avenue Emile Zola, 75015 Paris, France",
"Le Ragueneau": "202 rue Saint-Honor\u00e9, 75001 Paris, France",
"Le refuge": "72 rue lamarck, 75018 Paris, France",
"Drole d'endroit pour une rencontre": "58 rue de Montorgueil, 75002 Paris, France",
"Le Petit Choiseul": "23 rue saint augustin, 75002 Paris, France",
"O'Breizh": "27 rue de Penthi\u00e8vre, 75008 Paris, France",
"Le Supercoin": "3, rue Baudelique, 75018 Paris, France",
"Populettes": "86 bis rue Riquet, 75018 Paris, France",
"La Recoleta au Manoir": "229 avenue Gambetta, 75020 Paris, France",
"L'Assassin": "99 rue Jean-Pierre Timbaud, 75011 Paris, France",
"Le Pareloup": "80 Rue Saint-Charles, 75015 Paris, France",
"Caf\u00e9 Zen": "46 rue Victoire, 75009 Paris, France",
"La Brasserie Gait\u00e9": "3 rue de la Gait\u00e9, 75014 Paris, France",
"Au bon coin": "49 rue des Cloys, 75018 Paris, France",
"La Br\u00fblerie des Ternes": "111 rue mouffetard, 75005 Paris, France",
"Le Chat bossu": "126, rue du Faubourg Saint Antoine, 75012 Paris, France",
"Denfert caf\u00e9": "58 boulvevard Saint Jacques, 75014 Paris, France",
"Le Couvent": "69 rue Broca, 75013 Paris, France",
"Bagels & Coffee Corner": "Place de Clichy, 75017 Paris, France",
"La Perle": "78 rue vieille du temple, 75003 Paris, France",
"Le Caf\u00e9 frapp\u00e9": "95 rue Montmartre, 75002 Paris, France",
"L'\u00c9cir": "59 Boulevard Saint-Jacques, 75014 Paris, France",
"Le Descartes": "1 rue Thouin, 75005 Paris, France",
"Le petit club": "55 rue de la tombe Issoire, 75014 Paris, France",
"Le Relais Haussmann": "146, boulevard Haussmann, 75008 Paris, France",
"Au panini de la place": "47 rue Belgrand, 75020 Paris, France",
"Extra old caf\u00e9": "307 fg saint Antoine, 75011 Paris, France",
"Le Plein soleil": "90 avenue Parmentier, 75011 Paris, France",
"Le Pure caf\u00e9": "14 rue Jean Mac\u00e9, 75011 Paris, France",
"Le Village": "182 rue de Courcelles, 75017 Paris, France",
"Le Malar": "88 rue Saint-Dominique, 75007 Paris, France",
"Pause Caf\u00e9": "41 rue de Charonne, 75011 Paris, France",
"Chez Fafa": "44 rue Vinaigriers, 75010 Paris, France",
"Caf\u00e9 dans l'aerogare Air France Invalides": "2 rue Robert Esnault Pelterie, 75007 Paris, France",
"Le relais de la victoire": "73 rue de la Victoire, 75009 Paris, France",
"Caprice caf\u00e9": "12 avenue Jean Moulin, 75014 Paris, France",
"Caves populaires": "22 rue des Dames, 75017 Paris, France",
"Cafe de grenelle": "188 rue de Grenelle, 75007 Paris, France",
"Chez Prune": "36 rue Beaurepaire, 75010 Paris, France",
"L'anjou": "1 rue de Montholon, 75009 Paris, France",
"Le Brio": "216, rue Marcadet, 75018 Paris, France",
"Tamm Bara": "7 rue Clisson, 75013 Paris, France",
"La chaumi\u00e8re gourmande": "Route de la Muette \u00e0 Neuilly",
"Club hippique du Jardin d\u2019Acclimatation": "75016 Paris, France",
"Les P\u00e8res Populaires": "46 rue de Buzenval, 75020 Paris, France",
"Epicerie Musicale": "55bis quai de Valmy, 75010 Paris, France",
"Le Centenaire": "104 rue amelot, 75011 Paris, France",
"Le Zazabar": "116 Rue de M\u00e9nilmontant, 75020 Paris, France",
"Ragueneau": "202 rue Saint Honor\u00e9, 75001 Paris, France",
"L'In\u00e9vitable": "22 rue Linn\u00e9, 75005 Paris, France",
"Le sully": "13 rue du Faubourg Saint Denis, 75010 Paris, France",
"Le Dunois": "77 rue Dunois, 75013 Paris, France",
"La Montagne Sans Genevi\u00e8ve": "13 Rue du Pot de Fer, 75005 Paris, France",
"Le Caminito": "48 rue du Dessous des Berges, 75013 Paris, France",
"Le petit Bretonneau": "Le petit Bretonneau - \u00e0 l'int\u00e9rieur de l'H\u00f4pital, 75018 Paris, France",
"La chaumi\u00e8re gourmande": "Route de la Muette \u00e0 Neuilly",
"Club hippique du Jardin d\u2019Acclimatation": "75016 Paris, France",
"Le bal du pirate": "60 rue des bergers, 75015 Paris, France",
"Le Zazabar": "116 Rue de M\u00e9nilmontant, 75020 Paris, France",
"L'antre d'eux": "16 rue DE MEZIERES, 75006 Paris, France",
"l'orillon bar": "35 rue de l'orillon, 75011 Paris, France",
"zic zinc": "95 rue claude decaen, 75012 Paris, France",
"Les P\u00e8res Populaires": "46 rue de Buzenval, 75020 Paris, France",
"Epicerie Musicale": "55bis quai de Valmy, 75010 Paris, France",
"Le relais de la victoire": "73 rue de la Victoire, 75009 Paris, France",
"Le Centenaire": "104 rue amelot, 75011 Paris, France",
"Cafe de grenelle": "188 rue de Grenelle, 75007 Paris, France",
"Ragueneau": "202 rue Saint Honor\u00e9, 75001 Paris, France",
"Caf\u00e9 Pistache": "9 rue des petits champs, 75001 Paris, France",
"La Cagnotte": "13 Rue Jean-Baptiste Dumay, 75020 Paris, France",
"bistrot les timbr\u00e9s": "14 rue d'alleray, 75015 Paris, France",
"Le Killy Jen": "28 bis boulevard Diderot, 75012 Paris, France",
"Caf\u00e9 beauveau": "9 rue de Miromesnil, 75008 Paris, France",
"le 1 cinq": "172 rue de vaugirard, 75015 Paris, France",
"Au Vin Des Rues": "21 rue Boulard, 75014 Paris, France",
"Les Artisans": "106 rue Lecourbe, 75015 Paris, France",
"Peperoni": "83 avenue de Wagram, 75001 Paris, France",
"Le BB (Bouchon des Batignolles)": "2 rue Lemercier, 75017 Paris, France",
"La Libert\u00e9": "196 rue du faubourg saint-antoine, 75012 Paris, France",
"Chez Rutabaga": "16 rue des Petits Champs, 75002 Paris, France",
"La cantoche de Paname": "40 Boulevard Beaumarchais, 75011 Paris, France",
"Le Saint Ren\u00e9": "148 Boulevard de Charonne, 75020 Paris, France",
"La Brocante": "10 rue Rossini, 75009 Paris, France",
"Caf\u00e9 Clochette": "16 avenue Richerand, 75010 Paris, France",
"L'europ\u00e9en": "21 Bis Boulevard Diderot, 75012 Paris, France",
"NoMa": "39 rue Notre Dame de Nazareth, 75003 Paris, France",
"O'Paris": "1 Rue des Envierges, 75020 Paris, France",
"Le Brio": "216, rue Marcadet, 75018 Paris, France",
"Tamm Bara": "7 rue Clisson, 75013 Paris, France",
"Caf\u00e9 dans l'aerogare Air France Invalides": "2 rue Robert Esnault Pelterie, 75007 Paris, France",
"bistrot les timbr\u00e9s": "14 rue d'alleray, 75015 Paris, France",
"Caprice caf\u00e9": "12 avenue Jean Moulin, 75014 Paris, France",
"Caves populaires": "22 rue des Dames, 75017 Paris, France",
"Au Vin Des Rues": "21 rue Boulard, 75014 Paris, France",
"Chez Prune": "36 rue Beaurepaire, 75010 Paris, France",
"L'In\u00e9vitable": "22 rue Linn\u00e9, 75005 Paris, France",
"L'anjou": "1 rue de Montholon, 75009 Paris, France",
"Botak cafe": "1 rue Paul albert, 75018 Paris, France",
"La cantine de Zo\u00e9": "136 rue du Faubourg poissonni\u00e8re, 75010 Paris, France",
"Les caves populaires": "22 rue des Dames, 75017 Paris, France",
"Le Plomb du cantal": "3 rue Ga\u00eet\u00e9, 75014 Paris, France",
"Trois pi\u00e8ces cuisine": "101 rue des dames, 75017 Paris, France",
"Le Zinc": "61 avenue de la Motte Picquet, 75015 Paris, France",
"L'avant comptoir": "3 carrefour de l'Od\u00e9on, 75006 Paris, France",
"Les Vendangeurs": "6/8 rue Stanislas, 75006 Paris, France",
"Chez Luna": "108 rue de M\u00e9nilmontant, 75020 Paris, France",
"Le bar Fleuri": "1 rue du Plateau, 75019 Paris, France",
"Bistrot Saint-Antoine": "58 rue du Fbg Saint-Antoine, 75012 Paris, France",
"Chez Oscar": "11/13 boulevard Beaumarchais, 75004 Paris, France",
"Le Piquet": "48 avenue de la Motte Picquet, 75015 Paris, France",
"L'avant comptoir": "3 carrefour de l'Od\u00e9on, 75006 Paris, France",
"le chateau d'eau": "67 rue du Ch\u00e2teau d'eau, 75010 Paris, France",
"Les Vendangeurs": "6/8 rue Stanislas, 75006 Paris, France",
"maison du vin": "52 rue des plantes, 75014 Paris, France",
"Le Tournebride": "104 rue Mouffetard, 75005 Paris, France",
"Le Fronton": "63 rue de Ponthieu, 75008 Paris, France",
"Le BB (Bouchon des Batignolles)": "2 rue Lemercier, 75017 Paris, France",
"La cantine de Zo\u00e9": "136 rue du Faubourg poissonni\u00e8re, 75010 Paris, France",
"Chez Rutabaga": "16 rue des Petits Champs, 75002 Paris, France",
"Les caves populaires": "22 rue des Dames, 75017 Paris, France",
"Le Plomb du cantal": "3 rue Ga\u00eet\u00e9, 75014 Paris, France",
"Trois pi\u00e8ces cuisine": "101 rue des dames, 75017 Paris, France",
"La Brocante": "10 rue Rossini, 75009 Paris, France",
"Le Zinc": "61 avenue de la Motte Picquet, 75015 Paris, France",
"Chez Luna": "108 rue de M\u00e9nilmontant, 75020 Paris, France",
"Le bar Fleuri": "1 rue du Plateau, 75019 Paris, France",
"La Libert\u00e9": "196 rue du faubourg saint-antoine, 75012 Paris, France",
"La cantoche de Paname": "40 Boulevard Beaumarchais, 75011 Paris, France",
"Le Saint Ren\u00e9": "148 Boulevard de Charonne, 75020 Paris, France",
"Caf\u00e9 Clochette": "16 avenue Richerand, 75010 Paris, France",
"L'europ\u00e9en": "21 Bis Boulevard Diderot, 75012 Paris, France",
"NoMa": "39 rue Notre Dame de Nazareth, 75003 Paris, France",
"le lutece": "380 rue de vaugirard, 75015 Paris, France",
"O'Paris": "1 Rue des Envierges, 75020 Paris, France",
"Rivolux": "16 rue de Rivoli, 75004 Paris, France",
"Brasiloja": "16 rue Ganneron, 75018 Paris, France",
"Le caf\u00e9 Monde et M\u00e9dias": "Place de la R\u00e9publique, 75003 Paris, France",
"L'entrep\u00f4t": "157 rue Bercy 75012 Paris, 75012 Paris, France",
"Coffee Chope": "344Vrue Vaugirard, 75015 Paris, France",
"l'El\u00e9phant du nil": "125 Rue Saint-Antoine, 75004 Paris, France",
"Le Parc Vaugirard": "358 rue de Vaugirard, 75015 Paris, France",
"Pari's Caf\u00e9": "174 avenue de Clichy, 75017 Paris, France",
"Le Comptoir": "354 bis rue Vaugirard, 75015 Paris, France",
"Caf\u00e9 Varenne": "36 rue de Varenne, 75007 Paris, France",
"Melting Pot": "3 rue de Lagny, 75020 Paris, France",
"le Zango": "58 rue Daguerre, 75014 Paris, France",
"Chez Miamophile": "6 rue M\u00e9lingue, 75019 Paris, France",
"Institut des Cultures d'Islam": "19-23 rue L\u00e9on, 75018 Paris, France",
"Canopy Caf\u00e9 associatif": "19 rue Pajol, 75018 Paris, France",
"Caf\u00e9 rallye tournelles": "11 Quai de la Tournelle, 75005 Paris, France",
"Petits Freres des Pauvres": "47 rue de Batignolles, 75017 Paris, France",
"Brasserie le Morvan": "61 rue du ch\u00e2teau d'eau, 75010 Paris, France",
"Le Lucernaire": "53 rue Notre-Dame des Champs, 75006 Paris, France",
"L'Angle": "28 rue de Ponthieu, 75008 Paris, France",
"Le Caf\u00e9 d'avant": "35 rue Claude Bernard, 75005 Paris, France",
"Caf\u00e9 Dupont": "198 rue de la Convention, 75015 Paris, France",
"Le S\u00e9vign\u00e9": "15 rue du Parc Royal, 75003 Paris, France",
"L'Entracte": "place de l'opera, 75002 Paris, France",
"Panem": "18 rue de Crussol, 75011 Paris, France",
"Au pays de Vannes": "34 bis rue de Wattignies, 75012 Paris, France",
"Le Lucernaire": "53 rue Notre-Dame des Champs, 75006 Paris, France",
"Le Caf\u00e9 d'avant": "35 rue Claude Bernard, 75005 Paris, France",
"l'El\u00e9phant du nil": "125 Rue Saint-Antoine, 75004 Paris, France",
"L'\u00e2ge d'or": "26 rue du Docteur Magnan, 75013 Paris, France",
"Le S\u00e9vign\u00e9": "15 rue du Parc Royal, 75003 Paris, France",
"Le Comptoir": "354 bis rue Vaugirard, 75015 Paris, France",
"L'horizon": "93, rue de la Roquette, 75011 Paris, France",
"L'empreinte": "54, avenue Daumesnil, 75012 Paris, France",
"Caf\u00e9 Victor": "10 boulevard Victor, 75015 Paris, France",
"Caf\u00e9 Varenne": "36 rue de Varenne, 75007 Paris, France",
"Le Brigadier": "12 rue Blanche, 75009 Paris, France",
"Waikiki": "10 rue d\"Ulm, 75005 Paris, France"}
"Waikiki": "10 rue d\"Ulm, 75005 Paris, France",
"Le Parc Vaugirard": "358 rue de Vaugirard, 75015 Paris, France",
"Pari's Caf\u00e9": "174 avenue de Clichy, 75017 Paris, France",
"Melting Pot": "3 rue de Lagny, 75020 Paris, France",
"le Zango": "58 rue Daguerre, 75014 Paris, France",
"Chez Miamophile": "6 rue M\u00e9lingue, 75019 Paris, France",
"Le caf\u00e9 Monde et M\u00e9dias": "Place de la R\u00e9publique, 75003 Paris, France",
"Caf\u00e9 rallye tournelles": "11 Quai de la Tournelle, 75005 Paris, France",
"Brasserie le Morvan": "61 rue du ch\u00e2teau d'eau, 75010 Paris, France",
"L'entrep\u00f4t": "157 rue Bercy 75012 Paris, 75012 Paris, France"}

248
bonobo/examples/datasets/coffeeshops.txt
View File

@ -1,182 +1,182 @@
Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
les montparnos, 65 boulevard Pasteur, 75015 Paris, France
Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
Le Felteu, 1 rue Pecquay, 75004 Paris, France
Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France
Café Lea, 5 rue Claude Bernard, 75005 Paris, France
Le Bellerive, 71 quai de Seine, 75019 Paris, France
Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
O q de poule, 53 rue du ruisseau, 75018 Paris, France
Le café des amis, 125 rue Blomet, 75015 Paris, France
Le chantereine, 51 Rue Victoire, 75009 Paris, France
Le Müller, 11 rue Feutrier, 75018 Paris, France
La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France
Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France
La Renaissance, 112 Rue Championnet, 75018 Paris, France
Extérieur Quai, 5, rue d'Alsace, 75010 Paris, France
Le Sully, 6 Bd henri IV, 75004 Paris, France
Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
Le café des amis, 125 rue Blomet, 75015 Paris, France
Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
Le Café Livres, 10 rue Saint Martin, 75004 Paris, France
Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France
Le Square, 31 rue Saint-Dominique, 75007 Paris, France
Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
La Bauloise, 36 rue du hameau, 75015 Paris, France
Le Dellac, 14 rue Rougemont, 75009 Paris, France
Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
Le Sully, 6 Bd henri IV, 75004 Paris, France
Le Felteu, 1 rue Pecquay, 75004 Paris, France
Le bistrot de Maëlle et Augustin, 42 rue coquillère, 75001 Paris, France
Dédé la frite, 52 rue Notre-Dame des Victoires, 75002 Paris, France
Cardinal Saint-Germain, 11 boulevard Saint-Germain, 75005 Paris, France
Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France
Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
La Renaissance, 112 Rue Championnet, 75018 Paris, France
Le Square, 31 rue Saint-Dominique, 75007 Paris, France
Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
Café antoine, 17 rue Jean de la Fontaine, 75016 Paris, France
Au cerceau d'or, 129 boulevard sebastopol, 75002 Paris, France
Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France
Café Lea, 5 rue Claude Bernard, 75005 Paris, France
Le Bellerive, 71 quai de Seine, 75019 Paris, France
La Bauloise, 36 rue du hameau, 75015 Paris, France
Le Dellac, 14 rue Rougemont, 75009 Paris, France
La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France
Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France
Le Café Livres, 10 rue Saint Martin, 75004 Paris, France
Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France
Drole d'endroit pour une rencontre, 58 rue de Montorgueil, 75002 Paris, France
Le pari's café, 104 rue caulaincourt, 75018 Paris, France
Le Poulailler, 60 rue saint-sabin, 75011 Paris, France
Chai 33, 33 Cour Saint Emilion, 75012 Paris, France
L'Assassin, 99 rue Jean-Pierre Timbaud, 75011 Paris, France
l'Usine, 1 rue d'Avron, 75020 Paris, France
La Bricole, 52 rue Liebniz, 75018 Paris, France
le ronsard, place maubert, 75005 Paris, France
Face Bar, 82 rue des archives, 75003 Paris, France
American Kitchen, 49 rue bichat, 75010 Paris, France
La Marine, 55 bis quai de valmy, 75010 Paris, France
Le Bloc, 21 avenue Brochant, 75017 Paris, France
La Recoleta au Manoir, 229 avenue Gambetta, 75020 Paris, France
Le Pareloup, 80 Rue Saint-Charles, 75015 Paris, France
La Brasserie Gaité, 3 rue de la Gaité, 75014 Paris, France
Café Zen, 46 rue Victoire, 75009 Paris, France
O'Breizh, 27 rue de Penthièvre, 75008 Paris, France
Le Petit Choiseul, 23 rue saint augustin, 75002 Paris, France
Invitez vous chez nous, 7 rue Epée de Bois, 75005 Paris, France
La Cordonnerie, 142 Rue Saint-Denis 75002 Paris, 75002 Paris, France
Le Supercoin, 3, rue Baudelique, 75018 Paris, France
Populettes, 86 bis rue Riquet, 75018 Paris, France
Au bon coin, 49 rue des Cloys, 75018 Paris, France
Le Couvent, 69 rue Broca, 75013 Paris, France
La Brûlerie des Ternes, 111 rue mouffetard, 75005 Paris, France
L'Écir, 59 Boulevard Saint-Jacques, 75014 Paris, France
Le Chat bossu, 126, rue du Faubourg Saint Antoine, 75012 Paris, France
Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France
Le Café frappé, 95 rue Montmartre, 75002 Paris, France
La Perle, 78 rue vieille du temple, 75003 Paris, France
Le Descartes, 1 rue Thouin, 75005 Paris, France
Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France
Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France
Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France
Le Malar, 88 rue Saint-Dominique, 75007 Paris, France
Au panini de la place, 47 rue Belgrand, 75020 Paris, France
Le Village, 182 rue de Courcelles, 75017 Paris, France
Pause Café, 41 rue de Charonne, 75011 Paris, France
Le Pure café, 14 rue Jean Macé, 75011 Paris, France
Extra old café, 307 fg saint Antoine, 75011 Paris, France
Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France
En attendant l'or, 3 rue Faidherbe, 75011 Paris, France
Brûlerie San José, 30 rue des Petits-Champs, 75002 Paris, France
Café de la Mairie (du VIII), rue de Lisbonne, 75008 Paris, France
Le General Beuret, 9 Place du General Beuret, 75015 Paris, France
Le Cap Bourbon, 1 rue Louis le Grand, 75002 Paris, France
En attendant l'or, 3 rue Faidherbe, 75011 Paris, France
Café Martin, 2 place Martin Nadaud, 75001 Paris, France
Etienne, 14 rue Turbigo, Paris, 75001 Paris, France
L'ingénu, 184 bd Voltaire, 75011 Paris, France
Le Biz, 18 rue Favart, 75002 Paris, France
L'Olive, 8 rue L'Olive, 75018 Paris, France
Le pari's café, 104 rue caulaincourt, 75018 Paris, France
Le Poulailler, 60 rue saint-sabin, 75011 Paris, France
La Marine, 55 bis quai de valmy, 75010 Paris, France
American Kitchen, 49 rue bichat, 75010 Paris, France
Chai 33, 33 Cour Saint Emilion, 75012 Paris, France
Face Bar, 82 rue des archives, 75003 Paris, France
Le Bloc, 21 avenue Brochant, 75017 Paris, France
La Bricole, 52 rue Liebniz, 75018 Paris, France
le ronsard, place maubert, 75005 Paris, France
l'Usine, 1 rue d'Avron, 75020 Paris, France
La Cordonnerie, 142 Rue Saint-Denis 75002 Paris, 75002 Paris, France
Invitez vous chez nous, 7 rue Epée de Bois, 75005 Paris, France
Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France
Le Ragueneau, 202 rue Saint-Honoré, 75001 Paris, France
Le Biz, 18 rue Favart, 75002 Paris, France
Le Cap Bourbon, 1 rue Louis le Grand, 75002 Paris, France
Le General Beuret, 9 Place du General Beuret, 75015 Paris, France
Le Germinal, 95 avenue Emile Zola, 75015 Paris, France
Le Ragueneau, 202 rue Saint-Honoré, 75001 Paris, France
Le refuge, 72 rue lamarck, 75018 Paris, France
Drole d'endroit pour une rencontre, 58 rue de Montorgueil, 75002 Paris, France
Le Petit Choiseul, 23 rue saint augustin, 75002 Paris, France
O'Breizh, 27 rue de Penthièvre, 75008 Paris, France
Le Supercoin, 3, rue Baudelique, 75018 Paris, France
Populettes, 86 bis rue Riquet, 75018 Paris, France
La Recoleta au Manoir, 229 avenue Gambetta, 75020 Paris, France
L'Assassin, 99 rue Jean-Pierre Timbaud, 75011 Paris, France
Le Pareloup, 80 Rue Saint-Charles, 75015 Paris, France
Café Zen, 46 rue Victoire, 75009 Paris, France
La Brasserie Gaité, 3 rue de la Gaité, 75014 Paris, France
Au bon coin, 49 rue des Cloys, 75018 Paris, France
La Brûlerie des Ternes, 111 rue mouffetard, 75005 Paris, France
Le Chat bossu, 126, rue du Faubourg Saint Antoine, 75012 Paris, France
Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France
Le Couvent, 69 rue Broca, 75013 Paris, France
Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
La Perle, 78 rue vieille du temple, 75003 Paris, France
Le Café frappé, 95 rue Montmartre, 75002 Paris, France
L'Écir, 59 Boulevard Saint-Jacques, 75014 Paris, France
Le Descartes, 1 rue Thouin, 75005 Paris, France
Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France
Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France
Au panini de la place, 47 rue Belgrand, 75020 Paris, France
Extra old café, 307 fg saint Antoine, 75011 Paris, France
Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France
Le Pure café, 14 rue Jean Macé, 75011 Paris, France
Le Village, 182 rue de Courcelles, 75017 Paris, France
Le Malar, 88 rue Saint-Dominique, 75007 Paris, France
Pause Café, 41 rue de Charonne, 75011 Paris, France
Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France
Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France
Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
Caves populaires, 22 rue des Dames, 75017 Paris, France
Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
L'anjou, 1 rue de Montholon, 75009 Paris, France
Le Brio, 216, rue Marcadet, 75018 Paris, France
Tamm Bara, 7 rue Clisson, 75013 Paris, France
La chaumière gourmande, Route de la Muette à Neuilly
Club hippique du Jardin d’Acclimatation, 75016 Paris, France
Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France
Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France
Le Centenaire, 104 rue amelot, 75011 Paris, France
Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
Ragueneau, 202 rue Saint Honoré, 75001 Paris, France
L'Inévitable, 22 rue Linné, 75005 Paris, France
Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France
Le Dunois, 77 rue Dunois, 75013 Paris, France
La Montagne Sans Geneviève, 13 Rue du Pot de Fer, 75005 Paris, France
Le Caminito, 48 rue du Dessous des Berges, 75013 Paris, France
Le petit Bretonneau, Le petit Bretonneau - à l'intérieur de l'Hôpital, 75018 Paris, France
La chaumière gourmande, Route de la Muette à Neuilly
Club hippique du Jardin d’Acclimatation, 75016 Paris, France
Le bal du pirate, 60 rue des bergers, 75015 Paris, France
Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
L'antre d'eux, 16 rue DE MEZIERES, 75006 Paris, France
l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
zic zinc, 95 rue claude decaen, 75012 Paris, France
Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France
Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France
Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
Le Centenaire, 104 rue amelot, 75011 Paris, France
Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
Ragueneau, 202 rue Saint Honoré, 75001 Paris, France
Café Pistache, 9 rue des petits champs, 75001 Paris, France
La Cagnotte, 13 Rue Jean-Baptiste Dumay, 75020 Paris, France
bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
le 1 cinq, 172 rue de vaugirard, 75015 Paris, France
Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
Les Artisans, 106 rue Lecourbe, 75015 Paris, France
Peperoni, 83 avenue de Wagram, 75001 Paris, France
Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France
La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France
Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France
La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
La Brocante, 10 rue Rossini, 75009 Paris, France
Café Clochette, 16 avenue Richerand, 75010 Paris, France
L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France
NoMa, 39 rue Notre Dame de Nazareth, 75003 Paris, France
O'Paris, 1 Rue des Envierges, 75020 Paris, France
Le Brio, 216, rue Marcadet, 75018 Paris, France
Tamm Bara, 7 rue Clisson, 75013 Paris, France
Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France
bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
Caves populaires, 22 rue des Dames, 75017 Paris, France
Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
L'Inévitable, 22 rue Linné, 75005 Paris, France
L'anjou, 1 rue de Montholon, 75009 Paris, France
Botak cafe, 1 rue Paul albert, 75018 Paris, France
La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France
Les caves populaires, 22 rue des Dames, 75017 Paris, France
Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
maison du vin, 52 rue des plantes, 75014 Paris, France
Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France
La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France
Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France
Les caves populaires, 22 rue des Dames, 75017 Paris, France
Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
La Brocante, 10 rue Rossini, 75009 Paris, France
Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France
La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
Café Clochette, 16 avenue Richerand, 75010 Paris, France
L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France
NoMa, 39 rue Notre Dame de Nazareth, 75003 Paris, France
le lutece, 380 rue de vaugirard, 75015 Paris, France
O'Paris, 1 Rue des Envierges, 75020 Paris, France
Rivolux, 16 rue de Rivoli, 75004 Paris, France
Brasiloja, 16 rue Ganneron, 75018 Paris, France
Le café Monde et Médias, Place de la République, 75003 Paris, France
L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France
Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France
l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France
Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France
Pari's Café, 174 avenue de Clichy, 75017 Paris, France
Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
Café Varenne, 36 rue de Varenne, 75007 Paris, France
Melting Pot, 3 rue de Lagny, 75020 Paris, France
le Zango, 58 rue Daguerre, 75014 Paris, France
Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France
Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
Brasserie le Morvan, 61 rue du château d'eau, 75010 Paris, France
Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France
L'Angle, 28 rue de Ponthieu, 75008 Paris, France
Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France
Café Dupont, 198 rue de la Convention, 75015 Paris, France
Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France
L'Entracte, place de l'opera, 75002 Paris, France
Panem, 18 rue de Crussol, 75011 Paris, France
Au pays de Vannes, 34 bis rue de Wattignies, 75012 Paris, France
Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France
Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France
l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France
L'âge d'or, 26 rue du Docteur Magnan, 75013 Paris, France
Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France
Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
L'horizon, 93, rue de la Roquette, 75011 Paris, France
L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
Café Victor, 10 boulevard Victor, 75015 Paris, France
Café Varenne, 36 rue de Varenne, 75007 Paris, France
Le Brigadier, 12 rue Blanche, 75009 Paris, France
Waikiki, 10 rue d"Ulm, 75005 Paris, France
Waikiki, 10 rue d"Ulm, 75005 Paris, France
Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France
Pari's Café, 174 avenue de Clichy, 75017 Paris, France
Melting Pot, 3 rue de Lagny, 75020 Paris, France
le Zango, 58 rue Daguerre, 75014 Paris, France
Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
Le café Monde et Médias, Place de la République, 75003 Paris, France
Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France
Brasserie le Morvan, 61 rue du château d'eau, 75010 Paris, France
L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France

2
bonobo/examples/datasets/fablabs.py
View File

@ -96,8 +96,8 @@ graph = bonobo.Graph(
),
normalize,
filter_france,
bonobo.JsonWriter(path='fablabs.txt', ioformat='arg0'),
bonobo.Tee(display),
bonobo.JsonWriter(path='fablabs.txt'),
)
if __name__ == '__main__':

7
bonobo/examples/nodes/filter.py
View File

@ -9,13 +9,16 @@ class OddOnlyFilter(Filter):
@Filter
def MultiplesOfThreeOnlyFilter(self, i):
def multiples_of_three(i):
return not (i % 3)
graph = bonobo.Graph(
lambda: tuple(range(50)),
OddOnlyFilter(),
MultiplesOfThreeOnlyFilter(),
multiples_of_three,
print,
)
if __name__ == '__main__':
bonobo.run(graph)

3
bonobo/examples/nodes/slow.py
View File

@ -14,3 +14,6 @@ graph = bonobo.Graph(
pause,
print,
)
if __name__ == '__main__':
bonobo.run(graph)

2
bonobo/examples/tutorials/tut02e02_write.py
View File

@ -8,7 +8,7 @@ def split_one(line):
graph = bonobo.Graph(
bonobo.FileReader('coffeeshops.txt'),
split_one,
bonobo.JsonWriter('coffeeshops.json'),
bonobo.JsonWriter('coffeeshops.json', ioformat='arg0'),
)
if __name__ == '__main__':

7
bonobo/execution/base.py
View File

@ -58,6 +58,13 @@ class LoopingExecutionContext(Wrapper):
# XXX enhancers
self._enhancers = get_enhancers(self.wrapped)
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
self.stop()
def start(self):
if self.started:
raise RuntimeError('Cannot start a node twice ({}).'.format(get_name(self)))

6
bonobo/execution/node.py
View File

@ -3,7 +3,7 @@ from queue import Empty
from time import sleep
from bonobo.constants import INHERIT_INPUT, NOT_MODIFIED
from bonobo.errors import InactiveReadableError
from bonobo.errors import InactiveReadableError, UnrecoverableError
from bonobo.execution.base import LoopingExecutionContext
from bonobo.structs.bags import Bag
from bonobo.structs.inputs import Input
@ -93,6 +93,10 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
except Empty:
sleep(self.PERIOD)
continue
except UnrecoverableError as exc:
self.handle_error(exc, traceback.format_exc())
self.input.shutdown()
break
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())

4
bonobo/ext/console.py
View File

@ -65,7 +65,7 @@ class ConsoleOutputPlugin(Plugin):
for i in context.graph.topologically_sorted_indexes:
node = context[i]
name_suffix = '({})'.format(i) if settings.DEBUG else ''
name_suffix = '({})'.format(i) if settings.DEBUG.get() else ''
if node.alive:
_line = ''.join(
(
@ -100,7 +100,7 @@ class ConsoleOutputPlugin(Plugin):
print(MOVE_CURSOR_UP(t_cnt + 2), file=sys.stderr)
def _write(self, graph_context, rewind):
if settings.PROFILE:
if settings.PROFILE.get():
if self.counter % 10 and self._append_cache:
append = self._append_cache
else:

4
bonobo/ext/opendatasoft.py
View File

@ -13,13 +13,13 @@ def path_str(path):
class OpenDataSoftAPI(Configurable):
dataset = Option(str, required=True)
dataset = Option(str, positional=True)
endpoint = Option(str, default='{scheme}://{netloc}{path}')
scheme = Option(str, default='https')
netloc = Option(str, default='data.opendatasoft.com')
path = Option(path_str, default='/api/records/1.0/search/')
rows = Option(int, default=500)
limit = Option(int, default=None)
limit = Option(int, required=False)
timezone = Option(str, default='Europe/Paris')
kwargs = Option(dict, default=dict)

5
bonobo/logging.py
View File

@ -71,5 +71,8 @@ def get_logger(name='bonobo'):
return logging.getLogger(name)
# Compatibility with python logging
getLogger = get_logger
# Setup formating and level.
setup(level=settings.LOGGING_LEVEL)
setup(level=settings.LOGGING_LEVEL.get())

11
bonobo/nodes/__init__.py
View File

@ -1,9 +1,8 @@
from bonobo.nodes.io import __all__ as _all_io
from bonobo.nodes.io import *
from bonobo.nodes.basics import __all__ as _all_basics
from bonobo.nodes.basics import *
from bonobo.nodes.basics import __all__ as _all_basics
from bonobo.nodes.filter import Filter
from bonobo.nodes.io import *
from bonobo.nodes.io import __all__ as _all_io
from bonobo.nodes.throttle import RateLimited
__all__ = _all_basics + _all_io + ['Filter']
__all__ = _all_basics + _all_io + ['Filter', 'RateLimited']

49
bonobo/nodes/basics.py
View File

@ -1,23 +1,19 @@
import functools
from pprint import pprint as _pprint
import itertools
from colorama import Fore, Style
from bonobo import settings
from bonobo.config import Configurable, Option
from bonobo.config.processors import ContextProcessor
from bonobo.constants import NOT_MODIFIED
from bonobo.structs.bags import Bag
from bonobo.util.objects import ValueHolder
from bonobo.util.term import CLEAR_EOL
from bonobo.constants import NOT_MODIFIED
__all__ = [
'identity',
'Limit',
'Tee',
'count',
'pprint',
'PrettyPrinter',
'noop',
]
@ -73,7 +69,7 @@ def _count_counter(self, context):
class PrettyPrinter(Configurable):
def call(self, *args, **kwargs):
formater = self._format_quiet if settings.QUIET else self._format_console
formater = self._format_quiet if settings.QUIET.get() else self._format_console
for i, (item, value) in enumerate(itertools.chain(enumerate(args), kwargs.items())):
print(formater(i, item, value))
@ -87,47 +83,6 @@ class PrettyPrinter(Configurable):
)
pprint = PrettyPrinter()
pprint.__name__ = 'pprint'
def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):
from bonobo.constants import NOT_MODIFIED
def _pprint(*args, **kwargs):
nonlocal title_keys, sort, print_values
row = args[0]
for key in title_keys:
if key in row:
print(Style.BRIGHT, row.get(key), Style.RESET_ALL, sep='')
break
if print_values:
for k in sorted(row) if sort else row:
print(
' • ',
Fore.BLUE,
k,
Style.RESET_ALL,
' : ',
Fore.BLACK,
'(',
type(row[k]).__name__,
')',
Style.RESET_ALL,
' ',
repr(row[k]),
CLEAR_EOL,
)
yield NOT_MODIFIED
_pprint.__name__ = 'pprint'
return _pprint
def noop(*args, **kwargs): # pylint: disable=unused-argument
from bonobo.constants import NOT_MODIFIED
return NOT_MODIFIED

83
bonobo/nodes/io/base.py Normal file
View File

@ -0,0 +1,83 @@
from bonobo import settings
from bonobo.config import Configurable, ContextProcessor, Option, Service
from bonobo.errors import UnrecoverableValueError, UnrecoverableNotImplementedError
from bonobo.structs.bags import Bag
class IOFormatEnabled(Configurable):
ioformat = Option(default=settings.IOFORMAT.get)
def get_input(self, *args, **kwargs):
if self.ioformat == settings.IOFORMAT_ARG0:
if len(args) != 1 or len(kwargs):
raise UnrecoverableValueError(
'Wrong input formating: IOFORMAT=ARG0 implies one arg and no kwargs, got args={!r} and kwargs={!r}.'.
format(args, kwargs)
)
return args[0]
if self.ioformat == settings.IOFORMAT_KWARGS:
if len(args) or not len(kwargs):
raise UnrecoverableValueError(
'Wrong input formating: IOFORMAT=KWARGS ioformat implies no arg, got args={!r} and kwargs={!r}.'.
format(args, kwargs)
)
return kwargs
raise UnrecoverableNotImplementedError('Unsupported format.')
def get_output(self, row):
if self.ioformat == settings.IOFORMAT_ARG0:
return row
if self.ioformat == settings.IOFORMAT_KWARGS:
return Bag(**row)
raise UnrecoverableNotImplementedError('Unsupported format.')
class FileHandler(Configurable):
"""Abstract component factory for file-related components.
Args:
path (str): which path to use within the provided filesystem.
eol (str): which character to use to separate lines.
mode (str): which mode to use when opening the file.
fs (str): service name to use for filesystem.
"""
path = Option(str, required=True, positional=True) # type: str
eol = Option(str, default='\n') # type: str
mode = Option(str) # type: str
encoding = Option(str, default='utf-8') # type: str
fs = Service('fs') # type: str
@ContextProcessor
def file(self, context, fs):
with self.open(fs) as file:
yield file
def open(self, fs):
return fs.open(self.path, self.mode, encoding=self.encoding)
class Reader:
"""Abstract component factory for readers.
"""
def __call__(self, *args, **kwargs):
yield from self.read(*args, **kwargs)
def read(self, *args, **kwargs):
raise NotImplementedError('Abstract.')
class Writer:
"""Abstract component factory for writers.
"""
def __call__(self, *args, **kwargs):
return self.write(*args, **kwargs)
def write(self, *args, **kwargs):
raise NotImplementedError('Abstract.')

9
bonobo/nodes/io/csv.py
View File

@ -3,7 +3,8 @@ import csv
from bonobo.config import Option
from bonobo.config.processors import ContextProcessor
from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.file import FileHandler, FileReader, FileWriter
from bonobo.nodes.io.file import FileReader, FileWriter
from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
from bonobo.util.objects import ValueHolder
@ -25,10 +26,10 @@ class CsvHandler(FileHandler):
"""
delimiter = Option(str, default=';')
quotechar = Option(str, default='"')
headers = Option(tuple)
headers = Option(tuple, required=False)
class CsvReader(CsvHandler, FileReader):
class CsvReader(IOFormatEnabled, FileReader, CsvHandler):
"""
Reads a CSV and yield the values as dicts.
@ -64,7 +65,7 @@ class CsvReader(CsvHandler, FileReader):
yield self.get_output(dict(zip(_headers, row)))
class CsvWriter(CsvHandler, FileWriter):
class CsvWriter(IOFormatEnabled, FileWriter, CsvHandler):
@ContextProcessor
def writer(self, context, fs, file, lineno):
writer = csv.writer(file, delimiter=self.delimiter, quotechar=self.quotechar, lineterminator=self.eol)

82
bonobo/nodes/io/file.py
View File

@ -1,81 +1,11 @@
from bonobo import settings
from bonobo.config import Option, Service
from bonobo.config.configurables import Configurable
from bonobo.config import Option
from bonobo.config.processors import ContextProcessor
from bonobo.constants import NOT_MODIFIED
from bonobo.structs.bags import Bag
from bonobo.nodes.io.base import FileHandler, Reader, Writer
from bonobo.util.objects import ValueHolder
class FileHandler(Configurable):
"""Abstract component factory for file-related components.
Args:
path (str): which path to use within the provided filesystem.
eol (str): which character to use to separate lines.
mode (str): which mode to use when opening the file.
fs (str): service name to use for filesystem.
"""
path = Option(str, required=True, positional=True) # type: str
eol = Option(str, default='\n') # type: str
mode = Option(str) # type: str
encoding = Option(str, default='utf-8') # type: str
fs = Service('fs') # type: str
ioformat = Option(default=settings.IOFORMAT.get)
@ContextProcessor
def file(self, context, fs):
with self.open(fs) as file:
yield file
def open(self, fs):
return fs.open(self.path, self.mode, encoding=self.encoding)
def get_input(self, *args, **kwargs):
if self.ioformat == settings.IOFORMAT_ARG0:
assert len(args) == 1 and not len(kwargs), 'ARG0 format implies one arg and no kwargs.'
return args[0]
if self.ioformat == settings.IOFORMAT_KWARGS:
assert len(args) == 0 and len(kwargs), 'KWARGS format implies no arg.'
return kwargs
raise NotImplementedError('Unsupported format.')
def get_output(self, row):
if self.ioformat == settings.IOFORMAT_ARG0:
return row
if self.ioformat == settings.IOFORMAT_KWARGS:
return Bag(**row)
raise NotImplementedError('Unsupported format.')
class Reader(FileHandler):
"""Abstract component factory for readers.
"""
def __call__(self, *args, **kwargs):
yield from self.read(*args, **kwargs)
def read(self, *args, **kwargs):
raise NotImplementedError('Abstract.')
class Writer(FileHandler):
"""Abstract component factory for writers.
"""
def __call__(self, *args, **kwargs):
return self.write(*args)
def write(self, *args, **kwargs):
raise NotImplementedError('Abstract.')
class FileReader(Reader):
class FileReader(Reader, FileHandler):
"""Component factory for file-like readers.
On its own, it can be used to read a file and yield one row per line, trimming the "eol" character at the end if
@ -93,7 +23,7 @@ class FileReader(Reader):
yield line.rstrip(self.eol)
class FileWriter(Writer):
class FileWriter(Writer, FileHandler):
"""Component factory for file or file-like writers.
On its own, it can be used to write in a file one line per row that comes into this component. Extending it is
@ -107,11 +37,11 @@ class FileWriter(Writer):
lineno = ValueHolder(0)
yield lineno
def write(self, fs, file, lineno, row):
def write(self, fs, file, lineno, line):
"""
Write a row on the next line of opened file in context.
"""
self._write_line(file, (self.eol if lineno.value else '') + row)
self._write_line(file, (self.eol if lineno.value else '') + line)
lineno += 1
return NOT_MODIFIED

17
bonobo/nodes/io/json.py
View File

@ -1,15 +1,17 @@
import json
from bonobo.config.processors import ContextProcessor
from bonobo.nodes.io.file import FileWriter, FileReader
from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
from bonobo.nodes.io.file import FileReader, FileWriter
class JsonHandler():
class JsonHandler(FileHandler):
eol = ',\n'
prefix, suffix = '[', ']'
class JsonReader(JsonHandler, FileReader):
class JsonReader(IOFormatEnabled, FileReader, JsonHandler):
loader = staticmethod(json.load)
def read(self, fs, file):
@ -17,18 +19,21 @@ class JsonReader(JsonHandler, FileReader):
yield self.get_output(line)
class JsonWriter(JsonHandler, FileWriter):
class JsonWriter(IOFormatEnabled, FileWriter, JsonHandler):
@ContextProcessor
def envelope(self, context, fs, file, lineno):
file.write(self.prefix)
yield
file.write(self.suffix)
def write(self, fs, file, lineno, row):
def write(self, fs, file, lineno, *args, **kwargs):
"""
Write a json row on the next line of file pointed by ctx.file.
:param ctx:
:param row:
"""
return super().write(fs, file, lineno, json.dumps(row))
row = self.get_input(*args, **kwargs)
self._write_line(file, (self.eol if lineno.value else '') + json.dumps(row))
lineno += 1
return NOT_MODIFIED

12
bonobo/nodes/io/pickle.py
View File

@ -1,10 +1,11 @@
import pickle
from bonobo.config.processors import ContextProcessor
from bonobo.config import Option
from bonobo.config.processors import ContextProcessor
from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
from bonobo.nodes.io.file import FileReader, FileWriter
from bonobo.util.objects import ValueHolder
from .file import FileReader, FileWriter, FileHandler
class PickleHandler(FileHandler):
@ -16,10 +17,10 @@ class PickleHandler(FileHandler):
"""
item_names = Option(tuple)
item_names = Option(tuple, required=False)
class PickleReader(PickleHandler, FileReader):
class PickleReader(IOFormatEnabled, FileReader, PickleHandler):
"""
Reads a Python pickle object and yields the items in dicts.
"""
@ -56,8 +57,7 @@ class PickleReader(PickleHandler, FileReader):
yield self.get_output(dict(zip(i)) if is_dict else dict(zip(pickle_headers.value, i)))
class PickleWriter(PickleHandler, FileWriter):
class PickleWriter(IOFormatEnabled, FileWriter, PickleHandler):
mode = Option(str, default='wb')
def write(self, fs, file, lineno, item):

55
bonobo/nodes/throttle.py Normal file
View File

@ -0,0 +1,55 @@
import threading
import time
from bonobo.config import Configurable, ContextProcessor, Method, Option
class RateLimitBucket(threading.Thread):
daemon = True
@property
def stopped(self):
return self._stop_event.is_set()
def __init__(self, initial=1, period=1, amount=1):
super(RateLimitBucket, self).__init__()
self.semaphore = threading.BoundedSemaphore(initial)
self.amount = amount
self.period = period
self._stop_event = threading.Event()
def stop(self):
self._stop_event.set()
def run(self):
while not self.stopped:
time.sleep(self.period)
for _ in range(self.amount):
self.semaphore.release()
def wait(self):
return self.semaphore.acquire()
class RateLimited(Configurable):
handler = Method()
initial = Option(int, positional=True, default=1)
period = Option(int, positional=True, default=1)
amount = Option(int, positional=True, default=1)
@ContextProcessor
def bucket(self, context):
print(context)
bucket = RateLimitBucket(self.initial, self.amount, self.period)
bucket.start()
print(bucket)
yield bucket
bucket.stop()
bucket.join()
def call(self, bucket, *args, **kwargs):
print(bucket, args, kwargs)
bucket.wait()
return self.handler(*args, **kwargs)

55
bonobo/settings.py
View File

@ -5,6 +5,10 @@ from bonobo.errors import ValidationError
def to_bool(s):
if s is None:
return False
if type(s) is bool:
return s
if len(s):
if s.lower() in ('f', 'false', 'n', 'no', '0'):
return False
@ -13,7 +17,18 @@ def to_bool(s):
class Setting:
def __init__(self, name, default=None, validator=None):
__all__ = {}
@classmethod
def clear_all(cls):
for setting in Setting.__all__.values():
setting.clear()
def __new__(cls, name, *args, **kwargs):
Setting.__all__[name] = super().__new__(cls)
return Setting.__all__[name]
def __init__(self, name, default=None, validator=None, formatter=None):
self.name = name
if default:
@ -21,15 +36,14 @@ class Setting:
else:
self.default = lambda: None
if validator:
self.validator = validator
else:
self.validator = None
self.validator = validator
self.formatter = formatter
def __repr__(self):
return '<Setting {}={!r}>'.format(self.name, self.value)
return '<Setting {}={!r}>'.format(self.name, self.get())
def set(self, value):
value = self.formatter(value) if self.formatter else value
if self.validator and not self.validator(value):
raise ValidationError('Invalid value {!r} for setting {}.'.format(value, self.name))
self.value = value
@ -38,21 +52,35 @@ class Setting:
try:
return self.value
except AttributeError:
self.value = self.default()
value = os.environ.get(self.name, None)
if value is None:
value = self.default()
self.set(value)
return self.value
def clear(self):
try:
del self.value
except AttributeError:
pass
# Debug/verbose mode.
DEBUG = to_bool(os.environ.get('DEBUG', 'f'))
DEBUG = Setting('DEBUG', formatter=to_bool, default=False)
# Profile mode.
PROFILE = to_bool(os.environ.get('PROFILE', 'f'))
PROFILE = Setting('PROFILE', formatter=to_bool, default=False)
# Quiet mode.
QUIET = to_bool(os.environ.get('QUIET', 'f'))
QUIET = Setting('QUIET', formatter=to_bool, default=False)
# Logging level.
LOGGING_LEVEL = logging.DEBUG if DEBUG else logging.INFO
LOGGING_LEVEL = Setting(
'LOGGING_LEVEL',
formatter=logging._checkLevel,
validator=logging._checkLevel,
default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO
)
# Input/Output format for transformations
IOFORMAT_ARG0 = 'arg0'
@ -67,5 +95,8 @@ IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__co
def check():
if DEBUG and QUIET:
if DEBUG.get() and QUIET.get():
raise RuntimeError('I cannot be verbose and quiet at the same time.')
clear_all = Setting.clear_all

12
bonobo/strategies/executor.py
View File

@ -1,6 +1,5 @@
import time
import traceback
from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
from bonobo.constants import BEGIN, END
@ -31,12 +30,11 @@ class ExecutorStrategy(Strategy):
for plugin_context in context.plugins:
def _runner(plugin_context=plugin_context):
try:
plugin_context.start()
plugin_context.loop()
plugin_context.stop()
except Exception as exc:
print_error(exc, traceback.format_exc(), context=plugin_context)
with plugin_context:
try:
plugin_context.loop()
except Exception as exc:
print_error(exc, traceback.format_exc(), context=plugin_context)
futures.append(executor.submit(_runner))

18
bonobo/structs/inputs.py
View File

@ -77,6 +77,12 @@ class Input(Queue, Readable, Writable):
return Queue.put(self, data, block, timeout)
def _decrement_runlevel(self):
if self._runlevel == 1:
self.on_finalize()
self._runlevel -= 1
self.on_end()
def get(self, block=True, timeout=None):
if not self.alive:
raise InactiveReadableError('Cannot get() on an inactive {}.'.format(Readable.__name__))
@ -84,13 +90,7 @@ class Input(Queue, Readable, Writable):
data = Queue.get(self, block, timeout)
if data == END:
if self._runlevel == 1:
self.on_finalize()
self._runlevel -= 1
# callback
self.on_end()
self._decrement_runlevel()
if not self.alive:
raise InactiveReadableError(
@ -100,6 +100,10 @@ class Input(Queue, Readable, Writable):
return data
def shutdown(self):
while self._runlevel >= 1:
self._decrement_runlevel()
def empty(self):
self.mutex.acquire()
while self._qsize() and self.queue[0] == END:

6
bonobo/util/collections.py Normal file
View File

@ -0,0 +1,6 @@
import bisect
class sortedlist(list):
def insort(self, x):
bisect.insort(self, x)

116
bonobo/util/inspect.py Normal file
View File

@ -0,0 +1,116 @@
from collections import namedtuple
def isconfigurabletype(mixed):
"""
Check if the given argument is an instance of :class:`bonobo.config.ConfigurableMeta`, meaning it has all the
plumbery necessary to build :class:`bonobo.config.Configurable`-like instances.
:param mixed:
:return: bool
"""
from bonobo.config.configurables import ConfigurableMeta
return isinstance(mixed, ConfigurableMeta)
def isconfigurable(mixed):
"""
Check if the given argument is an instance of :class:`bonobo.config.Configurable`.
:param mixed:
:return: bool
"""
from bonobo.config.configurables import Configurable
return isinstance(mixed, Configurable)
def isoption(mixed):
"""
Check if the given argument is an instance of :class:`bonobo.config.Option`.
:param mixed:
:return: bool
"""
from bonobo.config.options import Option
return isinstance(mixed, Option)
def ismethod(mixed):
"""
Check if the given argument is an instance of :class:`bonobo.config.Method`.
:param mixed:
:return: bool
"""
from bonobo.config.options import Method
return isinstance(mixed, Method)
def iscontextprocessor(x):
"""
Check if the given argument is an instance of :class:`bonobo.config.ContextProcessor`.
:param mixed:
:return: bool
"""
from bonobo.config.processors import ContextProcessor
return isinstance(x, ContextProcessor)
def istype(mixed):
"""
Check if the given argument is a type object.
:param mixed:
:return: bool
"""
return isinstance(mixed, type)
ConfigurableInspection = namedtuple(
'ConfigurableInspection', [
'type',
'instance',
'options',
'processors',
'partial',
]
)
ConfigurableInspection.__enter__ = lambda self: self
ConfigurableInspection.__exit__ = lambda *exc_details: None
def inspect_node(mixed, *, _partial=None):
"""
If the given argument is somehow a :class:`bonobo.config.Configurable` object (either a subclass, an instance, or
a partially configured instance), then it will return a :class:`ConfigurableInspection` namedtuple, used to inspect
the configurable metadata (options). If you want to get the option values, you don't need this, it is only usefull
to perform introspection on a configurable.
If it's not looking like a configurable, it will raise a :class:`TypeError`.
:param mixed:
:return: ConfigurableInspection
:raise: TypeError
"""
if isconfigurabletype(mixed):
inst, typ = None, mixed
elif isconfigurable(mixed):
inst, typ = mixed, type(mixed)
elif hasattr(mixed, 'func'):
return inspect_node(mixed.func, _partial=(mixed.args, mixed.keywords))
else:
raise TypeError(
'Not a Configurable, nor a Configurable instance and not even a partially configured Configurable. Check your inputs.'
)
return ConfigurableInspection(
typ,
inst,
list(typ.__options__),
list(typ.__processors__),
_partial,
)

18
bonobo/util/testing.py
View File

@ -1,6 +1,7 @@
from contextlib import contextmanager
from unittest.mock import MagicMock
from bonobo import open_fs
from bonobo.execution.node import NodeExecutionContext
@ -17,3 +18,20 @@ def optional_contextmanager(cm, *, ignore=False):
else:
with cm:
yield
class FilesystemTester:
def __init__(self, extension='txt', mode='w'):
self.extension = extension
self.input_data = ''
self.mode = mode
def get_services_for_reader(self, tmpdir):
fs, filename = open_fs(tmpdir), 'input.' + self.extension
with fs.open(filename, self.mode) as fp:
fp.write(self.input_data)
return fs, filename, {'fs': fs}
def get_services_for_writer(self, tmpdir):
fs, filename = open_fs(tmpdir), 'output.' + self.extension
return fs, filename, {'fs': fs}

0
config/__init__.py
View File

2
docs/_templates/index.html vendored
View File

@ -49,7 +49,7 @@
</tr>
<tr>
<td>
<p class="biglink"><a class="biglink" href="{{ pathto("changes") }}">{% trans %}
<p class="biglink"><a class="biglink" target="_blank" href="https://github.com/python-bonobo/bonobo/tree/master/bonobo/examples">{% trans %}
Cookbook{% endtrans %}</a><br/>
<span class="linkdescr">{% trans %}examples and recipes{% endtrans %}</span></p>
</td>

17
docs/changelog.rst
View File

@ -1,6 +1,23 @@
Changelog
=========
v.0.4.2 - 18 june 2017
::::::::::::::::::::::
* [config] Implements a "requires()" service injection decorator for functions (api may change).
* [core] Execution contexts are now context managers.
* [fs] adds a defaut to current working directory in open_fs(...).
* [logging] Adds logging alias for easier imports.
* [stdlib] Fix I/O related nodes (especially json), there were bad bugs with ioformat.
Dependency updates
------------------
* Update bonobo-docker from 0.2.6 to 0.2.8
* Update dependencies.
* Update fs from 2.0.3 to 2.0.4
* Update requests from 2.17.3 to 2.18.1
v.0.4.0 - 10 june 2017
::::::::::::::::::::::

4
requirements-dev.txt
View File

@ -24,10 +24,10 @@ pytest-timeout==1.2.0
pytest==3.1.2
python-dateutil==2.6.0
pytz==2017.2
requests==2.17.3
requests==2.18.1
six==1.10.0
snowballstemmer==1.2.1
sphinx==1.6.2
sphinx==1.6.3
sphinxcontrib-websupport==1.0.1
termcolor==1.1.0
urllib3==1.21.1

13
requirements-docker.txt
View File

@ -1,19 +1,20 @@
-e .[docker]
appdirs==1.4.3
bonobo-docker==0.2.6
bonobo-docker==0.2.9
certifi==2017.4.17
chardet==3.0.4
colorama==0.3.9
docker-pycreds==0.2.1
docker==2.3.0
enum34==1.1.6
fs==2.0.3
fs==2.0.4
idna==2.5
pbr==3.0.1
packaging==16.8
pbr==3.1.1
psutil==5.2.2
pyparsing==2.2.0
pytz==2017.2
requests==2.17.3
requests==2.18.1
six==1.10.0
stevedore==1.23.0
urllib3==1.21.1
websocket-client==0.40.0
websocket-client==0.44.0

4
requirements-jupyter.txt
View File

@ -11,7 +11,7 @@ ipywidgets==6.0.0
jedi==0.10.2
jinja2==2.9.6
jsonschema==2.6.0
jupyter-client==5.0.1
jupyter-client==5.1.0
jupyter-console==5.1.0
jupyter-core==4.3.0
jupyter==1.0.0
@ -24,7 +24,7 @@ pandocfilters==1.4.1
pexpect==4.2.1
pickleshare==0.7.4
prompt-toolkit==1.0.14
ptyprocess==0.5.1
ptyprocess==0.5.2
pygments==2.2.0
python-dateutil==2.6.0
pyzmq==16.0.2

7
requirements.txt
View File

@ -3,15 +3,14 @@ appdirs==1.4.3
certifi==2017.4.17
chardet==3.0.4
colorama==0.3.9
enum34==1.1.6
fs==2.0.3
fs==2.0.4
idna==2.5
packaging==16.8
pbr==3.0.1
pbr==3.1.1
psutil==5.2.2
pyparsing==2.2.0
pytz==2017.2
requests==2.17.3
requests==2.18.1
six==1.10.0
stevedore==1.23.0
urllib3==1.21.1

56
tests/config/test_configurables.py
View File

@ -2,12 +2,17 @@ import pytest
from bonobo.config.configurables import Configurable
from bonobo.config.options import Option
from bonobo.util.inspect import inspect_node
class NoOptConfigurable(Configurable):
pass
class MyConfigurable(Configurable):
required_str = Option(str, required=True)
required_str = Option(str)
default_str = Option(str, default='foo')
integer = Option(int)
integer = Option(int, required=False)
class MyHarderConfigurable(MyConfigurable):
@ -25,14 +30,20 @@ class MyConfigurableUsingPositionalOptions(MyConfigurable):
def test_missing_required_option_error():
with inspect_node(MyConfigurable()) as ni:
assert ni.partial
with pytest.raises(TypeError) as exc:
MyConfigurable()
MyConfigurable(_final=True)
assert exc.match('missing 1 required option:')
def test_missing_required_options_error():
with inspect_node(MyHarderConfigurable()) as ni:
assert ni.partial
with pytest.raises(TypeError) as exc:
MyHarderConfigurable()
MyHarderConfigurable(_final=True)
assert exc.match('missing 2 required options:')
@ -50,6 +61,10 @@ def test_extraneous_options_error():
def test_defaults():
o = MyConfigurable(required_str='hello')
with inspect_node(o) as ni:
assert not ni.partial
assert o.required_str == 'hello'
assert o.default_str == 'foo'
assert o.integer == None
@ -57,6 +72,10 @@ def test_defaults():
def test_str_type_factory():
o = MyConfigurable(required_str=42)
with inspect_node(o) as ni:
assert not ni.partial
assert o.required_str == '42'
assert o.default_str == 'foo'
assert o.integer == None
@ -64,6 +83,10 @@ def test_str_type_factory():
def test_int_type_factory():
o = MyConfigurable(required_str='yo', default_str='bar', integer='42')
with inspect_node(o) as ni:
assert not ni.partial
assert o.required_str == 'yo'
assert o.default_str == 'bar'
assert o.integer == 42
@ -71,6 +94,10 @@ def test_int_type_factory():
def test_bool_type_factory():
o = MyHarderConfigurable(required_str='yes', also_required='True')
with inspect_node(o) as ni:
assert not ni.partial
assert o.required_str == 'yes'
assert o.default_str == 'foo'
assert o.integer == None
@ -79,6 +106,10 @@ def test_bool_type_factory():
def test_option_resolution_order():
o = MyBetterConfigurable()
with inspect_node(o) as ni:
assert not ni.partial
assert o.required_str == 'kaboom'
assert o.default_str == 'foo'
assert o.integer == None
@ -86,3 +117,20 @@ def test_option_resolution_order():
def test_option_positional():
o = MyConfigurableUsingPositionalOptions('1', '2', '3', required_str='hello')
with inspect_node(o) as ni:
assert not ni.partial
assert o.first == '1'
assert o.second == '2'
assert o.third == '3'
assert o.required_str == 'hello'
assert o.default_str == 'foo'
assert o.integer is None
def test_no_opt_configurable():
o = NoOptConfigurable()
with inspect_node(o) as ni:
assert not ni.partial

74
tests/config/test_methods.py
View File

@ -1,7 +1,5 @@
import pytest
from bonobo.config import Configurable, Method, Option
from bonobo.errors import ConfigurationError
from bonobo.util.inspect import inspect_node
class MethodBasedConfigurable(Configurable):
@ -13,23 +11,73 @@ class MethodBasedConfigurable(Configurable):
self.handler(*args, **kwargs)
def test_one_wrapper_only():
with pytest.raises(ConfigurationError):
def test_multiple_wrapper_suppored():
class TwoMethods(Configurable):
h1 = Method(required=True)
h2 = Method(required=True)
class TwoMethods(Configurable):
h1 = Method()
h2 = Method()
with inspect_node(TwoMethods) as ci:
assert ci.type == TwoMethods
assert not ci.instance
assert len(ci.options) == 2
assert not len(ci.processors)
assert not ci.partial
@TwoMethods
def OneMethod():
pass
with inspect_node(OneMethod) as ci:
assert ci.type == TwoMethods
assert not ci.instance
assert len(ci.options) == 2
assert not len(ci.processors)
assert ci.partial
@OneMethod
def transformation():
pass
with inspect_node(transformation) as ci:
assert ci.type == TwoMethods
assert ci.instance
assert len(ci.options) == 2
assert not len(ci.processors)
assert not ci.partial
def test_define_with_decorator():
calls = []
@MethodBasedConfigurable
def Concrete(self, *args, **kwargs):
def my_handler(*args, **kwargs):
calls.append((args, kwargs, ))
Concrete = MethodBasedConfigurable(my_handler)
assert callable(Concrete.handler)
assert Concrete.handler == my_handler
with inspect_node(Concrete) as ci:
assert ci.type == MethodBasedConfigurable
assert ci.partial
t = Concrete('foo', bar='baz')
assert callable(t.handler)
assert len(calls) == 0
t()
assert len(calls) == 1
def test_late_binding_method_decoration():
calls = []
@MethodBasedConfigurable(foo='foo')
def Concrete(*args, **kwargs):
calls.append((args, kwargs, ))
assert callable(Concrete.handler)
t = Concrete('foo', bar='baz')
t = Concrete(bar='baz')
assert callable(t.handler)
assert len(calls) == 0
@ -43,7 +91,7 @@ def test_define_with_argument():
def concrete_handler(*args, **kwargs):
calls.append((args, kwargs, ))
t = MethodBasedConfigurable('foo', bar='baz', handler=concrete_handler)
t = MethodBasedConfigurable(concrete_handler, 'foo', bar='baz')
assert callable(t.handler)
assert len(calls) == 0
t()
@ -71,7 +119,7 @@ def test_inheritance_then_decorate():
pass
@Inheriting
def Concrete(self, *args, **kwargs):
def Concrete(*args, **kwargs):
calls.append((args, kwargs, ))
assert callable(Concrete.handler)

66
tests/config/test_methods_partial.py Normal file
View File

@ -0,0 +1,66 @@
from unittest.mock import MagicMock
from bonobo.config import Configurable, ContextProcessor, Method, Option
from bonobo.util.inspect import inspect_node
class Bobby(Configurable):
handler = Method()
handler2 = Method()
foo = Option(positional=True)
bar = Option(required=False)
@ContextProcessor
def think(self, context):
yield 'different'
def call(self, think, *args, **kwargs):
self.handler('1', *args, **kwargs)
self.handler2('2', *args, **kwargs)
def test_partial():
C = Bobby
# inspect the configurable class
with inspect_node(C) as ci:
assert ci.type == Bobby
assert not ci.instance
assert len(ci.options) == 4
assert len(ci.processors) == 1
assert not ci.partial
# instanciate a partial instance ...
f1 = MagicMock()
C = C(f1)
with inspect_node(C) as ci:
assert ci.type == Bobby
assert not ci.instance
assert len(ci.options) == 4
assert len(ci.processors) == 1
assert ci.partial
assert ci.partial[0] == (f1, )
assert not len(ci.partial[1])
# instanciate a more complete partial instance ...
f2 = MagicMock()
C = C(f2)
with inspect_node(C) as ci:
assert ci.type == Bobby
assert not ci.instance
assert len(ci.options) == 4
assert len(ci.processors) == 1
assert ci.partial
assert ci.partial[0] == (f1, f2, )
assert not len(ci.partial[1])
c = C('foo')
with inspect_node(c) as ci:
assert ci.type == Bobby
assert ci.instance
assert len(ci.options) == 4
assert len(ci.processors) == 1
assert not ci.partial

16
tests/config/test_services.py
View File

@ -3,7 +3,7 @@ import time
import pytest
from bonobo.config import Configurable, Container, Exclusive, Service
from bonobo.config import Configurable, Container, Exclusive, Service, requires
from bonobo.config.services import validate_service_name
@ -94,3 +94,17 @@ def test_exclusive():
'hello', '0 0', '0 1', '0 2', '0 3', '0 4', '1 0', '1 1', '1 2', '1 3', '1 4', '2 0', '2 1', '2 2', '2 3',
'2 4', '3 0', '3 1', '3 2', '3 3', '3 4', '4 0', '4 1', '4 2', '4 3', '4 4'
]
def test_requires():
vcr = VCR()
services = Container(output=vcr.append)
@requires('output')
def append(out, x):
out(x)
svcargs = services.args_for(append)
assert len(svcargs) == 1
assert svcargs[0] == vcr.append

78
tests/io/test_csv.py
View File

@ -1,23 +1,21 @@
import pytest
from bonobo import Bag, CsvReader, CsvWriter, open_fs, settings
from bonobo import Bag, CsvReader, CsvWriter, settings
from bonobo.constants import BEGIN, END
from bonobo.execution.node import NodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
csv_tester = FilesystemTester('csv')
csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar'
def test_write_csv_to_file(tmpdir):
fs, filename = open_fs(tmpdir), 'output.csv'
def test_write_csv_to_file_arg0(tmpdir):
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
writer = CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0)
context = NodeExecutionContext(writer, services={'fs': fs})
context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
context.start()
context.step()
context.step()
context.stop()
with NodeExecutionContext(CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
context.step()
context.step()
with fs.open(filename) as fp:
assert fp.read() == 'foo\nbar\nbaz\n'
@ -26,19 +24,33 @@ def test_write_csv_to_file(tmpdir):
getattr(context, 'file')
def test_read_csv_from_file(tmpdir):
fs, filename = open_fs(tmpdir), 'input.csv'
with fs.open(filename, 'w') as fp:
fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar')
@pytest.mark.parametrize('add_kwargs', ({}, {
'ioformat': settings.IOFORMAT_KWARGS,
}, ))
def test_write_csv_to_file_kwargs(tmpdir, add_kwargs):
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
reader = CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0)
with NodeExecutionContext(CsvWriter(path=filename, **add_kwargs), services=services) as context:
context.write(BEGIN, Bag(**{'foo': 'bar'}), Bag(**{'foo': 'baz', 'ignore': 'this'}), END)
context.step()
context.step()
context = CapturingNodeExecutionContext(reader, services={'fs': fs})
with fs.open(filename) as fp:
assert fp.read() == 'foo\nbar\nbaz\n'
context.start()
context.write(BEGIN, Bag(), END)
context.step()
context.stop()
with pytest.raises(AttributeError):
getattr(context, 'file')
def test_read_csv_from_file_arg0(tmpdir):
fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
with CapturingNodeExecutionContext(
CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0),
services=services,
) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2
@ -59,19 +71,15 @@ def test_read_csv_from_file(tmpdir):
}
def test_read_csv_kwargs_output_formater(tmpdir):
fs, filename = open_fs(tmpdir), 'input.csv'
with fs.open(filename, 'w') as fp:
fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar')
def test_read_csv_from_file_kwargs(tmpdir):
fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
reader = CsvReader(path=filename, delimiter=',')
context = CapturingNodeExecutionContext(reader, services={'fs': fs})
context.start()
context.write(BEGIN, Bag(), END)
context.step()
context.stop()
with CapturingNodeExecutionContext(
CsvReader(path=filename, delimiter=','),
services=services,
) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2

60
tests/io/test_file.py
View File

@ -1,9 +1,22 @@
import pytest
from bonobo import Bag, FileReader, FileWriter, open_fs
from bonobo import Bag, FileReader, FileWriter
from bonobo.constants import BEGIN, END
from bonobo.execution.node import NodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
txt_tester = FilesystemTester('txt')
txt_tester.input_data = 'Hello\nWorld\n'
def test_file_writer_contextless(tmpdir):
fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
with FileWriter(path=filename).open(fs) as fp:
fp.write('Yosh!')
with fs.open(filename) as fp:
assert fp.read() == 'Yosh!'
@pytest.mark.parametrize(
@ -14,46 +27,23 @@ from bonobo.util.testing import CapturingNodeExecutionContext
]
)
def test_file_writer_in_context(tmpdir, lines, output):
fs, filename = open_fs(tmpdir), 'output.txt'
fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
writer = FileWriter(path=filename)
context = NodeExecutionContext(writer, services={'fs': fs})
context.start()
context.write(BEGIN, *map(Bag, lines), END)
for _ in range(len(lines)):
context.step()
context.stop()
with NodeExecutionContext(FileWriter(path=filename), services=services) as context:
context.write(BEGIN, *map(Bag, lines), END)
for _ in range(len(lines)):
context.step()
with fs.open(filename) as fp:
assert fp.read() == output
def test_file_writer_out_of_context(tmpdir):
fs, filename = open_fs(tmpdir), 'output.txt'
def test_file_reader(tmpdir):
fs, filename, services = txt_tester.get_services_for_reader(tmpdir)
writer = FileWriter(path=filename)
with writer.open(fs) as fp:
fp.write('Yosh!')
with fs.open(filename) as fp:
assert fp.read() == 'Yosh!'
def test_file_reader_in_context(tmpdir):
fs, filename = open_fs(tmpdir), 'input.txt'
with fs.open(filename, 'w') as fp:
fp.write('Hello\nWorld\n')
reader = FileReader(path=filename)
context = CapturingNodeExecutionContext(reader, services={'fs': fs})
context.start()
context.write(BEGIN, Bag(), END)
context.step()
context.stop()
with CapturingNodeExecutionContext(FileReader(path=filename), services=services) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2

56
tests/io/test_json.py
View File

@ -1,44 +1,48 @@
import pytest
from bonobo import Bag, JsonReader, JsonWriter, open_fs, settings
from bonobo import Bag, JsonReader, JsonWriter, settings
from bonobo.constants import BEGIN, END
from bonobo.execution.node import NodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
json_tester = FilesystemTester('json')
json_tester.input_data = '''[{"x": "foo"},{"x": "bar"}]'''
def test_write_json_to_file(tmpdir):
fs, filename = open_fs(tmpdir), 'output.json'
def test_write_json_arg0(tmpdir):
fs, filename, services = json_tester.get_services_for_writer(tmpdir)
writer = JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0)
context = NodeExecutionContext(writer, services={'fs': fs})
context.start()
context.write(BEGIN, Bag({'foo': 'bar'}), END)
context.step()
context.stop()
with NodeExecutionContext(JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
context.write(BEGIN, Bag({'foo': 'bar'}), END)
context.step()
with fs.open(filename) as fp:
assert fp.read() == '[{"foo": "bar"}]'
with pytest.raises(AttributeError):
getattr(context, 'file')
with pytest.raises(AttributeError):
getattr(context, 'first')
@pytest.mark.parametrize('add_kwargs', ({}, {
'ioformat': settings.IOFORMAT_KWARGS,
}, ))
def test_write_json_kwargs(tmpdir, add_kwargs):
fs, filename, services = json_tester.get_services_for_writer(tmpdir)
with NodeExecutionContext(JsonWriter(filename, **add_kwargs), services=services) as context:
context.write(BEGIN, Bag(**{'foo': 'bar'}), END)
context.step()
with fs.open(filename) as fp:
assert fp.read() == '[{"foo": "bar"}]'
def test_read_json_from_file(tmpdir):
fs, filename = open_fs(tmpdir), 'input.json'
with fs.open(filename, 'w') as fp:
fp.write('[{"x": "foo"},{"x": "bar"}]')
reader = JsonReader(filename, ioformat=settings.IOFORMAT_ARG0)
def test_read_json_arg0(tmpdir):
fs, filename, services = json_tester.get_services_for_reader(tmpdir)
context = CapturingNodeExecutionContext(reader, services={'fs': fs})
context.start()
context.write(BEGIN, Bag(), END)
context.step()
context.stop()
with CapturingNodeExecutionContext(
JsonReader(filename, ioformat=settings.IOFORMAT_ARG0),
services=services,
) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2

39
tests/io/test_pickle.py
View File

@ -2,24 +2,22 @@ import pickle
import pytest
from bonobo import Bag, PickleReader, PickleWriter, open_fs, settings
from bonobo import Bag, PickleReader, PickleWriter, settings
from bonobo.constants import BEGIN, END
from bonobo.execution.node import NodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
pickle_tester = FilesystemTester('pkl', mode='wb')
pickle_tester.input_data = pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']])
def test_write_pickled_dict_to_file(tmpdir):
fs, filename = open_fs(tmpdir), 'output.pkl'
fs, filename, services = pickle_tester.get_services_for_writer(tmpdir)
writer = PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0)
context = NodeExecutionContext(writer, services={'fs': fs})
context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
context.start()
context.step()
context.step()
context.stop()
with NodeExecutionContext(PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
context.step()
context.step()
with fs.open(filename, 'rb') as fp:
assert pickle.loads(fp.read()) == {'foo': 'bar'}
@ -29,18 +27,13 @@ def test_write_pickled_dict_to_file(tmpdir):
def test_read_pickled_list_from_file(tmpdir):
fs, filename = open_fs(tmpdir), 'input.pkl'
with fs.open(filename, 'wb') as fp:
fp.write(pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']]))
fs, filename, services = pickle_tester.get_services_for_reader(tmpdir)
reader = PickleReader(filename, ioformat=settings.IOFORMAT_ARG0)
context = CapturingNodeExecutionContext(reader, services={'fs': fs})
context.start()
context.write(BEGIN, Bag(), END)
context.step()
context.stop()
with CapturingNodeExecutionContext(
PickleReader(filename, ioformat=settings.IOFORMAT_ARG0), services=services
) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2

1
tests/test_basics.py
View File

@ -5,6 +5,7 @@ import pytest
import bonobo
from bonobo.config.processors import ContextCurrifier
from bonobo.constants import NOT_MODIFIED
from bonobo.util.inspect import inspect_node
def test_count():

63
tests/test_settings.py Normal file
View File

@ -0,0 +1,63 @@
import logging
from os import environ
from unittest.mock import patch
import pytest
from bonobo import settings
TEST_SETTING = 'TEST_SETTING'
def test_to_bool():
assert not settings.to_bool('')
assert not settings.to_bool('FALSE')
assert not settings.to_bool('NO')
assert not settings.to_bool('0')
assert settings.to_bool('yup')
assert settings.to_bool('True')
assert settings.to_bool('yes')
assert settings.to_bool('1')
def test_setting():
s = settings.Setting(TEST_SETTING)
assert s.get() is None
with patch.dict(environ, {TEST_SETTING: 'hello'}):
assert s.get() is None
s.clear()
assert s.get() == 'hello'
s = settings.Setting(TEST_SETTING, default='nope')
assert s.get() is 'nope'
with patch.dict(environ, {TEST_SETTING: 'hello'}):
assert s.get() == 'nope'
s.clear()
assert s.get() == 'hello'
def test_default_settings():
settings.clear_all()
assert settings.DEBUG.get() == False
assert settings.PROFILE.get() == False
assert settings.QUIET.get() == False
assert settings.LOGGING_LEVEL.get() == logging._checkLevel('INFO')
with patch.dict(environ, {'DEBUG': 't'}):
settings.clear_all()
assert settings.LOGGING_LEVEL.get() == logging._checkLevel('DEBUG')
settings.clear_all()
def test_check():
settings.check()
with patch.dict(environ, {'DEBUG': 't', 'PROFILE': 't', 'QUIET': 't'}):
settings.clear_all()
with pytest.raises(RuntimeError):
settings.check()
settings.clear_all()