First implementation of services and basic injection. Not working with CLI for now.

This commit is contained in:
Romain Dorgueil
2017-04-25 22:04:21 +02:00
parent 18abb39206
commit efcd4361cc
41 changed files with 538 additions and 324 deletions

View File

@ -12,13 +12,13 @@ import warnings
assert (sys.version_info >= (3, 5)), 'Python 3.5+ is required to use Bonobo.'
from ._version import __version__
from .basics import __all__ as __all_basics__
from .config import __all__ as __all_config__
from .context import __all__ as __all_context__
from .core import __all__ as __all_core__
from .execution import __all__ as __all_execution__
from .io import __all__ as __all_io__
from .util import __all__ as __all_util__
from .strategies import __all__ as __all_strategies__
__all__ = __all_config__ + __all_context__ + __all_core__ + __all_io__ + __all_util__ + [
__all__ = __all_basics__ + __all_config__ + __all_execution__ + __all_io__ + __all_strategies__ + [
'Bag',
'ErrorBag'
'Graph',
@ -29,14 +29,14 @@ __all__ = __all_config__ + __all_context__ + __all_core__ + __all_io__ + __all_u
'run',
]
from .basics import *
from .config import *
from .context import *
from .core import *
from .execution import *
from .io import *
from .strategies import *
from .structs.bags import *
from .structs.graphs import *
from .structs.tokens import *
from .util import *
DEFAULT_STRATEGY = 'threadpool'
@ -54,7 +54,7 @@ def get_examples_path(*pathsegments):
def create_strategy(name=None):
from bonobo.core.strategies.base import Strategy
from bonobo.strategies.base import Strategy
import logging
if isinstance(name, Strategy):
@ -87,7 +87,7 @@ def _is_jupyter_notebook():
return False
def run(graph, *chain, strategy=None, plugins=None):
def run(graph, *chain, strategy=None, plugins=None, services=None):
if len(chain):
warnings.warn('DEPRECATED. You should pass a Graph instance instead of a chain.')
from bonobo import Graph
@ -106,7 +106,7 @@ def run(graph, *chain, strategy=None, plugins=None):
if JupyterOutputPlugin not in plugins:
plugins.append(JupyterOutputPlugin)
return strategy.execute(graph, plugins=plugins)
return strategy.execute(graph, plugins=plugins, services=services)
del sys

100
bonobo/basics.py Normal file
View File

@ -0,0 +1,100 @@
import functools
from pprint import pprint as _pprint
from colorama import Fore, Style
from bonobo.config.processors import contextual
from bonobo.constants import NOT_MODIFIED
from bonobo.structs.bags import Bag
from bonobo.util.objects import ValueHolder
from bonobo.util.term import CLEAR_EOL
__all__ = [
'identity',
'Limit',
'Tee',
'count',
'pprint',
'PrettyPrint',
'noop',
]
def identity(x):
return x
def Limit(n=10):
i = 0
def _limit(*args, **kwargs):
nonlocal i, n
i += 1
if i <= n:
yield NOT_MODIFIED
_limit.__name__ = 'Limit({})'.format(n)
return _limit
def Tee(f):
@functools.wraps(f)
def wrapped(*args, **kwargs):
nonlocal f
f(*args, **kwargs)
return NOT_MODIFIED
return wrapped
@contextual
def count(counter, *args, **kwargs):
counter += 1
@count.add_context_processor
def _count_counter(self, context):
counter = ValueHolder(0)
yield counter
context.send(Bag(counter.value))
pprint = Tee(_pprint)
def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):
def _pprint(*args, **kwargs):
nonlocal title_keys, sort, print_values
row = args[0]
for key in title_keys:
if key in row:
print(Style.BRIGHT, row.get(key), Style.RESET_ALL, sep='')
break
if print_values:
for k in sorted(row) if sort else row:
print(
'',
Fore.BLUE,
k,
Style.RESET_ALL,
' : ',
Fore.BLACK,
'(',
type(row[k]).__name__,
')',
Style.RESET_ALL,
' ',
repr(row[k]),
CLEAR_EOL,
)
yield NOT_MODIFIED
_pprint.__name__ = 'pprint'
return _pprint
def noop(*args, **kwargs): # pylint: disable=unused-argument
return NOT_MODIFIED

11
bonobo/config/__init__.py Normal file
View File

@ -0,0 +1,11 @@
from bonobo.config.configurables import Configurable
from bonobo.config.options import Option
from bonobo.config.services import Container, Service
from bonobo.config.processors import ContextProcessor
__all__ = [
'Configurable',
'Container',
'Option',
'Service',
]

View File

@ -1,26 +1,11 @@
__all__ = [
'Configurable',
'Option',
]
class Option:
def __init__(self, type=None, *, required=False, default=None):
self.name = None
self.type = type
self.required = required
self.default = default
def __get__(self, inst, typ):
if not self.name in inst.__options_values__:
inst.__options_values__[self.name] = self.default() if callable(self.default) else self.default
return inst.__options_values__[self.name]
def __set__(self, inst, value):
inst.__options_values__[self.name] = self.type(value) if self.type else value
from bonobo.config.options import Option
class ConfigurableMeta(type):
"""
Metaclass for Configurables that will add options to a special __options__ dict.
"""
def __init__(cls, what, bases=None, dict=None):
super().__init__(what, bases, dict)
cls.__options__ = {}

27
bonobo/config/options.py Normal file
View File

@ -0,0 +1,27 @@
class Option:
"""
An Option is a descriptor for a required or optional parameter of a Configurable.
"""
_creation_counter = 0
def __init__(self, type=None, *, required=False, default=None):
self.name = None
self.type = type
self.required = required
self.default = default
# This hack is necessary for python3.5
self._creation_counter = Option._creation_counter
Option._creation_counter += 1
def get_default(self):
return self.default() if callable(self.default) else self.default
def __get__(self, inst, typ):
if not self.name in inst.__options_values__:
inst.__options_values__[self.name] = self.get_default()
return inst.__options_values__[self.name]
def __set__(self, inst, value):
inst.__options_values__[self.name] = self.type(value) if self.type else value

View File

@ -1,20 +1,12 @@
from functools import partial
import functools
import types
from bonobo.util.compat import deprecated_alias
_CONTEXT_PROCESSORS_ATTR = '__processors__'
def get_context_processors(mixed):
if isinstance(mixed, types.FunctionType):
yield from getattr(mixed, _CONTEXT_PROCESSORS_ATTR, ())
for cls in reversed((mixed if isinstance(mixed, type) else type(mixed)).__mro__):
yield from cls.__dict__.get(_CONTEXT_PROCESSORS_ATTR, ())
return ()
class ContextProcessor:
_creation_counter = 0
@ -47,7 +39,7 @@ def contextual(cls_or_func):
:param cls_or_func:
"""
if not add_context_processor.__name__ in cls_or_func.__dict__:
setattr(cls_or_func, add_context_processor.__name__, partial(add_context_processor, cls_or_func))
setattr(cls_or_func, add_context_processor.__name__, functools.partial(add_context_processor, cls_or_func))
if isinstance(cls_or_func, types.FunctionType):
try:
@ -63,6 +55,20 @@ def contextual(cls_or_func):
for name, value in cls_or_func.__dict__.items():
if isinstance(value, ContextProcessor):
_processors.append(value)
# This is needed for python 3.5, python 3.6 should be fine, but it's considered an implementation detail.
_processors.sort(key=lambda proc: proc._creation_counter)
return cls_or_func
def resolve_processors(mixed):
if isinstance(mixed, types.FunctionType):
yield from getattr(mixed, _CONTEXT_PROCESSORS_ATTR, ())
for cls in reversed((mixed if isinstance(mixed, type) else type(mixed)).__mro__):
yield from cls.__dict__.get(_CONTEXT_PROCESSORS_ATTR, ())
return ()
get_context_processors = deprecated_alias('get_context_processors', resolve_processors)

77
bonobo/config/services.py Normal file
View File

@ -0,0 +1,77 @@
import re
from bonobo.config.options import Option
_service_name_re = re.compile(r"^[^\d\W]\w*(:?\.[^\d\W]\w*)*$", re.UNICODE)
def validate_service_name(name):
if not _service_name_re.match(name):
raise ValueError('Invalid service name {!r}.'.format(name))
return name
class Service(Option):
"""
A Service is a special kind of option defining a dependency to something that will be resolved at runtime, using an
identifier. For example, you can create a Configurable that has a "database" Service in its attribute, meaning that
you'll define which database to use, by name, when creating the instance of this class, then provide an
implementation when running the graph using a strategy.
Example::
import bonobo
class QueryExtractor(bonobo.Configurable):
database = bonobo.Service(default='sqlalchemy.engine.default')
graph = bonobo.Graph(
QueryExtractor(database='sqlalchemy.engine.secondary'),
*more_transformations,
)
if __name__ == '__main__':
engine = create_engine('... dsn ...')
bonobo.run(graph, services={
'sqlalchemy.engine.secondary': engine
})
The main goal is not to tie transformations to actual dependencies, so the same can be run in different contexts
(stages like preprod, prod, or tenants like client1, client2, or anything you want).
"""
def __init__(self, type=None, *, required=False, default=None):
super().__init__(type, required=required, default=default)
def __set__(self, inst, value):
inst.__options_values__[self.name] = validate_service_name(value)
def resolve(self, inst, services):
name = getattr(inst, self.name)
if not name in services:
raise KeyError('Cannot resolve service {!r} using provided service collection.'.format(name))
return services.get(name)
class Container(dict):
def __new__(cls, *args, **kwargs):
if len(args) == 1:
assert not len(kwargs), 'only one usage at a time, my dear.'
if not(args[0]):
return super().__new__(cls)
if isinstance(args[0], cls):
return cls
return super().__new__(cls, *args, **kwargs)
def args_for(self, mixed):
try:
options = mixed.__options__
except AttributeError:
options = {}
return tuple(
option.resolve(mixed, self)
for name, option in options.items()
if isinstance(option, Service)
)

View File

@ -1,6 +0,0 @@
from bonobo.context.processors import contextual, ContextProcessor
__all__ = [
'ContextProcessor',
'contextual',
]

View File

@ -1,13 +1,2 @@
""" Core required libraries. """
from .services import inject, service
from .strategies.executor import ThreadPoolExecutorStrategy, ProcessPoolExecutorStrategy
from .strategies.naive import NaiveStrategy
__all__ = [
'NaiveStrategy',
'ProcessPoolExecutorStrategy',
'ThreadPoolExecutorStrategy',
'inject',
'service',
]

View File

@ -19,7 +19,7 @@ from queue import Queue
from bonobo.errors import AbstractError, InactiveWritableError, InactiveReadableError
from bonobo.constants import BEGIN, END
from bonobo.util import noop
from bonobo.basics import noop
BUFFER_SIZE = 8192

View File

@ -1,54 +0,0 @@
import functools
import itertools
class service:
def __init__(self, factory):
self.factory = factory
self.instance = None
# self.__call__ = functools.wraps(self.__call__)
self.children = set()
def __call__(self, *args, **kwargs):
if self.instance is None:
self.instance = self.factory(*args, **kwargs)
return self.instance
def __getitem__(self, item):
if item not in self.children:
raise KeyError(item)
return item
def define(self, *args, **kwargs):
new_service = type(self)(functools.partial(self.factory, *args, **kwargs))
self.children.add(new_service)
return new_service
call = lambda s: s()
def resolve(func):
return func()
def inject(*iargs, **ikwargs):
"""
Inject service dependencies.
TODO: ikwargs are ignored, implement that
"""
def wrapper(target):
@functools.wraps(target)
def wrapped(*args, **kwargs):
return target(
*itertools.chain(map(resolve, iargs), args),
**{ ** kwargs, ** {k: resolve(v)
for k, v in ikwargs.items()}}
)
return wrapped
return wrapper

View File

@ -1,7 +1,8 @@
import json
import os
from bonobo import Tee, JsonWriter, Graph, get_examples_path
from bonobo import JsonWriter, Graph, get_examples_path
from bonobo.basics import Tee
from bonobo.ext.opendatasoft import OpenDataSoftAPI
from colorama import Fore, Style

View File

@ -130,4 +130,7 @@
{"city": "Clermont-Ferrand", "kind_name": "mini_fab_lab", "links": ["http://acolab.fr"], "capabilities": "three_d_printing;circuit_production;vinyl_cutting", "url": "https://www.fablabs.io/labs/acolab", "coordinates": [45.7941993299, 3.07563051059], "name": "ACoLab", "phone": "+33(0)651800518", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/15/48/fd42c5cd-21ac-4abf-9a20-8f9bb602c7b1/ACoLab.jpg", "postal_code": "63000", "longitude": 3.07563051058958, "country_code": "fr", "latitude": 45.7941993298608, "address_1": "2 bis rue du Clos Perret", "address_notes": "Au quatri\u00e8me \u00e9tage du b\u00e2timent, entr\u00e9e par le 2bis rue du Clos Perret\r\n\r\nAdresse 'historique' (2013/Mai2015), chez les Petits D\u00e9brouillards d'Auvergne : 32 Rue du Pont Naturel, 63000 Clermont-Ferrand\r\nIl faut traverser la petite place entre les immeubles et descendre quelques marches.", "email": "contact@acolab.fr", "blurb": "Atelier Collaboratif - Ouvert les lundi et mercredi soir", "description": "FabLab associatif cr\u00e9e en 2013\r\n\u00c9quip\u00e9 d'une d\u00e9coupeuse vinyle, d'une imprimante 3D type Mendel Max, d'un petit tour \u00e0 m\u00e9taux, utilisation d'Arduino, de Raspberry Pi...\r\n\r\nBeaucoup de r\u00e9cup\u00e9ration et de bidouillages vari\u00e9s dans la bonne humeur et le partage.", "geometry": {"type": "Point", "coordinates": [3.07563051059, 45.7941993299]}, "country": "France"},
{"city": "brest", "kind_name": "fab_lab", "links": ["http://wiki.lesfabriquesduponant.net", "http://www.lesfabriquesduponant.net"], "url": "https://www.fablabs.io/labs/lesfabriquesduponant", "name": "Les Fabriques du Ponant", "longitude": -4.47982980000006, "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/48/32/6d2e62f0-0f08-424a-883e-b9a15e90ee8a/Les Fabriques du Ponant.jpg", "phone": "+33.685176295", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/02/08/10/11/15/20e09c48-5ac6-40fc-8462-bce909c24de0/531px-Logofabdupo.png", "postal_code": "29200", "coordinates": [48.4086189, -4.4798298], "country_code": "fr", "latitude": 48.4086189, "address_1": "40, rue Jules Lesven", "capabilities": "three_d_printing;cnc_milling;circuit_production;laser;vinyl_cutting", "email": "contact@lesfabriquesduponant.net", "blurb": "\"Les Fabrique du Ponant\" is run by \"T\u00e9l\u00e9com Bretagne\" and \"Les petits d\u00e9brouillards\". Its main goal is to propose digital manufacturing services, organise digital cultural events and digital education", "description": "Installed in high school Vauban in Brest, \"Les Fabrique du Ponant\" (which can be translate in \"Factories Ponant\") offer a coworking space, a fully equipped fablab, a webTV studio, a training room. \"Les Fabrique du Ponant\" organize demonstrations (initiation days and discovery), cultural events on digital as the \"Open Bidouille Camp\" or \"Science Hack Day\", trainings, educational activities.", "geometry": {"type": "Point", "coordinates": [-4.4798298, 48.4086189]}, "country": "France"},
{"city": "Tours", "coordinates": [47.3932037, 0.6687421], "kind_name": "mini_fab_lab", "links": ["http://funlab.fr"], "url": "https://www.fablabs.io/labs/funlab", "name": "FunLab Tours", "longitude": 0.668742100000031, "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/50/06/7863f4ba-28b3-4018-9351-c1d8c70a5b69/FunLab Tours.jpg", "phone": "+33603951216", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/13/52/7d30f2aa-d5b7-482a-8334-a72d17e0a6fe/FunLab Tours.jpg", "postal_code": "37000", "capabilities": "three_d_printing;cnc_milling;laser;vinyl_cutting", "country_code": "fr", "latitude": 47.3932037, "address_1": "49, boulevard Preuilly", "address_notes": "Nous sommes occupants de site MAME \"cit\u00e9 de la cr\u00e9ation et du num\u00e9rique\"", "email": "contact@funlab.fr", "blurb": "Fabrique d'Usages Num\u00e9riques", "description": "La communaut\u00e9 existe, des rencontres toutes les semaines. 49, Boulevard Preuilly, 37000 Tours.", "geometry": {"type": "Point", "coordinates": [0.6687421, 47.3932037]}, "country": "France"},
{"city": "Bron", "kind_name": "fab_lab", "links": ["http://fablab-lyon.fr"], "capabilities": "three_d_printing;cnc_milling;laser;vinyl_cutting", "url": "https://www.fablabs.io/labs/fabriquedobjetslibres", "name": "Fabrique d'Objets Libres", "email": "contact@fabriquedobjetslibres.fr", "coordinates": [45.7429334, 4.9082135], "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/50/01/0190e790-aaec-4f2f-9985-11156655145d/Fabrique d'Objets Libres.jpg", "county": "Rh\u00f4ne", "phone": "+33 7 68 01 40 26 (Tue-Sat 2pm-6pm)", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/13/49/73ea9f2d-0216-4f52-a6bf-2ff97ee474b2/Fabrique d'Objets Libres.jpg", "postal_code": "69500", "longitude": 4.90821349999999, "country_code": "fr", "latitude": 45.7429334, "address_1": "All\u00e9e Gaillard Romanet", "address_notes": "Au sous-sol de la MJC. Downstairs inside the MJC.", "address_2": "MJC Louis Aragon", "blurb": "Le fablab lyonnais, install\u00e9 \u00e0 la MJC Louis Aragon de Bron, ouvert tous les mercredis et formation hebdomadaire de fabrication num\u00e9rique. Projets autour du handicap, des arts et du recyclage.", "description": "La Fabrique d'Objets Libres est un fablab associatif sur Lyon et sa r\u00e9gion. Install\u00e9 \u00e0 la MJC Louis Aragon de Bron depuis janvier 2013, c'est un espace de cr\u00e9ation et de fabrication num\u00e9rique ouvert \u00e0 tous, qui permet \u00e0 chacun de d\u00e9couvrir, d'inventer et de fabriquer tout type d'objet.\r\n \r\nV\u00e9ritable laboratoire citoyen de fabrication, la Fabrique d\u2019Objets Libres met \u00e0 disposition de ses adh\u00e9rents des outils \u00e0 commande num\u00e9rique et des mati\u00e8res premi\u00e8res secondaires permettant de concevoir et de fabriquer localement des objets libres.\r\nC\u2019est une plate-forme pluridisciplinaire collaborative qui m\u00eale les profils (techniciens, informaticiens, ing\u00e9nieurs, scientifiques, bricoleurs, cr\u00e9ateurs...) et les g\u00e9n\u00e9rations afin de r\u00e9unir tous types de comp\u00e9tences.\r\n\r\nLe fablab est ouvert tous les mercredis pour les \"temps libres\", durant lesquels les adh\u00e9rents utilisent les machines librement. Par ailleurs, il propose un atelier hebdomadaire aux adh\u00e9rents de la MJC, \"De l'id\u00e9e \u00e0 l'objet\": en une dizaine de s\u00e9ances sur un trimestre, les participants apprennent \u00e0 utiliser toutes les machines du fablab pour r\u00e9aliser leurs objets, et r\u00e9fl\u00e9chissent autour d'une th\u00e9matique sociale comme le handicap, la musique, ou la ville.\r\n\r\nL'association organise \u00e9galement des \u00e9v\u00e9nements et ateliers th\u00e9matiques utilisant la fabrication num\u00e9rique autour de sujet plus vastes, comme l'art, avec les machines \u00e0 dessiner, ou le handicap, dans le cadre du projet Handilab, ou encore la fin de vie des objets, avec le Laboratoire de l'Obsolescence D\u00e9programm\u00e9e. Enfin, le fablab s'associe \u00e0 d'autres associations et des entreprises pour des projets communs.", "geometry": {"type": "Point", "coordinates": [4.9082135, 45.7429334]}, "country": "France"}
{"city": "Bron", "kind_name": "fab_lab", "links": ["http://fablab-lyon.fr"], "capabilities": "three_d_printing;cnc_milling;laser;vinyl_cutting", "url": "https://www.fablabs.io/labs/fabriquedobjetslibres", "name": "Fabrique d'Objets Libres", "email": "contact@fabriquedobjetslibres.fr", "coordinates": [45.7429334, 4.9082135], "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/50/01/0190e790-aaec-4f2f-9985-11156655145d/Fabrique d'Objets Libres.jpg", "county": "Rh\u00f4ne", "phone": "+33 7 68 01 40 26 (Tue-Sat 2pm-6pm)", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/13/49/73ea9f2d-0216-4f52-a6bf-2ff97ee474b2/Fabrique d'Objets Libres.jpg", "postal_code": "69500", "longitude": 4.90821349999999, "country_code": "fr", "latitude": 45.7429334, "address_1": "All\u00e9e Gaillard Romanet", "address_notes": "Au sous-sol de la MJC. Downstairs inside the MJC.", "address_2": "MJC Louis Aragon", "blurb": "Le fablab lyonnais, install\u00e9 \u00e0 la MJC Louis Aragon de Bron, ouvert tous les mercredis et formation hebdomadaire de fabrication num\u00e9rique. Projets autour du handicap, des arts et du recyclage.", "description": "La Fabrique d'Objets Libres est un fablab associatif sur Lyon et sa r\u00e9gion. Install\u00e9 \u00e0 la MJC Louis Aragon de Bron depuis janvier 2013, c'est un espace de cr\u00e9ation et de fabrication num\u00e9rique ouvert \u00e0 tous, qui permet \u00e0 chacun de d\u00e9couvrir, d'inventer et de fabriquer tout type d'objet.\r\n \r\nV\u00e9ritable laboratoire citoyen de fabrication, la Fabrique d\u2019Objets Libres met \u00e0 disposition de ses adh\u00e9rents des outils \u00e0 commande num\u00e9rique et des mati\u00e8res premi\u00e8res secondaires permettant de concevoir et de fabriquer localement des objets libres.\r\nC\u2019est une plate-forme pluridisciplinaire collaborative qui m\u00eale les profils (techniciens, informaticiens, ing\u00e9nieurs, scientifiques, bricoleurs, cr\u00e9ateurs...) et les g\u00e9n\u00e9rations afin de r\u00e9unir tous types de comp\u00e9tences.\r\n\r\nLe fablab est ouvert tous les mercredis pour les \"temps libres\", durant lesquels les adh\u00e9rents utilisent les machines librement. Par ailleurs, il propose un atelier hebdomadaire aux adh\u00e9rents de la MJC, \"De l'id\u00e9e \u00e0 l'objet\": en une dizaine de s\u00e9ances sur un trimestre, les participants apprennent \u00e0 utiliser toutes les machines du fablab pour r\u00e9aliser leurs objets, et r\u00e9fl\u00e9chissent autour d'une th\u00e9matique sociale comme le handicap, la musique, ou la ville.\r\n\r\nL'association organise \u00e9galement des \u00e9v\u00e9nements et ateliers th\u00e9matiques utilisant la fabrication num\u00e9rique autour de sujet plus vastes, comme l'art, avec les machines \u00e0 dessiner, ou le handicap, dans le cadre du projet Handilab, ou encore la fin de vie des objets, avec le Laboratoire de l'Obsolescence D\u00e9programm\u00e9e. Enfin, le fablab s'associe \u00e0 d'autres associations et des entreprises pour des projets communs.", "geometry": {"type": "Point", "coordinates": [4.9082135, 45.7429334]}, "country": "France"},
{"city": "N\u00e9ons-sur-Creuse", "kind_name": "fab_lab", "links": ["http://www.rurallab.org"], "url": "https://www.fablabs.io/labs/rurallab", "coordinates": [46.744746, 0.931698], "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/49/00/95c7b9f2-a034-4b2b-931d-43ced33ddfb1/RuralLab.jpg", "phone": "+33603318810", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/12/49/ec5f7c54-e6ce-40fd-b5c5-c4142d208e6b/RuralLab.jpg", "postal_code": "36220", "longitude": 0.931697999999983, "country_code": "fr", "latitude": 46.744746, "address_1": "Rue de l'\u00c9cole", "email": "rurallab36@gmail.com", "blurb": "A FabLab in the countryside in Neons sur Creuse, France", "name": "RuralLab", "geometry": {"type": "Point", "coordinates": [0.931698, 46.744746]}, "country": "France"},
{"city": "Gif-sur-Yvette", "kind_name": "supernode", "links": ["http://fablab.digiscope.fr/#!/", "http://fablabdigiscope.wordpress.com"], "url": "https://www.fablabs.io/labs/fablabdigiscope", "name": "(Fab)Lab Digiscope", "longitude": 2.16830979999997, "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/52/18/8d63351d-c2fb-4a90-8e58-bb45422202a6/(Fab)Lab Digiscope.jpg", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/15/46/51553da4-b295-426c-837f-934c311933ba/(Fab)Lab Digiscope.jpg", "postal_code": "91190", "coordinates": [48.7117632, 2.1683098], "country_code": "fr", "latitude": 48.7117632, "address_1": "660 Rue Noetzlin", "capabilities": "three_d_printing;cnc_milling;circuit_production;laser;precision_milling;vinyl_cutting", "email": "fablabdigiscope@gmail.com", "blurb": "(FAB)LAB DIGISCOPE is a fabrication laboratory dedicated to research in sciences | design | education | art | engineering and what ever field of research you come from. Open to Everyone. Book now!", "description": "(FAB)LAB DIGISCOPE is a fabrication laboratory dedicated to research in sciences | design | education | arts | engineering and what ever field of research you come from. We host Fab Academy and Bio Academy. We host Digital Fabrication Classes for EITC Master. Open to Everyone since the beginning.\r\n\r\nFablab Digiscope started in 2013 when Aviz-INRIA research team director Jean-Daniel Fekete and colleague researcher Pierre Dragicevic hired Romain Di Vozzo as a R&D Engineer to be the fablab manager of what would later become an attractive place on the new Campus Paris-Saclay. Fablab Digiscope is part of the Digiscope Project, a network of 10 high-performance platforms for interactive visualization of large datasets and complex computation for which Michel Beaudouin-Lafon is the scientific Director. Fablab Digiscope is mutualised between 10 institutions involved in research and education.\r\n\r\nRomain Di Vozzo runs and develops Fablab Digiscope everyday, trains publics, designs objects, shares creative thoughts, gives advices on designs, etc. Romain also actively collaborates to the globally distributed fablab network and with the Fab Foundation by operating as Fab Academy SuperNode, as Instructor for Fab Academy and Bio Academy, by giving conferences and workshops in France and abroad and by performing very small tasks that make the fablab network grow.", "geometry": {"type": "Point", "coordinates": [2.1683098, 48.7117632]}, "country": "France"},
{"city": "Metz", "kind_name": "fab_lab", "links": ["http://graoulab.org/wiki", "http://graoulab.org"], "url": "https://www.fablabs.io/labs/graoulab", "coordinates": [49.1262692, 6.182086], "name": "GraouLab", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/18/24/af4709d8-1f60-48a7-ba35-4c42ef40a195/GraouLab.jpg", "postal_code": "57000", "longitude": 6.18208600000003, "country_code": "fr", "latitude": 49.1262692, "capabilities": "three_d_printing;laser", "email": "contact@graoulab.org", "blurb": "The FabLab of Metz. A place for folks innovation.", "address_1": "7 Avenue de Blida", "geometry": {"type": "Point", "coordinates": [6.182086, 49.1262692]}, "country": "France"}

View File

@ -1,6 +1,7 @@
import bonobo
import bonobo.basics
graph = bonobo.Graph(range(42), bonobo.count, print)
graph = bonobo.Graph(range(42), bonobo.basics.count, print)
if __name__ == '__main__':
bonobo.run(graph)

View File

@ -1,17 +1,25 @@
import traceback
import sys
import traceback
from functools import partial
from queue import Empty
from time import sleep
from bonobo.constants import BEGIN, END, NOT_MODIFIED, INHERIT_INPUT
from bonobo.context.processors import get_context_processors
from bonobo.config import Container
from bonobo.config.processors import resolve_processors
from bonobo.constants import BEGIN, END, INHERIT_INPUT, NOT_MODIFIED
from bonobo.core.inputs import Input
from bonobo.core.statistics import WithStatistics
from bonobo.errors import InactiveReadableError
from bonobo.structs.bags import Bag, ErrorBag
from bonobo.util.iterators import ensure_tuple
from bonobo.util.objects import Wrapper
__all__ = [
'GraphExecutionContext',
'NodeExecutionContext',
'PluginExecutionContext',
]
class GraphExecutionContext:
@property
@ -26,10 +34,11 @@ class GraphExecutionContext:
def alive(self):
return any(node.alive for node in self.nodes)
def __init__(self, graph, plugins=None):
def __init__(self, graph, plugins=None, services=None):
self.graph = graph
self.nodes = [NodeExecutionContext(node, parent=self) for node in self.graph.nodes]
self.plugins = [PluginExecutionContext(plugin, parent=self) for plugin in plugins or ()]
self.services = Container(services) if services else Container()
for i, component_context in enumerate(self):
try:
@ -74,12 +83,6 @@ class GraphExecutionContext:
node.stop()
def ensure_tuple(tuple_or_mixed):
if isinstance(tuple_or_mixed, tuple):
return tuple_or_mixed
return (tuple_or_mixed, )
class LoopingExecutionContext(Wrapper):
alive = True
PERIOD = 0.25
@ -105,13 +108,16 @@ class LoopingExecutionContext(Wrapper):
assert self.state == (False,
False), ('{}.start() can only be called on a new node.').format(type(self).__name__)
assert self._context is None
self._started = True
self._context = ()
for processor in get_context_processors(self.wrapped):
_processed = processor(self.wrapped, self, *self._context)
try:
self._context = self.parent.services.args_for(self.wrapped) if self.parent else ()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
raise
for processor in resolve_processors(self.wrapped):
try:
# todo yield from ?
_processed = processor(self.wrapped, self, *self._context)
_append_to_context = next(_processed)
if _append_to_context is not None:
self._context += ensure_tuple(_append_to_context)
@ -127,9 +133,7 @@ class LoopingExecutionContext(Wrapper):
sleep(self.PERIOD)
def step(self):
"""
TODO xxx this is a step, not a loop
"""
"""Left as an exercise for the children."""
raise NotImplementedError('Abstract.')
def stop(self):
@ -177,37 +181,6 @@ class LoopingExecutionContext(Wrapper):
print(trace)
class PluginExecutionContext(LoopingExecutionContext):
PERIOD = 0.5
def __init__(self, wrapped, parent):
# Instanciate plugin. This is not yet considered stable, as at some point we may need a way to configure
# plugins, for example if it depends on an external service.
super().__init__(wrapped(self), parent)
def start(self):
super().start()
try:
self.wrapped.initialize()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
def shutdown(self):
try:
self.wrapped.finalize()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
finally:
self.alive = False
def step(self):
try:
self.wrapped.run()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
"""
todo: make the counter dependant of parent context?
@ -313,6 +286,37 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
self.send(_resolve(input_bag, output))
class PluginExecutionContext(LoopingExecutionContext):
PERIOD = 0.5
def __init__(self, wrapped, parent):
# Instanciate plugin. This is not yet considered stable, as at some point we may need a way to configure
# plugins, for example if it depends on an external service.
super().__init__(wrapped(self), parent)
def start(self):
super().start()
try:
self.wrapped.initialize()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
def shutdown(self):
try:
self.wrapped.finalize()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
finally:
self.alive = False
def step(self):
try:
self.wrapped.run()
except Exception as exc: # pylint: disable=broad-except
self.handle_error(exc, traceback.format_exc())
def _iter(mixed):
if isinstance(mixed, (dict, list, str)):
raise TypeError(type(mixed).__name__)

View File

@ -2,8 +2,9 @@ from urllib.parse import urlencode
import requests # todo: make this a service so we can substitute it ?
from bonobo.config import Configurable, Option
from bonobo.context import ContextProcessor, contextual
from bonobo.config import Option
from bonobo.config.processors import ContextProcessor, contextual
from bonobo.config.configurables import Configurable
from bonobo.util.compat import deprecated
from bonobo.util.objects import ValueHolder

View File

@ -1,7 +1,7 @@
import csv
from bonobo.config import Option
from bonobo.context import ContextProcessor, contextual
from bonobo.config.processors import ContextProcessor, contextual
from bonobo.util.objects import ValueHolder
from .file import FileReader, FileWriter, FileHandler

View File

@ -1,8 +1,8 @@
from io import BytesIO
from bonobo.config import Configurable, Option
from bonobo.context import ContextProcessor
from bonobo.context.processors import contextual
from bonobo.config import Option
from bonobo.config.processors import ContextProcessor, contextual
from bonobo.config.configurables import Configurable
from bonobo.util.objects import ValueHolder
__all__ = [

View File

@ -1,6 +1,6 @@
import json
from bonobo.context import ContextProcessor, contextual
from bonobo.config.processors import ContextProcessor, contextual
from .file import FileWriter, FileReader
__all__ = [

View File

@ -0,0 +1,8 @@
from bonobo.strategies.executor import ThreadPoolExecutorStrategy, ProcessPoolExecutorStrategy
from bonobo.strategies.naive import NaiveStrategy
__all__ = [
'NaiveStrategy',
'ProcessPoolExecutorStrategy',
'ThreadPoolExecutorStrategy',
]

View File

@ -1,4 +1,4 @@
from bonobo.context.execution import GraphExecutionContext
from bonobo.execution import GraphExecutionContext
class Strategy:

View File

@ -3,7 +3,7 @@ import time
from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
from bonobo.constants import BEGIN, END
from bonobo.core.strategies.base import Strategy
from bonobo.strategies.base import Strategy
from bonobo.structs.bags import Bag
@ -18,8 +18,8 @@ class ExecutorStrategy(Strategy):
def create_executor(self):
return self.executor_factory()
def execute(self, graph, *args, plugins=None, **kwargs):
context = self.create_graph_execution_context(graph, plugins=plugins)
def execute(self, graph, *args, plugins=None, services=None, **kwargs):
context = self.create_graph_execution_context(graph, plugins=plugins, services=services)
context.recv(BEGIN, Bag(), END)
executor = self.create_executor()

View File

@ -1,5 +1,5 @@
from bonobo.constants import BEGIN, END
from bonobo.core.strategies.base import Strategy
from bonobo.strategies.base import Strategy
from bonobo.structs.bags import Bag

View File

@ -1,103 +1 @@
""" Various simple utilities. """
import functools
from pprint import pprint as _pprint
from colorama import Fore, Style
from bonobo.constants import NOT_MODIFIED
from bonobo.context.processors import contextual
from bonobo.structs.bags import Bag
from bonobo.util.objects import ValueHolder
from bonobo.util.term import CLEAR_EOL
__all__ = [
'Limit',
'NOT_MODIFIED',
'PrettyPrint',
'Tee',
'count',
'noop',
'pprint',
]
def identity(x):
return x
def Limit(n=10):
i = 0
def _limit(*args, **kwargs):
nonlocal i, n
i += 1
if i <= n:
yield NOT_MODIFIED
_limit.__name__ = 'Limit({})'.format(n)
return _limit
def Tee(f):
@functools.wraps(f)
def wrapped(*args, **kwargs):
nonlocal f
f(*args, **kwargs)
return NOT_MODIFIED
return wrapped
@contextual
def count(counter, *args, **kwargs):
counter += 1
@count.add_context_processor
def _count_counter(self, context):
counter = ValueHolder(0)
yield counter
context.send(Bag(counter.value))
pprint = Tee(_pprint)
def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):
def _pprint(*args, **kwargs):
nonlocal title_keys, sort, print_values
row = args[0]
for key in title_keys:
if key in row:
print(Style.BRIGHT, row.get(key), Style.RESET_ALL, sep='')
break
if print_values:
for k in sorted(row) if sort else row:
print(
'',
Fore.BLUE,
k,
Style.RESET_ALL,
' : ',
Fore.BLACK,
'(',
type(row[k]).__name__,
')',
Style.RESET_ALL,
' ',
repr(row[k]),
CLEAR_EOL,
)
yield NOT_MODIFIED
_pprint.__name__ = 'pprint'
return _pprint
def noop(*args, **kwargs): # pylint: disable=unused-argument
return NOT_MODIFIED

View File

@ -26,6 +26,20 @@ def is_platform_32bit():
return struct.calcsize("P") * 8 < 64
def deprecated_alias(alias, func):
@functools.wraps(func)
def new_func(*args, **kwargs):
warnings.simplefilter('always', DeprecationWarning) # turn off filter
warnings.warn(
"Call to deprecated function alias {}, use {} instead.".format(alias, func.__name__),
category=DeprecationWarning, stacklevel=2
)
warnings.simplefilter('default', DeprecationWarning) # reset filter
return func(*args, **kwargs)
return new_func
def deprecated(func):
"""This is a decorator which can be used to mark functions
as deprecated. It will result in a warning being emmitted

View File

@ -15,3 +15,9 @@ def force_iterator(mixed):
return iter(mixed)
except TypeError:
return [mixed] if mixed else []
def ensure_tuple(tuple_or_mixed):
if isinstance(tuple_or_mixed, tuple):
return tuple_or_mixed
return (tuple_or_mixed,)

View File

@ -1,4 +1,4 @@
from bonobo.util import noop
from bonobo.basics import noop
def _create_lifecycle_functions(noun, verb):

View File

@ -1,6 +1,6 @@
from unittest.mock import MagicMock
from bonobo.context.execution import NodeExecutionContext
from bonobo.execution import NodeExecutionContext
class CapturingNodeExecutionContext(NodeExecutionContext):