paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implements bags, so we can pass arbitrary args/kwargs to functions.

Browse Source
This commit is contained in:
Romain Dorgueil
2016-12-25 12:40:28 +01:00
parent 9c4ec68b18
commit a3adb044bf
19 changed files with 151 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Projectfile
View File

@ -37,7 +37,7 @@ extras_require = {
'pytest-cov >=2.4,<2.5',
'sphinx',
'sphinx_rtd_theme',
'yapf',
],
}

13
bonobo/__init__.py
View File

@ -1,17 +1,10 @@
import sys
from .core import Graph, NaiveStrategy, ProcessPoolExecutorStrategy, ThreadPoolExecutorStrategy, inject, service
from .core import *
from .io import *
from .util import *
PY35 = (sys.version_info >= (3, 5))
assert PY35, 'Python 3.5+ is required to use Bonobo.'
__all__ = [
Graph,
NaiveStrategy,
ProcessPoolExecutorStrategy,
ThreadPoolExecutorStrategy,
inject,
service,
]
__version__ = '0.0.0'

15
bonobo/core/__init__.py
View File

@ -1,13 +1,16 @@
from .bags import Bag, Inherit
from .graphs import Graph
from .services import inject, service
from .strategies.executor import ThreadPoolExecutorStrategy, ProcessPoolExecutorStrategy
from .strategies.naive import NaiveStrategy
__all__ = [
Graph,
NaiveStrategy,
ProcessPoolExecutorStrategy,
ThreadPoolExecutorStrategy,
inject,
service,
'Bag',
'Graph',
'Inherit',
'NaiveStrategy',
'ProcessPoolExecutorStrategy',
'ThreadPoolExecutorStrategy',
'inject',
'service',
]

19
bonobo/core/bags.py Normal file
View File

@ -0,0 +1,19 @@
class Bag:
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
def apply(self, f, *args, **kwargs):
return f(*args, *self.args, **kwargs, **self.kwargs)
def __repr__(self):
return '<{} *{} **{}>'.format(type(self).__name__, self.args, self.kwargs)
class Inherit(Bag):
def override(self, input):
self.args = input.args + self.args
kwargs = dict(input.kwargs)
kwargs.update(self.kwargs)
self.kwargs = kwargs
return self

55
bonobo/core/contexts.py
View File

@ -3,11 +3,12 @@ from functools import partial
from queue import Empty
from time import sleep
from bonobo.core.bags import Bag
from bonobo.core.errors import InactiveReadableError
from bonobo.core.inputs import Input
from bonobo.core.stats import WithStatistics
from bonobo.util.lifecycle import get_initializer, get_finalizer
from bonobo.util.tokens import BEGIN, END, NEW, RUNNING, TERMINATED
from bonobo.util.tokens import Begin, End, New, Running, Terminated, NotModified
class ExecutionContext:
@ -22,8 +23,8 @@ class ExecutionContext:
component_context.outputs = [self[j].input for j in self.graph.outputs_of(i)]
except KeyError as e:
continue
component_context.input.on_begin = partial(component_context.send, BEGIN, _control=True)
component_context.input.on_end = partial(component_context.send, END, _control=True)
component_context.input.on_begin = partial(component_context.send, Begin, _control=True)
component_context.input.on_end = partial(component_context.send, End, _control=True)
def __getitem__(self, item):
return self.components[item]
@ -94,7 +95,7 @@ class ComponentExecutionContext(WithStatistics):
self.component = component
self.input = Input()
self.outputs = []
self.state = NEW
self.state = New
self.stats = {
'in': 0,
'out': 0,
@ -132,27 +133,33 @@ class ComponentExecutionContext(WithStatistics):
self.input.put(value)
def get(self):
row = self.input.get(timeout=1)
return row
# todo XXX if timeout, in stat is erroneous
self.stats['in'] += 1
return self.input.get(timeout=1)
def _call(self, row):
# timer = Timer()
# with timer:
args = () if row is None else (row, )
def _call(self, bag_or_arg):
# todo add timer
bag = bag_or_arg if hasattr(bag_or_arg, 'apply') else Bag(bag_or_arg)
if getattr(self.component, '_with_context', False):
return self.component(self, *args)
return self.component(*args)
return bag.apply(self.component, self)
return bag.apply(self.component)
def step(self):
# Pull data from the first available input channel.
"""Runs a transformation callable with given args/kwargs and flush the result into the right
output channel."""
row = self.get()
self.stats['in'] += 1
input_row = self.get()
results = self._call(row)
def _resolve(result):
nonlocal input_row
if result is NotModified:
return input_row
if hasattr(result, 'override'):
return result.override(input_row)
return result
results = self._call(input_row)
# self._exec_time += timer.duration
# Put data onto output channels
@ -160,7 +167,7 @@ class ComponentExecutionContext(WithStatistics):
results = iterable(results)
except TypeError:
if results:
self.send(results)
self.send(_resolve(results))
else:
# case with no result, an execution went through anyway, use for stats.
# self._exec_count += 1
@ -171,13 +178,13 @@ class ComponentExecutionContext(WithStatistics):
result = next(results)
except StopIteration as e:
break
self.send(result)
self.send(_resolve(result))
def run(self):
assert self.state is NEW, ('A {} can only be run once, and thus is expected to be in {} state at the '
'beginning of a run().').format(type(self).__name__, NEW)
assert self.state is New, ('A {} can only be run once, and thus is expected to be in {} state at the '
'beginning of a run().').format(type(self).__name__, New)
self.state = RUNNING
self.state = Running
try:
get_initializer(self.component)(self)
except Exception as e:
@ -197,10 +204,10 @@ class ComponentExecutionContext(WithStatistics):
except Exception as e:
self.handle_error(e, traceback.format_exc())
assert self.state is RUNNING, ('A {} must be in {} state when finalization starts.').format(
type(self).__name__, RUNNING)
assert self.state is Running, ('A {} must be in {} state when finalization starts.').format(
type(self).__name__, Running)
self.state = TERMINATED
self.state = Terminated
try:
get_finalizer(self.component)(self)
except Exception as e:

6
bonobo/core/graphs.py
View File

@ -1,4 +1,4 @@
from bonobo.util.tokens import BEGIN
from bonobo.util.tokens import Begin
class Graph:
@ -8,7 +8,7 @@ class Graph:
def __init__(self):
self.components = []
self.graph = {BEGIN: set()}
self.graph = {Begin: set()}
def outputs_of(self, idx, create=False):
if create and not idx in self.graph:
@ -20,7 +20,7 @@ class Graph:
self.components.append(c)
return i
def add_chain(self, *components, input=BEGIN):
def add_chain(self, *components, input=Begin):
for component in components:
next = self.add_component(component)
self.outputs_of(input, create=True).add(next)

10
bonobo/core/inputs.py
View File

@ -19,7 +19,7 @@ from queue import Queue
from bonobo.core.errors import AbstractError, InactiveWritableError, InactiveReadableError
from bonobo.util import noop
from bonobo.util.tokens import BEGIN, END
from bonobo.util.tokens import Begin, End
BUFFER_SIZE = 8192
@ -53,7 +53,7 @@ class Input(Queue, Readable, Writable):
def put(self, data, block=True, timeout=None):
# Begin token is a metadata to raise the input runlevel.
if data == BEGIN:
if data == Begin:
self._runlevel += 1
self._writable_runlevel += 1
@ -66,7 +66,7 @@ class Input(Queue, Readable, Writable):
if self._writable_runlevel < 1:
raise InactiveWritableError('Cannot put() on an inactive {}.'.format(Writable.__name__))
if data == END:
if data == End:
self._writable_runlevel -= 1
return Queue.put(self, data, block, timeout)
@ -77,7 +77,7 @@ class Input(Queue, Readable, Writable):
data = Queue.get(self, block, timeout)
if data == END:
if data == End:
self._runlevel -= 1
# callback
@ -92,7 +92,7 @@ class Input(Queue, Readable, Writable):
def empty(self):
self.mutex.acquire()
while self._qsize() and self.queue[0] == END:
while self._qsize() and self.queue[0] == End:
self._runlevel -= 1
Queue._get(self)
self.mutex.release()

11
bonobo/core/strategies/executor.py
View File

@ -3,8 +3,9 @@ from concurrent.futures import Executor
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures import ThreadPoolExecutor
from bonobo.core.bags import Bag
from bonobo.core.strategies.base import Strategy
from bonobo.util.tokens import BEGIN, END
from bonobo.util.tokens import Begin, End
class ExecutorStrategy(Strategy):
@ -19,10 +20,10 @@ class ExecutorStrategy(Strategy):
context = self.create_context(graph, plugins=plugins)
executor = self.executor_factory()
for i in graph.outputs_of(BEGIN):
context[i].recv(BEGIN)
context[i].recv(None)
context[i].recv(END)
for i in graph.outputs_of(Begin):
context[i].recv(Begin)
context[i].recv(Bag())
context[i].recv(End)
futures = []

1
bonobo/io/__init__.py
View File

@ -0,0 +1 @@
from .json import *

2
bonobo/io/json.py
View File

@ -2,6 +2,8 @@ import json
from bonobo.util.lifecycle import with_context, set_initializer, set_finalizer
__all__ = ['to_json', ]
def to_json(path_or_buf):
# todo different cases + documentation

10
bonobo/util/__init__.py
View File

@ -1,6 +1,16 @@
import functools
import pprint
from .tokens import NotModified
__all__ = [
'NotModified',
'head',
'log',
'noop',
'tee',
]
def head(n=10):
i = 0

13
bonobo/util/tokens.py
View File

@ -8,8 +8,11 @@ class Token:
return '<{}>'.format(self.__name__)
BEGIN = Token('Begin')
END = Token('End')
NEW = Token('New')
RUNNING = Token('Running')
TERMINATED = Token('Terminated')
Begin = Token('Begin')
End = Token('End')
New = Token('New')
Running = Token('Running')
Terminated = Token('Terminated')
NotModified = Token('NotModified')

33
doc/conf.py
View File

@ -13,13 +13,8 @@ from bonobo import __version__
# -- General configuration ------------------------------------------------
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.intersphinx',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.ifconfig',
'sphinx.ext.viewcode'
'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage',
'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'
]
# Add any paths that contain templates here, relative to this directory.
@ -86,7 +81,12 @@ html_theme_options = {
html_sidebars = {
'**': [
'sidebarlogo.html', 'localtoc.html', 'relations.html', 'searchbox.html', 'sidebarinfos.html', 'sourcelink.html',
'sidebarlogo.html',
'localtoc.html',
'relations.html',
'searchbox.html',
'sidebarinfos.html',
'sourcelink.html',
]
}
@ -124,30 +124,21 @@ latex_elements = {
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'Bonobo.tex', 'Bonobo Documentation',
'Romain Dorgueil', 'manual'),
]
latex_documents = [(master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual'), ]
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'bonobo', 'Bonobo Documentation',
[author], 1)
]
man_pages = [(master_doc, 'bonobo', 'Bonobo Documentation', [author], 1)]
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'Bonobo', 'Bonobo Documentation',
author, 'Bonobo', 'One line description of project.',
'Miscellaneous'),
]
texinfo_documents = [(master_doc, 'Bonobo', 'Bonobo Documentation', author, 'Bonobo',
'One line description of project.', 'Miscellaneous'), ]
# -- Options for Epub output ----------------------------------------------

19
examples/opendata_fablabs.py
View File

@ -22,7 +22,8 @@ def _getlink(x):
def normalize(row):
result = {
**row,
**
row,
'links': list(filter(None, map(_getlink, json.loads(row.get('links'))))),
'country': countries.get(alpha_2=row.get('country_code', '').upper()).name,
}
@ -37,11 +38,11 @@ def filter_france(row):
def display(row):
print(t.bold(row.get('name')))
address = list(filter(None, (
' '.join(filter(None, (row.get('postal_code', None), row.get('city', None)))),
row.get('county', None),
row.get('country'),
)))
address = list(
filter(None, (
' '.join(filter(None, (row.get('postal_code', None), row.get('city', None)))),
row.get('county', None),
row.get('country'), )))
print(' - {}: {address}'.format(t.blue('address'), address=', '.join(address)))
print(' - {}: {links}'.format(t.blue('links'), links=', '.join(row['links'])))
@ -51,10 +52,10 @@ def display(row):
if __name__ == '__main__':
console_run(
extract_ods(SEARCH_URL, DATASET, timezone='Europe/Paris'),
extract_ods(
SEARCH_URL, DATASET, timezone='Europe/Paris'),
normalize,
filter_france,
tee(display),
to_json('fablabs.json'),
output=True,
)
output=True, )

41
setup.py
View File

@ -21,29 +21,30 @@ setup(
name='bonobo',
description='Bonobo',
license='Apache License, Version 2.0',
install_requires=[],
install_requires=['psutil >=5.0,<5.1', ],
version=version,
long_description=read('README.rst'),
classifiers=read('classifiers.txt', tolines),
packages=find_packages(exclude=['ez_setup', 'example', 'test']),
include_package_data=True,
extras_require={'dev': ['coverage >=4.2,<4.3',
'mock >=2.0,<2.1',
'nose >=1.3,<1.4',
'pylint >=1.6,<1.7',
'pytest >=3,<4',
'pytest-cov >=2.4,<2.5',
'sphinx',
'sphinx_rtd_theme'],
'jupyter': ['ipywidgets >=6.0.0.beta5']
},
data_files=[
('share/jupyter/nbextensions/bonobo', [
'bonobo/ext/jupyter/static/extension.js',
'bonobo/ext/jupyter/static/index.js',
'bonobo/ext/jupyter/static/index.js.map',
]),
],
extras_require={
'dev': [
'coverage >=4.2,<4.3',
'mock >=2.0,<2.1',
'nose >=1.3,<1.4',
'pylint >=1.6,<1.7',
'pytest >=3,<4',
'pytest-cov >=2.4,<2.5',
'sphinx',
'sphinx_rtd_theme',
'yapf',
],
'jupyter': ['ipywidgets >=6.0.0.beta5']
},
data_files=[('share/jupyter/nbextensions/bonobo', [
'bonobo/ext/jupyter/static/extension.js',
'bonobo/ext/jupyter/static/index.js',
'bonobo/ext/jupyter/static/index.js.map',
]), ],
url='https://github.com/hartym/bonobo',
download_url='https://github.com/hartym/bonobo'.format(version=version),
)
download_url='https://github.com/hartym/bonobo'.format(version=version), )

6
tests/core/test_graph.py
View File

@ -1,7 +1,7 @@
import pytest
from bonobo.core.graphs import Graph
from bonobo.util.tokens import BEGIN
from bonobo.util.tokens import Begin
identity = lambda x: x
@ -10,7 +10,7 @@ def test_graph_outputs_of():
g = Graph()
# default graph only node
assert len(g.outputs_of(BEGIN)) == 0
assert len(g.outputs_of(Begin)) == 0
# unexisting node
with pytest.raises(KeyError):
@ -40,4 +40,4 @@ def test_graph_add_chain():
g.add_chain(identity, identity, identity)
assert len(g.components) == 3
assert len(g.outputs_of(BEGIN)) == 1
assert len(g.outputs_of(Begin)) == 1

10
tests/core/test_io.py
View File

@ -20,7 +20,7 @@ import pytest
from bonobo.core.errors import InactiveWritableError, InactiveReadableError
from bonobo.core.inputs import Input
from bonobo.util.tokens import BEGIN, END
from bonobo.util.tokens import Begin, End
def test_input_runlevels():
@ -32,15 +32,15 @@ def test_input_runlevels():
q.put('hello, unborn queue.')
# Begin
q.put(BEGIN)
q.put(Begin)
assert q.alive and q._runlevel == 1
q.put('foo')
# Second Begin
q.put(BEGIN)
q.put(Begin)
assert q.alive and q._runlevel == 2
q.put('bar')
q.put(END)
q.put(End)
# FIFO
assert q.get() == 'foo'
@ -56,7 +56,7 @@ def test_input_runlevels():
q.put('baz')
# Now kill the queue...
q.put(END)
q.put(End)
with pytest.raises(InactiveWritableError):
q.put('foo')

2
tests/core/test_services.py
View File

@ -4,6 +4,7 @@ from bonobo import inject, service
class MyFoo():
pass
def test_service_is_singleton():
@service
def foo():
@ -21,4 +22,3 @@ def test_service_is_singleton():
assert type(foo()) == type(foo2())
assert foo2() is not foo()

3
tests/core/test_stats.py
View File

@ -5,8 +5,7 @@ class MyThingWithStats(WithStatistics):
def get_stats(self, *args, **kwargs):
return (
('foo', 42),
('bar', 69),
)
('bar', 69), )
def test_with_statistics():