Merge remote-tracking branch 'upstream/0.2' into 0.2
This commit is contained in:
7
Makefile
7
Makefile
@ -1,7 +1,7 @@
|
||||
# This file has been auto-generated.
|
||||
# All changes will be lost, see Projectfile.
|
||||
#
|
||||
# Updated at 2017-01-19 12:12:07.294619
|
||||
# Updated at 2017-04-21 10:27:25.709949
|
||||
|
||||
PYTHON ?= $(shell which python)
|
||||
PYTHON_BASENAME ?= $(shell basename $(PYTHON))
|
||||
@ -10,6 +10,7 @@ PYTHON_REQUIREMENTS_DEV_FILE ?= requirements-dev.txt
|
||||
QUICK ?=
|
||||
VIRTUAL_ENV ?= .virtualenv-$(PYTHON_BASENAME)
|
||||
PIP ?= $(VIRTUAL_ENV)/bin/pip
|
||||
PIP_INSTALL_OPTIONS ?=
|
||||
PYTEST ?= $(VIRTUAL_ENV)/bin/pytest
|
||||
PYTEST_OPTIONS ?= --capture=no --cov=bonobo --cov-report html
|
||||
SPHINX_OPTS ?=
|
||||
@ -24,13 +25,13 @@ YAPF_OPTIONS ?= -rip
|
||||
# Installs the local project dependencies.
|
||||
install: $(VIRTUAL_ENV)
|
||||
if [ -z "$(QUICK)" ]; then \
|
||||
$(PIP) install -U pip wheel -r $(PYTHON_REQUIREMENTS_FILE) ; \
|
||||
$(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \
|
||||
fi
|
||||
|
||||
# Installs the local project dependencies, including development-only libraries.
|
||||
install-dev: $(VIRTUAL_ENV)
|
||||
if [ -z "$(QUICK)" ]; then \
|
||||
$(PIP) install -U pip wheel -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \
|
||||
$(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \
|
||||
fi
|
||||
|
||||
# Cleans up the local mess.
|
||||
|
||||
@ -57,11 +57,13 @@ data_files = [
|
||||
|
||||
entry_points = {
|
||||
'console_scripts': [
|
||||
'bonobo = bonobo.commands:entrypoint'
|
||||
'bonobo = bonobo.commands:entrypoint',
|
||||
'bb = bonobo.commands:entrypoint',
|
||||
],
|
||||
'bonobo.commands': [
|
||||
'init = bonobo.commands.init:register',
|
||||
'run = bonobo.commands.run:register',
|
||||
'version = bonobo.commands.version:register',
|
||||
],
|
||||
'edgy.project.features': [
|
||||
'bonobo = bonobo.ext.edgy.project.feature:BonoboFeature'
|
||||
|
||||
12
README.rst
12
README.rst
@ -3,13 +3,13 @@
|
||||
|
||||
Data-processing. By monkeys. For humans.
|
||||
|
||||
Bonobo is a data-processing library for python 3.5+ that emphasis writing
|
||||
Bonobo is a data-processing library for python 3.5+ that emphasises writing
|
||||
simple, atomic, plain old python functions and chaining them using a basic
|
||||
acyclic graph. The nodes will need a bit of plumbery to be runnable in
|
||||
different means (iteratively, in threads, in processes, on different machines
|
||||
...) but that should be as transparent as possible.
|
||||
|
||||
The only thing asked to the developer is to either write "pure" functions to
|
||||
The only thing asked of the developer is to write "pure" functions to
|
||||
process data (create a new dict, don't change in place, etc.), and everything
|
||||
should be fine from this point.
|
||||
|
||||
@ -65,13 +65,13 @@ Version 0.2
|
||||
* Changelog
|
||||
* Migration guide
|
||||
* Update documentation
|
||||
* Threaded does not terminate anymore
|
||||
* Threaded does not terminate anymore (fixed ?)
|
||||
* More tests
|
||||
|
||||
Bugs:
|
||||
|
||||
- KeyboardInterrupt does not work anymore.
|
||||
- ThreadPool does not stop anymore.
|
||||
- KeyboardInterrupt does not work anymore. (fixed ?)
|
||||
- ThreadPool does not stop anymore. (fiexd ?)
|
||||
|
||||
Configuration
|
||||
.............
|
||||
@ -102,7 +102,7 @@ Random thoughts and things to do
|
||||
* NaiveStrategy
|
||||
* PoolExecutionStrategy
|
||||
* ThreadPoolExecutionStrategy
|
||||
* ProcesPoolExecutionStrategy
|
||||
* ProcessPoolExecutionStrategy
|
||||
* ThreadExecutionStrategy
|
||||
* ProcessExecutionStrategy
|
||||
|
||||
|
||||
@ -1 +1 @@
|
||||
__version__ = '0.1.6'
|
||||
__version__ = '0.2.0'
|
||||
|
||||
9
bonobo/commands/version.py
Normal file
9
bonobo/commands/version.py
Normal file
@ -0,0 +1,9 @@
|
||||
import bonobo
|
||||
|
||||
|
||||
def execute():
|
||||
print('{} v.{}'.format(bonobo.__name__, bonobo.__version__))
|
||||
|
||||
|
||||
def register(parser):
|
||||
return execute
|
||||
@ -1,7 +1,7 @@
|
||||
import time
|
||||
|
||||
|
||||
class Timer(object):
|
||||
class Timer:
|
||||
"""
|
||||
Context manager used to time execution of stuff.
|
||||
"""
|
||||
|
||||
5
docs/_templates/index.html
vendored
5
docs/_templates/index.html
vendored
@ -3,7 +3,8 @@
|
||||
{% block body %}
|
||||
|
||||
<div style="border: 2px solid red; font-weight: bold; margin: 1em; padding: 1em">
|
||||
Rewrite in progress, things may be broken for now. Please give us some time to finish painting the walls.
|
||||
Bonobo is currently <strong>ALPHA</strong> software. That means that the doc is not finished, and that
|
||||
some APIs will change.
|
||||
</div>
|
||||
|
||||
<h1 style="text-align: center">
|
||||
@ -111,4 +112,4 @@
|
||||
{% endtrans %}
|
||||
</p>
|
||||
|
||||
{% endblock %}
|
||||
{% endblock %}
|
||||
|
||||
1
docs/_templates/sidebarinfos.html
vendored
1
docs/_templates/sidebarinfos.html
vendored
@ -7,4 +7,5 @@
|
||||
<p>
|
||||
<iframe src="http://ghbtns.com/github-btn.html?user=python-bonobo&repo=bonobo&type=watch&count=true&size=small"
|
||||
allowtransparency="true" frameborder="0" scrolling="0" width="200px" height="35px"></iframe>
|
||||
<script async defer src="https://bonobo-slack.herokuapp.com/slackin.js"></script>
|
||||
</p>
|
||||
|
||||
@ -82,7 +82,7 @@ For example, doing the following may cause unexpected problems:
|
||||
'foo': compute_something()
|
||||
})
|
||||
# Still bad! Don't mutate the dict!
|
||||
d['bar']: compute_anotherthing()
|
||||
d['bar'] = compute_anotherthing()
|
||||
return d
|
||||
|
||||
The problem is easy to understand: as **Bonobo** won't make copies of your dict, the same dict will be passed along the
|
||||
|
||||
@ -11,7 +11,7 @@ happened because of **rdc.etl**.
|
||||
|
||||
It would have been counterproductive to migrate the same codebase:
|
||||
|
||||
* a lot of mistakes were impossible to fix in a backward compatible way (for example, transormations were stateful,
|
||||
* a lot of mistakes were impossible to fix in a backward compatible way (for example, transformations were stateful,
|
||||
making them more complicated to write and impossible to reuse, a lot of effort was used to make the components have
|
||||
multi-inputs and multi-outputs, although in 99% of the case it's useless, etc.).
|
||||
* we also wanted to develop something that took advantage of modern python versions, hence the choice of 3.5+.
|
||||
|
||||
@ -15,7 +15,7 @@ Let's write a first data transformation
|
||||
We'll start with the simplest transformation possible.
|
||||
|
||||
In **Bonobo**, a transformation is a plain old python callable, not more, not less. Let's write one that takes a string
|
||||
and uppercase it.
|
||||
and uppercases it.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@ -68,7 +68,7 @@ Let's chain the three transformations together and run the transformation graph:
|
||||
}
|
||||
|
||||
We use the :func:`bonobo.run` helper that hides the underlying object composition necessary to actually run the
|
||||
transformations in parralel, because it's simpler.
|
||||
transformations in parallel, because it's simpler.
|
||||
|
||||
Depending on what you're doing, you may use the shorthand helper method, or the verbose one. Always favor the shorter,
|
||||
if you don't need to tune the graph or the execution strategy (see below).
|
||||
@ -113,12 +113,12 @@ Concepts and definitions
|
||||
by yielding values (a.k.a returning a generator).
|
||||
* Transformation graph (or Graph): a set of transformations tied together in a :class:`bonobo.Graph` instance, which is a simple
|
||||
directed acyclic graph (also refered as a DAG, sometimes).
|
||||
* Node: a transformation within the context of a transformation graph. The node defines what to do whith a
|
||||
transformation's output, and especially what other node to feed with the output.
|
||||
* Execution strategy (or strategy): a way to run a transformation graph. It's responsibility is mainly to parralelize
|
||||
* Node: a transformation within the context of a transformation graph. The node defines what to do with a
|
||||
transformation's output, and especially what other nodes to feed with the output.
|
||||
* Execution strategy (or strategy): a way to run a transformation graph. It's responsibility is mainly to parallelize
|
||||
(or not) the transformations, on one or more process and/or computer, and to setup the right queuing mechanism for
|
||||
transformations' inputs and outputs.
|
||||
* Execution context (or context): a wrapper around a node that holds the state for it. If the node need the state, there
|
||||
* Execution context (or context): a wrapper around a node that holds the state for it. If the node needs state, there
|
||||
are tools available in bonobo to feed it to the transformation using additional call parameters, and so every
|
||||
transformation will be atomic.
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@ Working with files
|
||||
==================
|
||||
|
||||
Bonobo would not be of any use if the aim was to uppercase small lists of strings. In fact, Bonobo should not be used
|
||||
if you don't expect any gain from parralelization/distribution of tasks.
|
||||
if you don't expect any gain from parallelization/distribution of tasks.
|
||||
|
||||
Let's take the following graph as an example:
|
||||
|
||||
@ -19,7 +19,7 @@ the :class:`bonobo.ThreadPoolExecutorStrategy`), which allows to start running `
|
||||
of data, and `C` as soon as `B` yielded the first line of data, even if `A` or `B` still have data to yield.
|
||||
|
||||
The great thing is that you generally don't have to think about it. Just be aware that your components will be run in
|
||||
parralel (with the default strategy), and don't worry too much about blocking components, as they won't block their
|
||||
parallel (with the default strategy), and don't worry too much about blocking components, as they won't block their
|
||||
siblings when run in bonobo.
|
||||
|
||||
That being said, let's try to write a more real-world like transformation.
|
||||
|
||||
24
setup.py
24
setup.py
@ -10,9 +10,12 @@ tolines = lambda c: list(filter(None, map(lambda s: s.strip(), c.split('\n'))))
|
||||
|
||||
|
||||
def read(filename, flt=None):
|
||||
with open(filename) as f:
|
||||
content = f.read().strip()
|
||||
return flt(content) if callable(flt) else content
|
||||
try:
|
||||
with open(filename) as f:
|
||||
content = f.read().strip()
|
||||
return flt(content) if callable(flt) else content
|
||||
except EnvironmentError:
|
||||
return ''
|
||||
|
||||
|
||||
# Py3 compatibility hacks, borrowed from IPython.
|
||||
@ -26,8 +29,12 @@ except NameError:
|
||||
|
||||
|
||||
version_ns = {}
|
||||
execfile(os.path.join(root_dir, 'bonobo/_version.py'), version_ns)
|
||||
version = version_ns.get('__version__', 'dev')
|
||||
try:
|
||||
execfile(os.path.join(root_dir, 'bonobo/_version.py'), version_ns)
|
||||
except EnvironmentError:
|
||||
version = 'dev'
|
||||
else:
|
||||
version = version_ns.get('__version__', 'dev')
|
||||
|
||||
setup(
|
||||
name='bonobo',
|
||||
@ -58,8 +65,11 @@ setup(
|
||||
'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5']
|
||||
},
|
||||
entry_points={
|
||||
'bonobo.commands': ['init = bonobo.commands.init:register', 'run = bonobo.commands.run:register'],
|
||||
'console_scripts': ['bonobo = bonobo.commands:entrypoint'],
|
||||
'bonobo.commands': [
|
||||
'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register',
|
||||
'version = bonobo.commands.version:register'
|
||||
],
|
||||
'console_scripts': ['bonobo = bonobo.commands:entrypoint', 'bb = bonobo.commands:entrypoint'],
|
||||
'edgy.project.features': ['bonobo = '
|
||||
'bonobo.ext.edgy.project.feature:BonoboFeature']
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user