From b283d96c500129e02777957e32388e56ae57cb2a Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Fri, 3 Mar 2017 07:56:59 +0100 Subject: [PATCH 1/9] Update index.html --- docs/_templates/index.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/_templates/index.html b/docs/_templates/index.html index 60fdff4..9312f4a 100644 --- a/docs/_templates/index.html +++ b/docs/_templates/index.html @@ -3,7 +3,8 @@ {% block body %}
- Rewrite in progress, things may be broken for now. Please give us some time to finish painting the walls. + Bonobo is currently ALPHA software. That means that the doc is not finished, and that + some APIs will change.

@@ -111,4 +112,4 @@ {% endtrans %}

-{% endblock %} \ No newline at end of file +{% endblock %} From 24e009d890f915c459df73b2e5c2db4db0ad7e8b Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Fri, 21 Apr 2017 11:55:20 +0200 Subject: [PATCH 2/9] adds version command, adds bb alias --- Makefile | 7 +++-- Projectfile | 4 ++- README.rst | 6 ++-- bonobo/commands/version.py | 9 ++++++ setup.py | 59 +++++++++++++++++++++++--------------- 5 files changed, 55 insertions(+), 30 deletions(-) create mode 100644 bonobo/commands/version.py diff --git a/Makefile b/Makefile index a978826..244e031 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-01-19 12:12:07.294619 +# Updated at 2017-04-21 10:27:25.709949 PYTHON ?= $(shell which python) PYTHON_BASENAME ?= $(shell basename $(PYTHON)) @@ -10,6 +10,7 @@ PYTHON_REQUIREMENTS_DEV_FILE ?= requirements-dev.txt QUICK ?= VIRTUAL_ENV ?= .virtualenv-$(PYTHON_BASENAME) PIP ?= $(VIRTUAL_ENV)/bin/pip +PIP_INSTALL_OPTIONS ?= PYTEST ?= $(VIRTUAL_ENV)/bin/pytest PYTEST_OPTIONS ?= --capture=no --cov=bonobo --cov-report html SPHINX_OPTS ?= @@ -24,13 +25,13 @@ YAPF_OPTIONS ?= -rip # Installs the local project dependencies. install: $(VIRTUAL_ENV) if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel -r $(PYTHON_REQUIREMENTS_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ fi # Installs the local project dependencies, including development-only libraries. install-dev: $(VIRTUAL_ENV) if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ fi # Cleans up the local mess. diff --git a/Projectfile b/Projectfile index 2ef816b..0873ff4 100644 --- a/Projectfile +++ b/Projectfile @@ -57,11 +57,13 @@ data_files = [ entry_points = { 'console_scripts': [ - 'bonobo = bonobo.commands:entrypoint' + 'bonobo = bonobo.commands:entrypoint', + 'bb = bonobo.commands:entrypoint', ], 'bonobo.commands': [ 'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register', + 'version = bonobo.commands.version:register', ], 'edgy.project.features': [ 'bonobo = bonobo.ext.edgy.project.feature:BonoboFeature' diff --git a/README.rst b/README.rst index 2d7a1f1..8f66ce6 100644 --- a/README.rst +++ b/README.rst @@ -65,13 +65,13 @@ Version 0.2 * Changelog * Migration guide * Update documentation -* Threaded does not terminate anymore +* Threaded does not terminate anymore (fixed ?) * More tests Bugs: -- KeyboardInterrupt does not work anymore. -- ThreadPool does not stop anymore. +- KeyboardInterrupt does not work anymore. (fixed ?) +- ThreadPool does not stop anymore. (fiexd ?) Configuration ............. diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py new file mode 100644 index 0000000..332286a --- /dev/null +++ b/bonobo/commands/version.py @@ -0,0 +1,9 @@ +import bonobo + + +def execute(): + print('{} v.{}'.format(bonobo.__name__, bonobo.__version__)) + + +def register(parser): + return execute diff --git a/setup.py b/setup.py index a9b983c..e08d0ac 100644 --- a/setup.py +++ b/setup.py @@ -10,9 +10,12 @@ tolines = lambda c: list(filter(None, map(lambda s: s.strip(), c.split('\n')))) def read(filename, flt=None): - with open(filename) as f: - content = f.read().strip() - return flt(content) if callable(flt) else content + try: + with open(filename) as f: + content = f.read().strip() + return flt(content) if callable(flt) else content + except EnvironmentError: + return '' # Py3 compatibility hacks, borrowed from IPython. @@ -26,43 +29,53 @@ except NameError: version_ns = {} -execfile(os.path.join(root_dir, 'bonobo/_version.py'), version_ns) -version = version_ns.get('__version__', 'dev') +try: + execfile(os.path.join(root_dir, 'bonobo/_version.py'), version_ns) +except EnvironmentError: + version = 'dev' +else: + version = version_ns.get('__version__', 'dev') setup( name='bonobo', description='Bonobo', license='Apache License, Version 2.0', install_requires=[ - 'blessings >=1.6,<1.7', 'psutil >=5.0,<5.1', 'requests >=2.12,<2.13', 'stevedore >=1.19,<1.20', - 'toolz >=0.8,<0.9' + 'blessings >=1.6,<1.7', 'psutil >=5.0,<5.1', 'requests >=2.12,<2.13', + 'stevedore >=1.19,<1.20', 'toolz >=0.8,<0.9' ], version=version, long_description=read('README.rst'), classifiers=read('classifiers.txt', tolines), packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, - data_files=[ - ( - 'share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map' - ] - ) - ], + data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/ext/jupyter/static/extension.js', + 'bonobo/ext/jupyter/static/index.js', + 'bonobo/ext/jupyter/static/index.js.map' + ])], extras_require={ 'dev': [ - 'coverage >=4.3,<4.4', 'mock >=2.0,<2.1', 'nose >=1.3,<1.4', 'pylint >=1.6,<1.7', 'pytest >=3,<4', - 'pytest-cov >=2.4,<2.5', 'pytest-timeout >=1.2,<1.3', 'sphinx', 'sphinx_rtd_theme', 'yapf' + 'coverage >=4.3,<4.4', 'mock >=2.0,<2.1', 'nose >=1.3,<1.4', + 'pylint >=1.6,<1.7', 'pytest >=3,<4', 'pytest-cov >=2.4,<2.5', + 'pytest-timeout >=1.2,<1.3', 'sphinx', 'sphinx_rtd_theme', 'yapf' ], 'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5'] }, entry_points={ - 'bonobo.commands': ['init = bonobo.commands.init:register', 'run = bonobo.commands.run:register'], - 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], - 'edgy.project.features': ['bonobo = ' - 'bonobo.ext.edgy.project.feature:BonoboFeature'] + 'bonobo.commands': [ + 'init = bonobo.commands.init:register', + 'run = bonobo.commands.run:register', + 'version = bonobo.commands.version:register' + ], + 'console_scripts': [ + 'bonobo = bonobo.commands:entrypoint', + 'bb = bonobo.commands:entrypoint' + ], + 'edgy.project.features': + ['bonobo = ' + 'bonobo.ext.edgy.project.feature:BonoboFeature'] }, url='https://bonobo-project.org/', - download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), -) + download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'. + format(version=version), ) From 511c4b451be315b7a61623f42a5b7affa3ff5d3f Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Fri, 21 Apr 2017 11:56:46 +0200 Subject: [PATCH 3/9] release: 0.1.7 --- bonobo/_version.py | 2 +- setup.py | 39 ++++++++++++++++++--------------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/bonobo/_version.py b/bonobo/_version.py index 2fb2513..124e462 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = '0.1.6' +__version__ = '0.1.7' diff --git a/setup.py b/setup.py index e08d0ac..1d12879 100644 --- a/setup.py +++ b/setup.py @@ -41,41 +41,38 @@ setup( description='Bonobo', license='Apache License, Version 2.0', install_requires=[ - 'blessings >=1.6,<1.7', 'psutil >=5.0,<5.1', 'requests >=2.12,<2.13', - 'stevedore >=1.19,<1.20', 'toolz >=0.8,<0.9' + 'blessings >=1.6,<1.7', 'psutil >=5.0,<5.1', 'requests >=2.12,<2.13', 'stevedore >=1.19,<1.20', + 'toolz >=0.8,<0.9' ], version=version, long_description=read('README.rst'), classifiers=read('classifiers.txt', tolines), packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, - data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/ext/jupyter/static/extension.js', - 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map' - ])], + data_files=[ + ( + 'share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js', + 'bonobo/ext/jupyter/static/index.js.map' + ] + ) + ], extras_require={ 'dev': [ - 'coverage >=4.3,<4.4', 'mock >=2.0,<2.1', 'nose >=1.3,<1.4', - 'pylint >=1.6,<1.7', 'pytest >=3,<4', 'pytest-cov >=2.4,<2.5', - 'pytest-timeout >=1.2,<1.3', 'sphinx', 'sphinx_rtd_theme', 'yapf' + 'coverage >=4.3,<4.4', 'mock >=2.0,<2.1', 'nose >=1.3,<1.4', 'pylint >=1.6,<1.7', 'pytest >=3,<4', + 'pytest-cov >=2.4,<2.5', 'pytest-timeout >=1.2,<1.3', 'sphinx', 'sphinx_rtd_theme', 'yapf' ], 'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5'] }, entry_points={ 'bonobo.commands': [ - 'init = bonobo.commands.init:register', - 'run = bonobo.commands.run:register', + 'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register' ], - 'console_scripts': [ - 'bonobo = bonobo.commands:entrypoint', - 'bb = bonobo.commands:entrypoint' - ], - 'edgy.project.features': - ['bonobo = ' - 'bonobo.ext.edgy.project.feature:BonoboFeature'] + 'console_scripts': ['bonobo = bonobo.commands:entrypoint', 'bb = bonobo.commands:entrypoint'], + 'edgy.project.features': ['bonobo = ' + 'bonobo.ext.edgy.project.feature:BonoboFeature'] }, url='https://bonobo-project.org/', - download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'. - format(version=version), ) + download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), +) From e617774b8047cb05219d146a7ccccc7d6cdc151a Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Fri, 21 Apr 2017 12:11:54 +0200 Subject: [PATCH 4/9] release: 0.2.0 --- bonobo/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bonobo/_version.py b/bonobo/_version.py index 124e462..7fd229a 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = '0.1.7' +__version__ = '0.2.0' From b33076d6a4559e58d32ccce1f1fbcd69159674a9 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 22 Apr 2017 11:29:40 +0200 Subject: [PATCH 5/9] Add slack channel. --- docs/_templates/sidebarinfos.html | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_templates/sidebarinfos.html b/docs/_templates/sidebarinfos.html index 32ead12..deacc18 100644 --- a/docs/_templates/sidebarinfos.html +++ b/docs/_templates/sidebarinfos.html @@ -7,4 +7,5 @@

+

From 1399bbe733ba17bb102d16183ff8ef38a02f6177 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 22 Apr 2017 18:29:48 +0200 Subject: [PATCH 6/9] Update purity.rst --- docs/guide/purity.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/purity.rst b/docs/guide/purity.rst index cf9d47f..05784c5 100644 --- a/docs/guide/purity.rst +++ b/docs/guide/purity.rst @@ -82,7 +82,7 @@ For example, doing the following may cause unexpected problems: 'foo': compute_something() }) # Still bad! Don't mutate the dict! - d['bar']: compute_anotherthing() + d['bar'] = compute_anotherthing() return d The problem is easy to understand: as **Bonobo** won't make copies of your dict, the same dict will be passed along the From ffc6b2b45e750ca2e8c6a844d0ded6c435309cef Mon Sep 17 00:00:00 2001 From: Dave Willmer Date: Sat, 22 Apr 2017 22:16:52 +0200 Subject: [PATCH 7/9] Fix minor typos etc --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 8f66ce6..589559f 100644 --- a/README.rst +++ b/README.rst @@ -3,13 +3,13 @@ Data-processing. By monkeys. For humans. -Bonobo is a data-processing library for python 3.5+ that emphasis writing +Bonobo is a data-processing library for python 3.5+ that emphasises writing simple, atomic, plain old python functions and chaining them using a basic acyclic graph. The nodes will need a bit of plumbery to be runnable in different means (iteratively, in threads, in processes, on different machines ...) but that should be as transparent as possible. -The only thing asked to the developer is to either write "pure" functions to +The only thing asked of the developer is to write "pure" functions to process data (create a new dict, don't change in place, etc.), and everything should be fine from this point. @@ -102,7 +102,7 @@ Random thoughts and things to do * NaiveStrategy * PoolExecutionStrategy * ThreadPoolExecutionStrategy - * ProcesPoolExecutionStrategy + * ProcessPoolExecutionStrategy * ThreadExecutionStrategy * ProcessExecutionStrategy From 061255bf9975ea57073ea18ca7238b8b598afba9 Mon Sep 17 00:00:00 2001 From: Dave Willmer Date: Sat, 22 Apr 2017 22:47:41 +0200 Subject: [PATCH 8/9] Minor typos etc --- bonobo/util/time.py | 2 +- docs/history.rst | 2 +- docs/tutorial/tut01.rst | 4 ++-- docs/tutorial/tut02.rst | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bonobo/util/time.py b/bonobo/util/time.py index 534b662..14de016 100644 --- a/bonobo/util/time.py +++ b/bonobo/util/time.py @@ -1,7 +1,7 @@ import time -class Timer(object): +class Timer: """ Context manager used to time execution of stuff. """ diff --git a/docs/history.rst b/docs/history.rst index 70eea0a..c7a8362 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -11,7 +11,7 @@ happened because of **rdc.etl**. It would have been counterproductive to migrate the same codebase: - * a lot of mistakes were impossible to fix in a backward compatible way (for example, transormations were stateful, + * a lot of mistakes were impossible to fix in a backward compatible way (for example, transformations were stateful, making them more complicated to write and impossible to reuse, a lot of effort was used to make the components have multi-inputs and multi-outputs, although in 99% of the case it's useless, etc.). * we also wanted to develop something that took advantage of modern python versions, hence the choice of 3.5+. diff --git a/docs/tutorial/tut01.rst b/docs/tutorial/tut01.rst index 6504298..b44f811 100644 --- a/docs/tutorial/tut01.rst +++ b/docs/tutorial/tut01.rst @@ -68,7 +68,7 @@ Let's chain the three transformations together and run the transformation graph: } We use the :func:`bonobo.run` helper that hides the underlying object composition necessary to actually run the -transformations in parralel, because it's simpler. +transformations in parallel, because it's simpler. Depending on what you're doing, you may use the shorthand helper method, or the verbose one. Always favor the shorter, if you don't need to tune the graph or the execution strategy (see below). @@ -115,7 +115,7 @@ Concepts and definitions directed acyclic graph (also refered as a DAG, sometimes). * Node: a transformation within the context of a transformation graph. The node defines what to do whith a transformation's output, and especially what other node to feed with the output. -* Execution strategy (or strategy): a way to run a transformation graph. It's responsibility is mainly to parralelize +* Execution strategy (or strategy): a way to run a transformation graph. It's responsibility is mainly to parallelize (or not) the transformations, on one or more process and/or computer, and to setup the right queuing mechanism for transformations' inputs and outputs. * Execution context (or context): a wrapper around a node that holds the state for it. If the node need the state, there diff --git a/docs/tutorial/tut02.rst b/docs/tutorial/tut02.rst index 2ceeb55..f053c89 100644 --- a/docs/tutorial/tut02.rst +++ b/docs/tutorial/tut02.rst @@ -2,7 +2,7 @@ Working with files ================== Bonobo would not be of any use if the aim was to uppercase small lists of strings. In fact, Bonobo should not be used -if you don't expect any gain from parralelization/distribution of tasks. +if you don't expect any gain from parallelization/distribution of tasks. Let's take the following graph as an example: @@ -19,7 +19,7 @@ the :class:`bonobo.ThreadPoolExecutorStrategy`), which allows to start running ` of data, and `C` as soon as `B` yielded the first line of data, even if `A` or `B` still have data to yield. The great thing is that you generally don't have to think about it. Just be aware that your components will be run in -parralel (with the default strategy), and don't worry too much about blocking components, as they won't block their +parallel (with the default strategy), and don't worry too much about blocking components, as they won't block their siblings when run in bonobo. That being said, let's try to write a more real-world like transformation. From 42519ab468eff8eac4bde449207c16c0757339c6 Mon Sep 17 00:00:00 2001 From: Dave Willmer Date: Sat, 22 Apr 2017 22:54:35 +0200 Subject: [PATCH 9/9] Minor typos --- docs/tutorial/tut01.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorial/tut01.rst b/docs/tutorial/tut01.rst index b44f811..d61c544 100644 --- a/docs/tutorial/tut01.rst +++ b/docs/tutorial/tut01.rst @@ -15,7 +15,7 @@ Let's write a first data transformation We'll start with the simplest transformation possible. In **Bonobo**, a transformation is a plain old python callable, not more, not less. Let's write one that takes a string -and uppercase it. +and uppercases it. .. code-block:: python @@ -113,12 +113,12 @@ Concepts and definitions by yielding values (a.k.a returning a generator). * Transformation graph (or Graph): a set of transformations tied together in a :class:`bonobo.Graph` instance, which is a simple directed acyclic graph (also refered as a DAG, sometimes). -* Node: a transformation within the context of a transformation graph. The node defines what to do whith a - transformation's output, and especially what other node to feed with the output. +* Node: a transformation within the context of a transformation graph. The node defines what to do with a + transformation's output, and especially what other nodes to feed with the output. * Execution strategy (or strategy): a way to run a transformation graph. It's responsibility is mainly to parallelize (or not) the transformations, on one or more process and/or computer, and to setup the right queuing mechanism for transformations' inputs and outputs. -* Execution context (or context): a wrapper around a node that holds the state for it. If the node need the state, there +* Execution context (or context): a wrapper around a node that holds the state for it. If the node needs state, there are tools available in bonobo to feed it to the transformation using additional call parameters, and so every transformation will be atomic.