From 06dc6b6e35480737badade5d7584f2a3e91ea538 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 29 Oct 2017 23:22:45 +0100 Subject: [PATCH 1/7] [bugfix] Trying to fix a race condition that happens on empty graphs sometimes, on some computers. --- bonobo/ext/console.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index 1053e04..577b345 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -50,13 +50,15 @@ class ConsoleOutputPlugin(Plugin): """ - def initialize(self): + def __init__(self, context): + super(ConsoleOutputPlugin, self).__init__(context) self.prefix = '' self.counter = 0 self._append_cache = '' self.isatty = sys.stdout.isatty() self.iswindows = (sys.platform == 'win32') + def initialize(self): self._stdout = sys.stdout self.stdout = IOBuffer() self.redirect_stdout = redirect_stdout(self._stdout if self.iswindows else self.stdout) From ba9f7171d4487b25f886e9340130dd8bcb82134e Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 29 Oct 2017 23:32:34 +0100 Subject: [PATCH 2/7] [bugfix] Trying to fix a race condition that happens on empty graphs sometimes, on some computers. --- bonobo/ext/console.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index 577b345..da84592 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -52,6 +52,9 @@ class ConsoleOutputPlugin(Plugin): def __init__(self, context): super(ConsoleOutputPlugin, self).__init__(context) + self._reset() + + def _reset() self.prefix = '' self.counter = 0 self._append_cache = '' @@ -59,6 +62,7 @@ class ConsoleOutputPlugin(Plugin): self.iswindows = (sys.platform == 'win32') def initialize(self): + self._reset() self._stdout = sys.stdout self.stdout = IOBuffer() self.redirect_stdout = redirect_stdout(self._stdout if self.iswindows else self.stdout) From b3e74aff853ea708b522aef2756373c119179e62 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 29 Oct 2017 23:36:44 +0100 Subject: [PATCH 3/7] Fixes syntax error. --- bonobo/ext/console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index da84592..a917647 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -54,7 +54,7 @@ class ConsoleOutputPlugin(Plugin): super(ConsoleOutputPlugin, self).__init__(context) self._reset() - def _reset() + def _reset(): self.prefix = '' self.counter = 0 self._append_cache = '' From 2ad47ae958d68462269f5a4ecb6ca579a3620fd2 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 29 Oct 2017 23:40:58 +0100 Subject: [PATCH 4/7] You should not code when tired... --- bonobo/ext/console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index a917647..e42a2b7 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -54,7 +54,7 @@ class ConsoleOutputPlugin(Plugin): super(ConsoleOutputPlugin, self).__init__(context) self._reset() - def _reset(): + def _reset(self): self.prefix = '' self.counter = 0 self._append_cache = '' From 3df81b4d35d2706b39e7803308dc10b1eee36f00 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 30 Oct 2017 08:34:33 +0100 Subject: [PATCH 5/7] release: 0.5.2 --- Makefile | 4 ++-- bonobo/_version.py | 2 +- requirements-dev.txt | 2 +- requirements-docker.txt | 4 +++- requirements-jupyter.txt | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index fdab6c6..777e014 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -# This file has been auto-generated by Medikit. All changes will be lost. -# Updated on 2017-10-21. +# Generated by Medikit 0.4a5 on 2017-10-30. +# All changes will be overriden. PACKAGE ?= bonobo PYTHON ?= $(shell which python) diff --git a/bonobo/_version.py b/bonobo/_version.py index 93b60a1..45869b6 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = '0.5.1' +__version__ = '0.5.2' diff --git a/requirements-dev.txt b/requirements-dev.txt index 4e005a7..553fefc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -27,7 +27,7 @@ pytz==2017.2 requests==2.18.4 six==1.11.0 snowballstemmer==1.2.1 -sphinx==1.6.4 +sphinx==1.6.5 sphinxcontrib-websupport==1.0.1 termcolor==1.1.0 urllib3==1.22 diff --git a/requirements-docker.txt b/requirements-docker.txt index 9e68208..d6f1160 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,8 +1,9 @@ -e .[docker] appdirs==1.4.3 -bonobo-docker==0.2.12 +bonobo-docker==0.5.0 certifi==2017.7.27.1 chardet==3.0.4 +click==6.7 colorama==0.3.9 docker-pycreds==0.2.1 docker==2.3.0 @@ -12,6 +13,7 @@ packaging==16.8 pbr==3.1.1 psutil==5.4.0 pyparsing==2.2.0 +python-dotenv==0.7.1 pytz==2017.2 requests==2.18.4 six==1.11.0 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 2ad75ab..4e1d024 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -16,7 +16,7 @@ jupyter-console==5.2.0 jupyter-core==4.3.0 jupyter==1.0.0 markupsafe==1.0 -mistune==0.7.4 +mistune==0.8 nbconvert==5.3.1 nbformat==4.4.0 notebook==5.2.0 From b0c8440e3543a6a1ac86ae5ddd3274e613e835b1 Mon Sep 17 00:00:00 2001 From: CW Andrews Date: Sat, 11 Nov 2017 17:33:01 -0500 Subject: [PATCH 6/7] Update graphs.rst Rewording and other English fixes for readability. --- docs/guide/graphs.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/guide/graphs.rst b/docs/guide/graphs.rst index 1c753a1..3d2d4b1 100644 --- a/docs/guide/graphs.rst +++ b/docs/guide/graphs.rst @@ -1,9 +1,8 @@ Graphs ====== -Graphs are the glue that ties transformations together. It's the only data-structure bonobo can execute directly. Graphs -must be acyclic, and can contain as much nodes as your system can handle. Although this number can be rather high in -theory, extreme practical cases usually do not exceed hundreds of nodes (and this is already extreme, really). +Graphs are the glue that ties transformations together. They are the only data-structure bonobo can execute directly. Graphs +must be acyclic, and can contain as many nodes as your system can handle. However, although in theory the number of nodes can be rather high, practical use cases usually do not exceed more than a few hundred nodes and only then in extreme cases. Definitions @@ -50,7 +49,7 @@ Non-linear graphs Divergences / forks ------------------- -To create two or more divergent data streams ("fork"), you should specify `_input` kwarg to `add_chain`. +To create two or more divergent data streams ("forks"), you should specify the `_input` kwarg to `add_chain`. .. code-block:: python @@ -74,12 +73,12 @@ Resulting graph: "b" -> "f" -> "g"; } -.. note:: Both branch will receive the same data, at the same time. +.. note:: Both branches will receive the same data and at the same time. -Convergences / merges +Convergence / merges --------------------- -To merge two data streams ("merge"), you can use the `_output` kwarg to `add_chain`, or use named nodes (see below). +To merge two data streams, you can use the `_output` kwarg to `add_chain`, or use named nodes (see below). .. code-block:: python @@ -88,7 +87,7 @@ To merge two data streams ("merge"), you can use the `_output` kwarg to `add_cha graph = bonobo.Graph() - # Here we mark _input to None, so normalize won't get the "begin" impulsion. + # Here we set _input to None, so normalize won't start on its own but only after it receives input from the other chains. graph.add_chain(normalize, store, _input=None) # Add two different chains @@ -122,7 +121,7 @@ Resulting graph: Named nodes ::::::::::: -Using above code to create convergences can lead to hard to read code, because you have to define the "target" stream +Using above code to create convergences often leads to code which is hard to read, because you have to define the "target" stream before the streams that logically goes to the beginning of the transformation graph. To overcome that, one can use "named" nodes: @@ -194,7 +193,7 @@ You can also run a python module: $ bonobo run -m my.own.etlmod -In each case, bonobo's CLI will look for an instance of :class:`bonobo.Graph` in your file/module, create the plumbery +In each case, bonobo's CLI will look for an instance of :class:`bonobo.Graph` in your file/module, create the plumbing needed to execute it, and run it. If you're in an interactive terminal context, it will use :class:`bonobo.ext.console.ConsoleOutputPlugin` for display. From cf120f6d650724a9d76c3efbdd71fbc5cb282259 Mon Sep 17 00:00:00 2001 From: Kenneth Koski Date: Mon, 13 Nov 2017 16:30:56 -0600 Subject: [PATCH 7/7] Adding services to naive execution --- bonobo/strategies/naive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bonobo/strategies/naive.py b/bonobo/strategies/naive.py index cab9c57..49f80cb 100644 --- a/bonobo/strategies/naive.py +++ b/bonobo/strategies/naive.py @@ -4,8 +4,8 @@ from bonobo.structs.bags import Bag class NaiveStrategy(Strategy): - def execute(self, graph, *args, plugins=None, **kwargs): - context = self.create_graph_execution_context(graph, plugins=plugins) + def execute(self, graph, *args, plugins=None, services=None, **kwargs): + context = self.create_graph_execution_context(graph, plugins=plugins, services=services) context.write(BEGIN, Bag(), END) # TODO: how to run plugins in "naive" mode ?