work in progress: working on nodes lifecycle.

Makefile

@@ -1,4 +1,4 @@
-# Generated by Medikit 0.6.3 on 2018-06-11.
+# Generated by Medikit 0.6.3 on 2018-07-22.
 # All changes will be overriden.
 # Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
 

bonobo/execution/contexts/base.py

@@ -53,8 +53,7 @@ class Lifecycle:
 
     @property
     def should_loop(self):
-        # TODO XXX started/stopped?
+        return self.alive and not any((self.defunct, self.killed))
-        return not any((self.defunct, self.killed))
 
     @property
     def status(self):

bonobo/execution/contexts/graph.py

@@ -1,14 +1,19 @@
+import logging
 from functools import partial
+from queue import Empty
 from time import sleep
 
 from bonobo.config import create_container
 from bonobo.constants import BEGIN, END
+from bonobo.errors import InactiveReadableError
 from bonobo.execution import events
 from bonobo.execution.contexts.base import BaseContext
 from bonobo.execution.contexts.node import NodeExecutionContext
 from bonobo.execution.contexts.plugin import PluginExecutionContext
 from whistle import EventDispatcher
 
+logger = logging.getLogger(__name__)
+
 
 class GraphExecutionContext(BaseContext):
     """
@@ -104,11 +109,16 @@ class GraphExecutionContext(BaseContext):
             sleep(self.TICK_PERIOD)
 
     def loop(self):
-        while self.should_loop:
+        nodes = set(node for node in self.nodes if node.should_loop)
-            self.tick()
+        while self.should_loop and len(nodes):
-            for node in self.nodes:
+            self.tick(pause=False)
-                if node.should_loop:
+            for node in list(nodes):
+                try:
                     node.step()
+                except Empty:
+                    continue
+                except InactiveReadableError:
+                    nodes.discard(node)
 
     def stop(self, stopper=None):
         super(GraphExecutionContext, self).stop()

bonobo/execution/contexts/node.py

@@ -120,20 +120,22 @@ class NodeExecutionContext(BaseContext, WithStatistics):
                 break
             except Empty:
                 sleep(TICK_PERIOD)  # XXX: How do we determine this constant?
-                continue
-            except (
-                    NotImplementedError,
-                    UnrecoverableError,
-            ):
-                self.fatal(sys.exc_info())  # exit loop
-            except Exception:  # pylint: disable=broad-except
-                self.error(sys.exc_info())  # does not exit loop
-            except BaseException:
-                self.fatal(sys.exc_info())  # exit loop
 
         logger.debug('Node loop ends for {!r}.'.format(self))
 
     def step(self):
+        try:
+            self._step()
+        except InactiveReadableError:
+            raise
+        except (NotImplementedError, UnrecoverableError, ):
+            self.fatal(sys.exc_info())  # exit loop
+        except Exception:  # pylint: disable=broad-except
+            self.error(sys.exc_info())  # does not exit loop
+        except BaseException:
+            self.fatal(sys.exc_info())  # exit loop
+
+    def _step(self):
         """
         A single step in the loop.
 
@@ -280,7 +282,7 @@ class NodeExecutionContext(BaseContext, WithStatistics):
         If Queue raises (like Timeout or Empty), stat won't be changed.
 
         """
-        input_bag = self.input.get()
+        input_bag = self.input.get(timeout=0)
 
         # Store or check input type
         if self._input_type is None:

requirements-dev.txt

@@ -1,6 +1,6 @@
 -e .[dev]
 -r requirements.txt
-alabaster==0.7.10
+alabaster==0.7.11
 arrow==0.12.1
 atomicwrites==1.1.5
 attrs==18.1.0
@@ -13,7 +13,7 @@ cookiecutter==1.5.1
 coverage==4.5.1
 docutils==0.14
 future==0.16.0
-idna==2.6
+idna==2.7
 imagesize==1.0.0
 jinja2-time==0.2.0
 jinja2==2.10
@@ -22,20 +22,20 @@ more-itertools==4.2.0
 packaging==17.1
 pluggy==0.6.0
 poyo==0.4.1
-py==1.5.3
+py==1.5.4
 pygments==2.2.0
 pyparsing==2.2.0
 pytest-cov==2.5.1
-pytest-timeout==1.2.1
+pytest-timeout==1.3.0
-pytest==3.6.1
+pytest==3.6.3
 python-dateutil==2.7.3
-pytz==2018.4
+pytz==2018.5
-requests==2.18.4
+requests==2.19.1
 six==1.11.0
 snowballstemmer==1.2.1
 sphinx-sitemap==0.2
-sphinx==1.7.5
+sphinx==1.7.6
 sphinxcontrib-websupport==1.1.0
-urllib3==1.22
+urllib3==1.23
 whichcraft==0.4.1
 yapf==0.22.0

requirements-docker.txt

@@ -7,24 +7,23 @@ chardet==3.0.4
 colorama==0.3.9
 docker-pycreds==0.3.0
 docker==2.7.0
-fs==2.0.23
+fs==2.0.25
-graphviz==0.8.3
+graphviz==0.8.4
-idna==2.6
+idna==2.7
 jinja2==2.10
 markupsafe==1.0
 mondrian==0.7.0
 packaging==17.1
-pbr==4.0.4
+pbr==4.1.1
 psutil==5.4.6
 pyparsing==2.2.0
 python-slugify==1.2.5
-pytz==2018.4
+pytz==2018.5
-requests==2.18.4
+requests==2.19.1
 semantic-version==2.6.0
 six==1.11.0
-stevedore==1.28.0
+stevedore==1.29.0
-typing==3.6.4
 unidecode==1.0.22
-urllib3==1.22
+urllib3==1.23
 websocket-client==0.48.0
 whistle==1.0.1

requirements-jupyter.txt

@@ -10,7 +10,7 @@ ipykernel==4.8.2
 ipython-genutils==0.2.0
 ipython==6.4.0
 ipywidgets==6.0.1
-jedi==0.12.0
+jedi==0.12.1
 jinja2==2.10
 jsonschema==2.6.0
 jupyter-client==5.2.3
@@ -21,23 +21,24 @@ markupsafe==1.0
 mistune==0.8.3
 nbconvert==5.3.1
 nbformat==4.4.0
-notebook==5.5.0
+notebook==5.6.0
 pandocfilters==1.4.2
-parso==0.2.1
+parso==0.3.1
 pexpect==4.6.0
 pickleshare==0.7.4
+prometheus-client==0.3.0
 prompt-toolkit==1.0.15
-ptyprocess==0.5.2
+ptyprocess==0.6.0
 pygments==2.2.0
 python-dateutil==2.7.3
-pyzmq==17.0.0
+pyzmq==17.1.0
 qtconsole==4.3.1
 send2trash==1.5.0
 simplegeneric==0.8.1
 six==1.11.0
 terminado==0.8.1
 testpath==0.3.1
-tornado==5.0.2
+tornado==5.1
 traitlets==4.3.2
 wcwidth==0.1.7
 webencodings==0.5.1

requirements-sqlalchemy.txt

@@ -5,23 +5,22 @@ bonobo-sqlalchemy==0.6.0
 certifi==2018.4.16
 chardet==3.0.4
 colorama==0.3.9
-fs==2.0.23
+fs==2.0.25
-graphviz==0.8.3
+graphviz==0.8.4
-idna==2.6
+idna==2.7
 jinja2==2.10
 markupsafe==1.0
 mondrian==0.7.0
 packaging==17.1
-pbr==4.0.4
+pbr==4.1.1
 psutil==5.4.6
 pyparsing==2.2.0
 python-slugify==1.2.5
-pytz==2018.4
+pytz==2018.5
-requests==2.18.4
+requests==2.19.1
 six==1.11.0
-sqlalchemy==1.2.8
+sqlalchemy==1.2.10
-stevedore==1.28.0
+stevedore==1.29.0
-typing==3.6.4
 unidecode==1.0.22
-urllib3==1.22
+urllib3==1.23
 whistle==1.0.1

requirements.txt

@@ -3,22 +3,21 @@ appdirs==1.4.3
 certifi==2018.4.16
 chardet==3.0.4
 colorama==0.3.9
-fs==2.0.23
+fs==2.0.25
-graphviz==0.8.3
+graphviz==0.8.4
-idna==2.6
+idna==2.7
 jinja2==2.10
 markupsafe==1.0
 mondrian==0.7.0
 packaging==17.1
-pbr==4.0.4
+pbr==4.1.1
 psutil==5.4.6
 pyparsing==2.2.0
 python-slugify==1.2.5
-pytz==2018.4
+pytz==2018.5
-requests==2.18.4
+requests==2.19.1
 six==1.11.0
-stevedore==1.28.0
+stevedore==1.29.0
-typing==3.6.4
 unidecode==1.0.22
-urllib3==1.22
+urllib3==1.23
 whistle==1.0.1

setup.py

@@ -1,4 +1,4 @@
-# Generated by Medikit 0.6.3 on 2018-06-11.
+# Generated by Medikit 0.6.3 on 2018-07-22.
 # All changes will be overriden.
 # Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
 

tests/execution/contexts/test_execution_contexts_graph.py

@@ -1,4 +1,5 @@
 from bonobo import Graph
+from bonobo.constants import EMPTY, BEGIN, END
 from bonobo.execution.contexts import GraphExecutionContext
 
 
@@ -49,9 +50,8 @@ def test_lifecycle_of_graph_with_recoverable_error():
 def test_lifecycle_of_graph_with_unrecoverable_error():
     graph = Graph([1, 2, 3], raise_an_unrecoverrable_error, print)
     with GraphExecutionContext(graph) as context:
-        assert context.started
+        assert context.started and context.alive and not context.stopped
-        assert context.alive
+        context.write(BEGIN, EMPTY, END)
-        assert not context.stopped
         context.loop()
     assert context.started
     assert not context.alive