feat, wip: refactoring and simplification of how casts are made.

bonobo/_api.py

@@ -10,7 +10,7 @@ to another is maximal.
 from bonobo.execution.strategies import create_strategy
 from bonobo.nodes import *
 from bonobo.nodes import __all__ as _all_nodes
-from bonobo.structs import Graph
+from bonobo.structs.graphs import Graph
 from bonobo.util.api import ApiHelper
 from bonobo.util.environ import parse_args, get_argument_parser
 from bonobo.registry import create_reader, create_writer

bonobo/constants.py

@@ -20,30 +20,13 @@
 
 
 """
-
-
-class Token:
-    def __init__(self, name):
-        self.__name__ = name
-
-    def __repr__(self):
-        return '<{}>'.format(self.__name__)
-
-
-class Flag(Token):
-    must_be_first = False
-    must_be_last = False
-    allows_data = True
-
+from bonobo.structs.tokens import Token
+from bonobo.util.envelopes import UnchangedEnvelope
 
 BEGIN = Token('Begin')
 END = Token('End')
 
-INHERIT = Flag('Inherit')
-NOT_MODIFIED = Flag('NotModified')
-NOT_MODIFIED.must_be_first = True
-NOT_MODIFIED.must_be_last = True
-NOT_MODIFIED.allows_data = False
+NOT_MODIFIED = UnchangedEnvelope()
 
 EMPTY = tuple()
 

bonobo/execution/contexts/graph.py

@@ -4,7 +4,7 @@ from queue import Empty
 from time import sleep
 
 from bonobo.config import create_container
-from bonobo.constants import BEGIN, END
+from bonobo.constants import BEGIN, END, EMPTY
 from bonobo.errors import InactiveReadableError
 from bonobo.execution import events
 from bonobo.execution.contexts.base import BaseContext
@@ -120,6 +120,10 @@ class GraphExecutionContext(BaseContext):
                 except InactiveReadableError:
                     nodes.discard(node)
 
+    def run_until_complete(self):
+        self.write(BEGIN, EMPTY, END)
+        self.loop()
+
     def stop(self, stopper=None):
         super(GraphExecutionContext, self).stop()
 

bonobo/execution/contexts/node.py

@@ -7,12 +7,14 @@ from types import GeneratorType
 
 from bonobo.config import create_container
 from bonobo.config.processors import ContextCurrifier
-from bonobo.constants import NOT_MODIFIED, BEGIN, END, TICK_PERIOD, Token, Flag, INHERIT
+from bonobo.constants import BEGIN, END, TICK_PERIOD
 from bonobo.errors import InactiveReadableError, UnrecoverableError, UnrecoverableTypeError
 from bonobo.execution.contexts.base import BaseContext
 from bonobo.structs.inputs import Input
-from bonobo.util import get_name, isconfigurabletype, ensure_tuple
+from bonobo.structs.tokens import Token, Flag
+from bonobo.util import get_name, isconfigurabletype, ensure_tuple, deprecated
 from bonobo.util.bags import BagType
+from bonobo.util.envelopes import isenvelope, F_NOT_MODIFIED, F_INHERIT
 from bonobo.util.statistics import WithStatistics
 
 logger = logging.getLogger(__name__)
@@ -329,17 +331,29 @@ class NodeExecutionContext(BaseContext, WithStatistics):
         :return: Bag
         """
 
-        tokens, _output = split_token(_output)
+        if isenvelope(_output):
+            _output, _flags, _options = _output.unfold()
+        else:
+            _flags, _options = [], {}
 
-        if NOT_MODIFIED in tokens:
-            return ensure_tuple(_input, cls=(self.output_type or tuple))
+        if len(_flags):
+            # TODO: parse flags to check constraints are respected (like not modified alone, etc.)
 
-        if INHERIT in tokens:
-            if self._output_type is None:
-                self._output_type = concat_types(self._input_type, self._input_length, self._output_type, len(_output))
-            _output = _input + ensure_tuple(_output)
+            if F_NOT_MODIFIED in _flags:
+                return _input
 
-        return ensure_tuple(_output, cls=(self._output_type or tuple))
+            if F_INHERIT in _flags:
+                if self._output_type is None:
+                    self._output_type = concat_types(
+                        self._input_type, self._input_length, self._output_type, len(_output)
+                    )
+                _output = _input + ensure_tuple(_output)
+
+        if not self._output_type:
+            if issubclass(type(_output), tuple):
+                self._output_type = type(_output)
+
+        return ensure_tuple(_output, cls=self._output_type)
 
     def _send(self, value, _control=False):
         """
@@ -367,6 +381,7 @@ def isflag(param):
     return isinstance(param, Flag)
 
 
+@deprecated
 def split_token(output):
     """
     Split an output into token tuple, real output tuple.
@@ -392,6 +407,7 @@ def split_token(output):
     output = output[i:]
     if not data_allowed and len(output):
         raise ValueError('Output data provided after a flag that does not allow data.')
+
     return flags, output
 
 

bonobo/structs/__init__.py

@@ -1,5 +0,0 @@
-from bonobo.structs.graphs import Graph
-
-__all__ = [
-    'Graph',
-]

bonobo/structs/tokens.py

@@ -0,0 +1,12 @@
+class Token:
+    def __init__(self, name):
+        self.__name__ = name
+
+    def __repr__(self):
+        return '<{}>'.format(self.__name__)
+
+
+class Flag(Token):
+    must_be_first = False
+    must_be_last = False
+    allows_data = True

bonobo/util/collections.py

@@ -3,7 +3,17 @@ import functools
 
 
 class sortedlist(list):
+    """
+    A list with an insort() method that wan be used to maintain sorted lists. The list by itself is not sorted, it's
+    up to the user to not insert unsorted elements.
+    """
+
     def insort(self, x):
+        """
+        If the list is sorted, insert the element in the right place. Otherwise, unpredictable behaviour.
+
+        :param x:
+        """
         bisect.insort(self, x)
 
 

bonobo/util/envelopes.py

@@ -0,0 +1,32 @@
+from bonobo.structs.tokens import Flag
+
+F_INHERIT = Flag('Inherit')
+
+F_NOT_MODIFIED = Flag('NotModified')
+F_NOT_MODIFIED.must_be_first = True
+F_NOT_MODIFIED.must_be_last = True
+F_NOT_MODIFIED.allows_data = False
+
+
+class Envelope:
+    def __init__(self, content, *, flags=None, **options):
+        self._content = content
+        self._flags = set(flags or ())
+        self._options = options
+
+    def unfold(self):
+        return self._content, self._flags, self._options
+
+
+class AppendingEnvelope(Envelope):
+    def __init__(self, content, **options):
+        super().__init__(content, flags={F_INHERIT}, **options)
+
+
+class UnchangedEnvelope(Envelope):
+    def __init__(self, **options):
+        super().__init__(None, flags={F_NOT_MODIFIED}, **options)
+
+
+def isenvelope(mixed):
+    return isinstance(mixed, Envelope)

bonobo/util/testing.py

@@ -11,7 +11,7 @@ import pytest
 
 from bonobo import open_fs, __main__, get_examples_path
 from bonobo.commands import entrypoint
-from bonobo.constants import Token
+from bonobo.structs.tokens import Token
 from bonobo.execution.contexts.graph import GraphExecutionContext
 from bonobo.execution.contexts.node import NodeExecutionContext