Merge remote-tracking branch 'upstream/develop' into develop

MANIFEST.in

@@ -1,2 +1,3 @@
 include *.txt
+include bonobo/bonobo.svg
 recursive-include bonobo *.py-tpl

appveyor.yml

@@ -17,11 +17,19 @@ environment:
       PYTHON_ARCH: "64"
 
     - PYTHON: "C:\\Python36"
-      PYTHON_VERSION: "3.6.1"
+      PYTHON_VERSION: "3.6.7"
       PYTHON_ARCH: "32"
 
     - PYTHON: "C:\\Python36-x64"
-      PYTHON_VERSION: "3.6.1"
+      PYTHON_VERSION: "3.6.7"
+      PYTHON_ARCH: "64"
+
+    - PYTHON: "C:\\Python37"
+      PYTHON_VERSION: "3.7.1"
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python37-x64"
+      PYTHON_VERSION: "3.7.1"
       PYTHON_ARCH: "64"
 
 build: false

bonobo/_api.py

@@ -149,6 +149,7 @@ api.register_group(
     LdjsonReader,
     LdjsonWriter,
     Limit,
+    MapFields,
     OrderFields,
     PickleReader,
     PickleWriter,

bonobo/config/processors.py

@@ -1,4 +1,4 @@
-from collections import Iterable
+from collections.abc import Iterable
 from contextlib import contextmanager
 from functools import partial
 from inspect import signature

bonobo/contrib/django/commands.py

@@ -64,6 +64,8 @@ class ETLCommand(BaseCommand):
                 print(term.lightwhite("{}. {}".format(i + 1, graph.name or repr(graph).strip("<>"))))
                 result = bonobo.run(graph, services=services, strategy=strategy)
                 results.append(result)
+                for node in result.nodes:
+                    print(node.get_statistics_as_string(), node.get_flags_as_string())
                 print(term.lightblack(" ... return value: " + str(result)))
 
         return results

bonobo/errors.py

@@ -48,6 +48,10 @@ class UnrecoverableTypeError(UnrecoverableError, TypeError):
     pass
 
 
+class UnrecoverableAttributeError(UnrecoverableError, AttributeError):
+    pass
+
+
 class UnrecoverableValueError(UnrecoverableError, ValueError):
     pass
 

bonobo/nodes/basics.py

@@ -10,6 +10,7 @@ from bonobo.config import Configurable, Method, Option, use_context, use_no_inpu
 from bonobo.config.functools import transformation_factory
 from bonobo.config.processors import ContextProcessor, use_context_processor
 from bonobo.constants import NOT_MODIFIED
+from bonobo.errors import UnrecoverableAttributeError
 from bonobo.util.objects import ValueHolder
 from bonobo.util.term import CLEAR_EOL
 
@@ -18,6 +19,7 @@ __all__ = [
     "Format",
     "Limit",
     "OrderFields",
+    "MapFields",
     "PrettyPrinter",
     "Rename",
     "SetFields",
@@ -314,6 +316,46 @@ def Format(**formats):
     return _Format
 
 
+@transformation_factory
+def MapFields(function, key=True):
+    """
+    Transformation factory that maps `function` on the values of a row.
+    It can be applied either to
+    1. all columns (`key=True`),
+    2. no column (`key=False`), or
+    3. a subset of columns by passing a callable, which takes column name and returns `bool`
+     (same as the parameter `function` in `filter`).
+
+    :param function: callable
+    :param key: bool or callable
+    :return: callable
+    """
+    @use_raw_input
+    def _MapFields(bag):
+        try:
+            factory = type(bag)._make
+        except AttributeError:
+            factory = type(bag)
+
+        if callable(key):
+            try:
+                fields = bag._fields
+            except AttributeError as e:
+                raise UnrecoverableAttributeError(
+                    'This transformation works only on objects with named'
+                    ' fields (namedtuple, BagType, ...).') from e
+
+            return factory(
+                function(value) if key(key_) else value for key_, value in zip(fields, bag)
+            )
+        elif key:
+            return factory(function(value) for value in bag)
+        else:
+            return NOT_MODIFIED
+
+    return _MapFields
+
+
 def _count(self, context):
     counter = yield ValueHolder(0)
     context.send(counter.get())

bonobo/nodes/io/csv.py

@@ -6,6 +6,7 @@ from bonobo.constants import NOT_MODIFIED
 from bonobo.nodes.io.base import FileHandler
 from bonobo.nodes.io.file import FileReader, FileWriter
 from bonobo.util import ensure_tuple
+from bonobo.util.collections import tuple_or_const
 
 
 class CsvHandler(FileHandler):
@@ -36,7 +37,7 @@ class CsvHandler(FileHandler):
 
     # Fields (renamed from headers)
     headers = RenamedOption("fields")
-    fields = Option(ensure_tuple, required=False)
+    fields = Option(tuple_or_const, required=False)
 
     def get_dialect_kwargs(self):
         return {
@@ -108,7 +109,7 @@ class CsvWriter(FileWriter, CsvHandler):
 
     def write(self, file, context, *values, fs):
         context.setdefault("lineno", 0)
-        fields = context.get_input_fields()
+        fields = context.get_input_fields() if self.fields is None else self.fields
 
         if not context.lineno:
             context.writer = self.writer_factory(file)
@@ -126,8 +127,7 @@ class CsvWriter(FileWriter, CsvHandler):
                 )
             context.writer(values)
         else:
-            for arg in values:
-                context.writer(ensure_tuple(arg))
+            context.writer(ensure_tuple(values))
 
         return NOT_MODIFIED
 

bonobo/util/collections.py

@@ -1,5 +1,6 @@
 import bisect
 import functools
+from collections import Sequence
 
 
 class sortedlist(list):
@@ -32,6 +33,16 @@ def _with_length_check(f):
     return _wrapped
 
 
+def tuple_or_const(tuple_or_mixed, *, consts=(None, False), **kwargs):
+    if tuple_or_mixed in consts:
+        return tuple_or_mixed
+    if isinstance(tuple_or_mixed, str):
+        pass
+    elif isinstance(tuple_or_mixed, Sequence):
+        tuple_or_mixed = tuple(tuple_or_mixed)
+    return ensure_tuple(tuple_or_mixed, **kwargs)
+
+
 @_with_length_check
 def ensure_tuple(tuple_or_mixed, *, cls=None):
     """

bonobo/util/environ.py

@@ -6,7 +6,7 @@ import warnings
 from contextlib import contextmanager
 
 __escape_decoder = codecs.getdecoder("unicode_escape")
-__posix_variable = re.compile("\$\{[^\}]*\}")
+__posix_variable = re.compile(r"\$\{[^\}]*\}")
 
 
 def parse_var(var):

bonobo/util/errors.py

@@ -24,7 +24,7 @@ def sweeten_errors():
             length = len(pre_re.sub("\\1\\2\\3", arg))
 
             arg = pre_re.sub(w("\\1") + term.bold("\\2") + w("\\3"), arg)
-            arg = re.sub("^  \$ (.*)", term.lightblack("  $ ") + term.reset("\\1"), arg)
+            arg = re.sub(r"^  \$ (.*)", term.lightblack("  $ ") + term.reset("\\1"), arg)
 
             return (arg, length)
 

docs/install.rst

@@ -89,7 +89,7 @@ of your python interpreter.
 
 .. note:: Once again, we use `develop` here. New features should go to `develop`, while bugfixes can go to `master`.
 
-If you can't find the "source" directory, try trunning this:
+If you can't find the "source" directory, try running this:
 
 .. code-block:: shell-session
 
@@ -122,7 +122,7 @@ Preview versions
 ----------------
 
 Sometimes, there are pre-versions available (before a major release, for example). By default, pip does not target
-pre-versions to avoid accidental upgrades to a potentially instable software, but you can easily opt-in:
+pre-versions to avoid accidental upgrades to a potentially unstable version, but you can easily opt-in:
 
 .. code-block:: shell-session
 
@@ -135,10 +135,10 @@ Supported platforms
 Linux, OSX and other Unixes
 ---------------------------
 
-Bonobo test suite runs continuously on Linux, and core developpers use both OSX and Linux machines. Also, there are jobs
+Bonobo test suite runs continuously on Linux, and core developers use both OSX and Linux machines. Also, there are jobs
 running on production linux machines everyday, so the support for those platforms should be quite excellent.
 
-If you're using some esotheric UNIX machine, there can be surprises (although we're not aware, yet). We do not support
+If you're using some esoteric UNIX machine, there can be surprises (although we're not aware, yet). We do not support
 officially those platforms, but if you can actually fix the problems on those systems, we'll be glad to integrate
 your patches (as long as it is tested, for both existing linux environments and your strange systems).
 

tests/nodes/io/test_csv.py

@@ -100,7 +100,7 @@ class CsvWriterTest(Csv, WriterTest, TestCase):
     @incontext()
     def test_nofields_multiple_args(self, context):
         # multiple args are iterated onto and flattened in output
-        context.write_sync((L1, L2), (L3, L4))
+        context.write_sync(L1, L2, L3, L4)
         context.stop()
 
         assert self.readlines() == ("a,hey", "b,bee", "c,see", "d,dee")
@@ -111,18 +111,10 @@ class CsvWriterTest(Csv, WriterTest, TestCase):
         with pytest.raises(TypeError):
             context.write_sync((L1, L2), (L3,))
 
-    @incontext()
-    def test_nofields_single_arg(self, context):
-        # single args are just dumped, shapes can vary.
-        context.write_sync((L1,), (LL,), (L3,))
-        context.stop()
-
-        assert self.readlines() == ("a,hey", "i,have,more,values", "c,see")
-
     @incontext()
     def test_nofields_empty_args(self, context):
         # empty calls are ignored
         context.write_sync(EMPTY, EMPTY, EMPTY)
         context.stop()
 
-        assert self.readlines() == ()
+        assert self.readlines() == ('', '', '')

tests/nodes/test_basics.py

@@ -7,6 +7,7 @@ import pytest
 import bonobo
 from bonobo.constants import EMPTY, NOT_MODIFIED
 from bonobo.util import ValueHolder, ensure_tuple
+from bonobo.util.bags import BagType
 from bonobo.util.testing import BufferingNodeExecutionContext, ConfigurableNodeTest, StaticNodeTest
 
 
@@ -113,3 +114,28 @@ def test_methodcaller():
     with BufferingNodeExecutionContext(methodcaller("zfill", 5)) as context:
         context.write_sync("a", "bb", "ccc")
     assert context.get_buffer() == list(map(ensure_tuple, ["0000a", "000bb", "00ccc"]))
+
+
+MyBag = BagType("MyBag", ["a", "b", "c"])
+
+
+@pytest.mark.parametrize("input_, key, expected", [
+    (MyBag(1, 2, 3), True, MyBag(1, 4, 9)),
+    (MyBag(1, 2, 3), False, MyBag(1, 2, 3)),
+    (MyBag(1, 2, 3), lambda x: x == 'c', MyBag(1, 2, 9)),
+    ((1, 2, 3), True, (1, 4, 9)),
+    ((1, 2, 3), False, (1, 2, 3)),
+])
+def test_map_fields(input_, key, expected):
+    with BufferingNodeExecutionContext(bonobo.MapFields(lambda x: x**2, key)) as context:
+        context.write_sync(input_)
+    assert context.status == '-'
+    [got] = context.get_buffer()
+    assert expected == got
+
+
+def test_map_fields_error():
+    with BufferingNodeExecutionContext(bonobo.MapFields(lambda x: x**2, lambda x: x == 'c')) as context:
+        context.write_sync(tuple())
+    assert context.status == '!'
+    assert context.defunct

tests/util/test_collections.py

@@ -1,7 +1,7 @@
 import pytest
 
 from bonobo.util import ensure_tuple, sortedlist
-from bonobo.util.collections import cast, tuplize
+from bonobo.util.collections import cast, tuplize, tuple_or_const
 
 
 def test_sortedlist():
@@ -13,6 +13,13 @@ def test_sortedlist():
     assert l == [1, 2, 2, 3]
 
 
+def test_tuple_or_const():
+    assert tuple_or_const(()) == ()
+    assert tuple_or_const((1, )) == (1, )
+    assert tuple_or_const((1, 2, )) == (1, 2, )
+    assert tuple_or_const([1, 2, ]) == (1, 2, )
+    assert tuple_or_const("aaa") == ('aaa', )
+
 def test_ensure_tuple():
     assert ensure_tuple("a") == ("a",)
     assert ensure_tuple(("a",)) == ("a",)