Merge remote-tracking branch 'upstream/develop' into develop

bonobo/_api.py

@@ -149,6 +149,7 @@ api.register_group(
     LdjsonReader,
     LdjsonWriter,
     Limit,
+    MapFields,
     OrderFields,
     PickleReader,
     PickleWriter,

bonobo/config/processors.py

@@ -1,4 +1,4 @@
-from collections import Iterable
+from collections.abc import Iterable
 from contextlib import contextmanager
 from functools import partial
 from inspect import signature

bonobo/contrib/django/commands.py

@@ -64,6 +64,8 @@ class ETLCommand(BaseCommand):
                 print(term.lightwhite("{}. {}".format(i + 1, graph.name or repr(graph).strip("<>"))))
                 result = bonobo.run(graph, services=services, strategy=strategy)
                 results.append(result)
+                for node in result.nodes:
+                    print(node.get_statistics_as_string(), node.get_flags_as_string())
                 print(term.lightblack(" ... return value: " + str(result)))
 
         return results

bonobo/errors.py

@@ -48,6 +48,10 @@ class UnrecoverableTypeError(UnrecoverableError, TypeError):
     pass
 
 
+class UnrecoverableAttributeError(UnrecoverableError, AttributeError):
+    pass
+
+
 class UnrecoverableValueError(UnrecoverableError, ValueError):
     pass
 

bonobo/nodes/basics.py

@@ -10,6 +10,7 @@ from bonobo.config import Configurable, Method, Option, use_context, use_no_inpu
 from bonobo.config.functools import transformation_factory
 from bonobo.config.processors import ContextProcessor, use_context_processor
 from bonobo.constants import NOT_MODIFIED
+from bonobo.errors import UnrecoverableAttributeError
 from bonobo.util.objects import ValueHolder
 from bonobo.util.term import CLEAR_EOL
 
@@ -18,6 +19,7 @@ __all__ = [
     "Format",
     "Limit",
     "OrderFields",
+    "MapFields",
     "PrettyPrinter",
     "Rename",
     "SetFields",
@@ -314,6 +316,46 @@ def Format(**formats):
     return _Format
 
 
+@transformation_factory
+def MapFields(function, key=True):
+    """
+    Transformation factory that maps `function` on the values of a row.
+    It can be applied either to
+    1. all columns (`key=True`),
+    2. no column (`key=False`), or
+    3. a subset of columns by passing a callable, which takes column name and returns `bool`
+     (same as the parameter `function` in `filter`).
+
+    :param function: callable
+    :param key: bool or callable
+    :return: callable
+    """
+    @use_raw_input
+    def _MapFields(bag):
+        try:
+            factory = type(bag)._make
+        except AttributeError:
+            factory = type(bag)
+
+        if callable(key):
+            try:
+                fields = bag._fields
+            except AttributeError as e:
+                raise UnrecoverableAttributeError(
+                    'This transformation works only on objects with named'
+                    ' fields (namedtuple, BagType, ...).') from e
+
+            return factory(
+                function(value) if key(key_) else value for key_, value in zip(fields, bag)
+            )
+        elif key:
+            return factory(function(value) for value in bag)
+        else:
+            return NOT_MODIFIED
+
+    return _MapFields
+
+
 def _count(self, context):
     counter = yield ValueHolder(0)
     context.send(counter.get())

bonobo/nodes/io/csv.py

@@ -6,6 +6,7 @@ from bonobo.constants import NOT_MODIFIED
 from bonobo.nodes.io.base import FileHandler
 from bonobo.nodes.io.file import FileReader, FileWriter
 from bonobo.util import ensure_tuple
+from bonobo.util.collections import tuple_or_const
 
 
 class CsvHandler(FileHandler):
@@ -36,7 +37,7 @@ class CsvHandler(FileHandler):
 
     # Fields (renamed from headers)
     headers = RenamedOption("fields")
-    fields = Option(ensure_tuple, required=False)
+    fields = Option(tuple_or_const, required=False)
 
     def get_dialect_kwargs(self):
         return {
@@ -108,7 +109,7 @@ class CsvWriter(FileWriter, CsvHandler):
 
     def write(self, file, context, *values, fs):
         context.setdefault("lineno", 0)
-        fields = context.get_input_fields()
+        fields = context.get_input_fields() if self.fields is None else self.fields
 
         if not context.lineno:
             context.writer = self.writer_factory(file)
@@ -126,8 +127,7 @@ class CsvWriter(FileWriter, CsvHandler):
                 )
             context.writer(values)
         else:
-            for arg in values:
-                context.writer(ensure_tuple(arg))
+            context.writer(ensure_tuple(values))
 
         return NOT_MODIFIED
 

bonobo/util/collections.py

@@ -1,5 +1,6 @@
 import bisect
 import functools
+from collections import Sequence
 
 
 class sortedlist(list):
@@ -32,6 +33,16 @@ def _with_length_check(f):
     return _wrapped
 
 
+def tuple_or_const(tuple_or_mixed, *, consts=(None, False), **kwargs):
+    if tuple_or_mixed in consts:
+        return tuple_or_mixed
+    if isinstance(tuple_or_mixed, str):
+        pass
+    elif isinstance(tuple_or_mixed, Sequence):
+        tuple_or_mixed = tuple(tuple_or_mixed)
+    return ensure_tuple(tuple_or_mixed, **kwargs)
+
+
 @_with_length_check
 def ensure_tuple(tuple_or_mixed, *, cls=None):
     """

bonobo/util/environ.py

@@ -6,7 +6,7 @@ import warnings
 from contextlib import contextmanager
 
 __escape_decoder = codecs.getdecoder("unicode_escape")
-__posix_variable = re.compile("\$\{[^\}]*\}")
+__posix_variable = re.compile(r"\$\{[^\}]*\}")
 
 
 def parse_var(var):

bonobo/util/errors.py

@@ -24,7 +24,7 @@ def sweeten_errors():
             length = len(pre_re.sub("\\1\\2\\3", arg))
 
             arg = pre_re.sub(w("\\1") + term.bold("\\2") + w("\\3"), arg)
-            arg = re.sub("^  \$ (.*)", term.lightblack("  $ ") + term.reset("\\1"), arg)
+            arg = re.sub(r"^  \$ (.*)", term.lightblack("  $ ") + term.reset("\\1"), arg)
 
             return (arg, length)