diff --git a/MANIFEST.in b/MANIFEST.in
index f0e3359..6277daa 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1,3 @@
include *.txt
+include bonobo/bonobo.svg
recursive-include bonobo *.py-tpl
diff --git a/appveyor.yml b/appveyor.yml
index 9cf3107..a12aa5a 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -17,11 +17,19 @@ environment:
PYTHON_ARCH: "64"
- PYTHON: "C:\\Python36"
- PYTHON_VERSION: "3.6.1"
+ PYTHON_VERSION: "3.6.7"
PYTHON_ARCH: "32"
- PYTHON: "C:\\Python36-x64"
- PYTHON_VERSION: "3.6.1"
+ PYTHON_VERSION: "3.6.7"
+ PYTHON_ARCH: "64"
+
+ - PYTHON: "C:\\Python37"
+ PYTHON_VERSION: "3.7.1"
+ PYTHON_ARCH: "32"
+
+ - PYTHON: "C:\\Python37-x64"
+ PYTHON_VERSION: "3.7.1"
PYTHON_ARCH: "64"
build: false
diff --git a/bonobo/__init__.py b/bonobo/__init__.py
index 35dc5c9..2487780 100644
--- a/bonobo/__init__.py
+++ b/bonobo/__init__.py
@@ -6,6 +6,7 @@
# Licensed under Apache License 2.0, read the LICENSE file in the root of the source tree.
import sys
+from pathlib import Path
if sys.version_info < (3, 5):
raise RuntimeError("Python 3.5+ is required to use Bonobo.")
@@ -28,6 +29,7 @@ from bonobo._api import (
LdjsonReader,
LdjsonWriter,
Limit,
+ MapFields,
OrderFields,
PickleReader,
PickleWriter,
@@ -52,7 +54,8 @@ from bonobo._api import (
from bonobo._version import __version__
__all__ = ["__version__"] + __all__
-__logo__ = ''
+with (Path(__file__).parent / "bonobo.svg").open() as f:
+ __logo__ = f.read()
__doc__ = __doc__
__version__ = __version__
@@ -69,4 +72,4 @@ def _repr_html_():
).format(__logo__, "
".join(get_versions(all=True)))
-del sys
+del sys, Path, f
diff --git a/bonobo/_api.py b/bonobo/_api.py
index e07caf3..78ed72f 100644
--- a/bonobo/_api.py
+++ b/bonobo/_api.py
@@ -149,6 +149,7 @@ api.register_group(
LdjsonReader,
LdjsonWriter,
Limit,
+ MapFields,
OrderFields,
PickleReader,
PickleWriter,
diff --git a/bonobo/bonobo.svg b/bonobo/bonobo.svg
new file mode 100644
index 0000000..c81b1b4
--- /dev/null
+++ b/bonobo/bonobo.svg
@@ -0,0 +1 @@
+
diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py
index 6e9b466..3a6a184 100644
--- a/bonobo/config/processors.py
+++ b/bonobo/config/processors.py
@@ -1,4 +1,4 @@
-from collections import Iterable
+from collections.abc import Iterable
from contextlib import contextmanager
from functools import partial
from inspect import signature
diff --git a/bonobo/contrib/django/commands.py b/bonobo/contrib/django/commands.py
index aaa67a3..f2a8a15 100644
--- a/bonobo/contrib/django/commands.py
+++ b/bonobo/contrib/django/commands.py
@@ -64,6 +64,8 @@ class ETLCommand(BaseCommand):
print(term.lightwhite("{}. {}".format(i + 1, graph.name or repr(graph).strip("<>"))))
result = bonobo.run(graph, services=services, strategy=strategy)
results.append(result)
+ for node in result.nodes:
+ print(node.get_statistics_as_string(), node.get_flags_as_string())
print(term.lightblack(" ... return value: " + str(result)))
return results
diff --git a/bonobo/errors.py b/bonobo/errors.py
index c0fb3fa..09995d7 100644
--- a/bonobo/errors.py
+++ b/bonobo/errors.py
@@ -48,6 +48,10 @@ class UnrecoverableTypeError(UnrecoverableError, TypeError):
pass
+class UnrecoverableAttributeError(UnrecoverableError, AttributeError):
+ pass
+
+
class UnrecoverableValueError(UnrecoverableError, ValueError):
pass
diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py
index e55f393..b0663c2 100644
--- a/bonobo/nodes/basics.py
+++ b/bonobo/nodes/basics.py
@@ -10,6 +10,7 @@ from bonobo.config import Configurable, Method, Option, use_context, use_no_inpu
from bonobo.config.functools import transformation_factory
from bonobo.config.processors import ContextProcessor, use_context_processor
from bonobo.constants import NOT_MODIFIED
+from bonobo.errors import UnrecoverableAttributeError
from bonobo.util.objects import ValueHolder
from bonobo.util.term import CLEAR_EOL
@@ -18,6 +19,7 @@ __all__ = [
"Format",
"Limit",
"OrderFields",
+ "MapFields",
"PrettyPrinter",
"Rename",
"SetFields",
@@ -314,6 +316,46 @@ def Format(**formats):
return _Format
+@transformation_factory
+def MapFields(function, key=True):
+ """
+ Transformation factory that maps `function` on the values of a row.
+ It can be applied either to
+ 1. all columns (`key=True`),
+ 2. no column (`key=False`), or
+ 3. a subset of columns by passing a callable, which takes column name and returns `bool`
+ (same as the parameter `function` in `filter`).
+
+ :param function: callable
+ :param key: bool or callable
+ :return: callable
+ """
+ @use_raw_input
+ def _MapFields(bag):
+ try:
+ factory = type(bag)._make
+ except AttributeError:
+ factory = type(bag)
+
+ if callable(key):
+ try:
+ fields = bag._fields
+ except AttributeError as e:
+ raise UnrecoverableAttributeError(
+ 'This transformation works only on objects with named'
+ ' fields (namedtuple, BagType, ...).') from e
+
+ return factory(
+ function(value) if key(key_) else value for key_, value in zip(fields, bag)
+ )
+ elif key:
+ return factory(function(value) for value in bag)
+ else:
+ return NOT_MODIFIED
+
+ return _MapFields
+
+
def _count(self, context):
counter = yield ValueHolder(0)
context.send(counter.get())
diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py
index c97407f..c3a9b5d 100644
--- a/bonobo/nodes/io/csv.py
+++ b/bonobo/nodes/io/csv.py
@@ -6,6 +6,7 @@ from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.base import FileHandler
from bonobo.nodes.io.file import FileReader, FileWriter
from bonobo.util import ensure_tuple
+from bonobo.util.collections import tuple_or_const
class CsvHandler(FileHandler):
@@ -36,7 +37,7 @@ class CsvHandler(FileHandler):
# Fields (renamed from headers)
headers = RenamedOption("fields")
- fields = Option(ensure_tuple, required=False)
+ fields = Option(tuple_or_const, required=False)
def get_dialect_kwargs(self):
return {
@@ -108,7 +109,7 @@ class CsvWriter(FileWriter, CsvHandler):
def write(self, file, context, *values, fs):
context.setdefault("lineno", 0)
- fields = context.get_input_fields()
+ fields = context.get_input_fields() if self.fields is None else self.fields
if not context.lineno:
context.writer = self.writer_factory(file)
@@ -126,8 +127,7 @@ class CsvWriter(FileWriter, CsvHandler):
)
context.writer(values)
else:
- for arg in values:
- context.writer(ensure_tuple(arg))
+ context.writer(ensure_tuple(values))
return NOT_MODIFIED
diff --git a/bonobo/util/collections.py b/bonobo/util/collections.py
index 9133e09..9e903ec 100644
--- a/bonobo/util/collections.py
+++ b/bonobo/util/collections.py
@@ -1,5 +1,6 @@
import bisect
import functools
+from collections import Sequence
class sortedlist(list):
@@ -32,6 +33,16 @@ def _with_length_check(f):
return _wrapped
+def tuple_or_const(tuple_or_mixed, *, consts=(None, False), **kwargs):
+ if tuple_or_mixed in consts:
+ return tuple_or_mixed
+ if isinstance(tuple_or_mixed, str):
+ pass
+ elif isinstance(tuple_or_mixed, Sequence):
+ tuple_or_mixed = tuple(tuple_or_mixed)
+ return ensure_tuple(tuple_or_mixed, **kwargs)
+
+
@_with_length_check
def ensure_tuple(tuple_or_mixed, *, cls=None):
"""
diff --git a/bonobo/util/environ.py b/bonobo/util/environ.py
index b1a6635..bba0734 100644
--- a/bonobo/util/environ.py
+++ b/bonobo/util/environ.py
@@ -6,7 +6,7 @@ import warnings
from contextlib import contextmanager
__escape_decoder = codecs.getdecoder("unicode_escape")
-__posix_variable = re.compile("\$\{[^\}]*\}")
+__posix_variable = re.compile(r"\$\{[^\}]*\}")
def parse_var(var):
diff --git a/bonobo/util/errors.py b/bonobo/util/errors.py
index a14ebdd..cc72928 100644
--- a/bonobo/util/errors.py
+++ b/bonobo/util/errors.py
@@ -24,7 +24,7 @@ def sweeten_errors():
length = len(pre_re.sub("\\1\\2\\3", arg))
arg = pre_re.sub(w("\\1") + term.bold("\\2") + w("\\3"), arg)
- arg = re.sub("^ \$ (.*)", term.lightblack(" $ ") + term.reset("\\1"), arg)
+ arg = re.sub(r"^ \$ (.*)", term.lightblack(" $ ") + term.reset("\\1"), arg)
return (arg, length)
diff --git a/docs/install.rst b/docs/install.rst
index e220ab0..1876e49 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -89,7 +89,7 @@ of your python interpreter.
.. note:: Once again, we use `develop` here. New features should go to `develop`, while bugfixes can go to `master`.
-If you can't find the "source" directory, try trunning this:
+If you can't find the "source" directory, try running this:
.. code-block:: shell-session
@@ -122,7 +122,7 @@ Preview versions
----------------
Sometimes, there are pre-versions available (before a major release, for example). By default, pip does not target
-pre-versions to avoid accidental upgrades to a potentially instable software, but you can easily opt-in:
+pre-versions to avoid accidental upgrades to a potentially unstable version, but you can easily opt-in:
.. code-block:: shell-session
@@ -135,10 +135,10 @@ Supported platforms
Linux, OSX and other Unixes
---------------------------
-Bonobo test suite runs continuously on Linux, and core developpers use both OSX and Linux machines. Also, there are jobs
+Bonobo test suite runs continuously on Linux, and core developers use both OSX and Linux machines. Also, there are jobs
running on production linux machines everyday, so the support for those platforms should be quite excellent.
-If you're using some esotheric UNIX machine, there can be surprises (although we're not aware, yet). We do not support
+If you're using some esoteric UNIX machine, there can be surprises (although we're not aware, yet). We do not support
officially those platforms, but if you can actually fix the problems on those systems, we'll be glad to integrate
your patches (as long as it is tested, for both existing linux environments and your strange systems).
diff --git a/tests/nodes/io/test_csv.py b/tests/nodes/io/test_csv.py
index 3173768..7ed8476 100644
--- a/tests/nodes/io/test_csv.py
+++ b/tests/nodes/io/test_csv.py
@@ -100,7 +100,7 @@ class CsvWriterTest(Csv, WriterTest, TestCase):
@incontext()
def test_nofields_multiple_args(self, context):
# multiple args are iterated onto and flattened in output
- context.write_sync((L1, L2), (L3, L4))
+ context.write_sync(L1, L2, L3, L4)
context.stop()
assert self.readlines() == ("a,hey", "b,bee", "c,see", "d,dee")
@@ -111,18 +111,10 @@ class CsvWriterTest(Csv, WriterTest, TestCase):
with pytest.raises(TypeError):
context.write_sync((L1, L2), (L3,))
- @incontext()
- def test_nofields_single_arg(self, context):
- # single args are just dumped, shapes can vary.
- context.write_sync((L1,), (LL,), (L3,))
- context.stop()
-
- assert self.readlines() == ("a,hey", "i,have,more,values", "c,see")
-
@incontext()
def test_nofields_empty_args(self, context):
# empty calls are ignored
context.write_sync(EMPTY, EMPTY, EMPTY)
context.stop()
- assert self.readlines() == ()
+ assert self.readlines() == ('', '', '')
diff --git a/tests/nodes/test_basics.py b/tests/nodes/test_basics.py
index d977653..9decb8e 100644
--- a/tests/nodes/test_basics.py
+++ b/tests/nodes/test_basics.py
@@ -7,6 +7,7 @@ import pytest
import bonobo
from bonobo.constants import EMPTY, NOT_MODIFIED
from bonobo.util import ValueHolder, ensure_tuple
+from bonobo.util.bags import BagType
from bonobo.util.testing import BufferingNodeExecutionContext, ConfigurableNodeTest, StaticNodeTest
@@ -113,3 +114,28 @@ def test_methodcaller():
with BufferingNodeExecutionContext(methodcaller("zfill", 5)) as context:
context.write_sync("a", "bb", "ccc")
assert context.get_buffer() == list(map(ensure_tuple, ["0000a", "000bb", "00ccc"]))
+
+
+MyBag = BagType("MyBag", ["a", "b", "c"])
+
+
+@pytest.mark.parametrize("input_, key, expected", [
+ (MyBag(1, 2, 3), True, MyBag(1, 4, 9)),
+ (MyBag(1, 2, 3), False, MyBag(1, 2, 3)),
+ (MyBag(1, 2, 3), lambda x: x == 'c', MyBag(1, 2, 9)),
+ ((1, 2, 3), True, (1, 4, 9)),
+ ((1, 2, 3), False, (1, 2, 3)),
+])
+def test_map_fields(input_, key, expected):
+ with BufferingNodeExecutionContext(bonobo.MapFields(lambda x: x**2, key)) as context:
+ context.write_sync(input_)
+ assert context.status == '-'
+ [got] = context.get_buffer()
+ assert expected == got
+
+
+def test_map_fields_error():
+ with BufferingNodeExecutionContext(bonobo.MapFields(lambda x: x**2, lambda x: x == 'c')) as context:
+ context.write_sync(tuple())
+ assert context.status == '!'
+ assert context.defunct
diff --git a/tests/util/test_collections.py b/tests/util/test_collections.py
index bd43ac2..127752a 100644
--- a/tests/util/test_collections.py
+++ b/tests/util/test_collections.py
@@ -1,7 +1,7 @@
import pytest
from bonobo.util import ensure_tuple, sortedlist
-from bonobo.util.collections import cast, tuplize
+from bonobo.util.collections import cast, tuplize, tuple_or_const
def test_sortedlist():
@@ -13,6 +13,13 @@ def test_sortedlist():
assert l == [1, 2, 2, 3]
+def test_tuple_or_const():
+ assert tuple_or_const(()) == ()
+ assert tuple_or_const((1, )) == (1, )
+ assert tuple_or_const((1, 2, )) == (1, 2, )
+ assert tuple_or_const([1, 2, ]) == (1, 2, )
+ assert tuple_or_const("aaa") == ('aaa', )
+
def test_ensure_tuple():
assert ensure_tuple("a") == ("a",)
assert ensure_tuple(("a",)) == ("a",)