From ea707c9e84215eabc18a24d29ba53b6f5457522b Mon Sep 17 00:00:00 2001 From: borismo <2323800+borismo@users.noreply.github.com> Date: Sat, 27 Oct 2018 14:55:05 +0200 Subject: [PATCH 1/9] Add configuration for Windows Python 3.7 Also update 3.6 version --- appveyor.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 9cf3107..a12aa5a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,11 +17,19 @@ environment: PYTHON_ARCH: "64" - PYTHON: "C:\\Python36" - PYTHON_VERSION: "3.6.1" + PYTHON_VERSION: "3.6.7" PYTHON_ARCH: "32" - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.1" + PYTHON_VERSION: "3.6.7" + PYTHON_ARCH: "64" + + - PYTHON: "C:\\Python37" + PYTHON_VERSION: "3.7.1" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7.1" PYTHON_ARCH: "64" build: false From 5499c548b089d758a0660531e208e6fe99375e01 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 Oct 2018 15:29:19 +0200 Subject: [PATCH 2/9] Allows to provide False fields to CsvWriter, or to override field headers using the fields= option. --- bonobo/nodes/io/csv.py | 5 +++-- bonobo/util/collections.py | 11 +++++++++++ tests/util/test_collections.py | 9 ++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py index c97407f..52d066b 100644 --- a/bonobo/nodes/io/csv.py +++ b/bonobo/nodes/io/csv.py @@ -6,6 +6,7 @@ from bonobo.constants import NOT_MODIFIED from bonobo.nodes.io.base import FileHandler from bonobo.nodes.io.file import FileReader, FileWriter from bonobo.util import ensure_tuple +from bonobo.util.collections import tuple_or_const class CsvHandler(FileHandler): @@ -36,7 +37,7 @@ class CsvHandler(FileHandler): # Fields (renamed from headers) headers = RenamedOption("fields") - fields = Option(ensure_tuple, required=False) + fields = Option(tuple_or_const, required=False) def get_dialect_kwargs(self): return { @@ -108,7 +109,7 @@ class CsvWriter(FileWriter, CsvHandler): def write(self, file, context, *values, fs): context.setdefault("lineno", 0) - fields = context.get_input_fields() + fields = context.get_input_fields() if self.fields is None else self.fields if not context.lineno: context.writer = self.writer_factory(file) diff --git a/bonobo/util/collections.py b/bonobo/util/collections.py index 9133e09..9e903ec 100644 --- a/bonobo/util/collections.py +++ b/bonobo/util/collections.py @@ -1,5 +1,6 @@ import bisect import functools +from collections import Sequence class sortedlist(list): @@ -32,6 +33,16 @@ def _with_length_check(f): return _wrapped +def tuple_or_const(tuple_or_mixed, *, consts=(None, False), **kwargs): + if tuple_or_mixed in consts: + return tuple_or_mixed + if isinstance(tuple_or_mixed, str): + pass + elif isinstance(tuple_or_mixed, Sequence): + tuple_or_mixed = tuple(tuple_or_mixed) + return ensure_tuple(tuple_or_mixed, **kwargs) + + @_with_length_check def ensure_tuple(tuple_or_mixed, *, cls=None): """ diff --git a/tests/util/test_collections.py b/tests/util/test_collections.py index bd43ac2..127752a 100644 --- a/tests/util/test_collections.py +++ b/tests/util/test_collections.py @@ -1,7 +1,7 @@ import pytest from bonobo.util import ensure_tuple, sortedlist -from bonobo.util.collections import cast, tuplize +from bonobo.util.collections import cast, tuplize, tuple_or_const def test_sortedlist(): @@ -13,6 +13,13 @@ def test_sortedlist(): assert l == [1, 2, 2, 3] +def test_tuple_or_const(): + assert tuple_or_const(()) == () + assert tuple_or_const((1, )) == (1, ) + assert tuple_or_const((1, 2, )) == (1, 2, ) + assert tuple_or_const([1, 2, ]) == (1, 2, ) + assert tuple_or_const("aaa") == ('aaa', ) + def test_ensure_tuple(): assert ensure_tuple("a") == ("a",) assert ensure_tuple(("a",)) == ("a",) From 71ee34012629253826266f101efcee2ba0a68561 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 Oct 2018 15:39:25 +0200 Subject: [PATCH 3/9] Fix django multiple command output. --- bonobo/contrib/django/commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bonobo/contrib/django/commands.py b/bonobo/contrib/django/commands.py index 076ac23..2982165 100644 --- a/bonobo/contrib/django/commands.py +++ b/bonobo/contrib/django/commands.py @@ -61,9 +61,11 @@ class ETLCommand(BaseCommand): for i, graph in enumerate(graph_coll): if not isinstance(graph, bonobo.Graph): raise ValueError('Expected a Graph instance, got {!r}.'.format(graph)) - print(term.lightwhite('{}. {}'.format(i + 1, graph.name))) + print(term.lightwhite('{}. {}'.format(i + 1, graph.name or repr(graph).strip('<>')))) result = bonobo.run(graph, services=services, strategy=strategy) results.append(result) + for node in result.nodes: + print(node.get_statistics_as_string(), node.get_flags_as_string()) print(term.lightblack(" ... return value: " + str(result))) print() From 07160cdcccedd49451bd04fff9e1fd9969a83ac1 Mon Sep 17 00:00:00 2001 From: Jostein Leira Date: Sat, 27 Oct 2018 16:24:21 +0200 Subject: [PATCH 4/9] Fix problem with csv writer writing every field on own line, if not header information. Update tests to accommodate change. --- bonobo/nodes/io/csv.py | 3 +-- tests/nodes/io/test_csv.py | 12 ++---------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py index c97407f..4841896 100644 --- a/bonobo/nodes/io/csv.py +++ b/bonobo/nodes/io/csv.py @@ -126,8 +126,7 @@ class CsvWriter(FileWriter, CsvHandler): ) context.writer(values) else: - for arg in values: - context.writer(ensure_tuple(arg)) + context.writer(ensure_tuple(values)) return NOT_MODIFIED diff --git a/tests/nodes/io/test_csv.py b/tests/nodes/io/test_csv.py index 3173768..7ed8476 100644 --- a/tests/nodes/io/test_csv.py +++ b/tests/nodes/io/test_csv.py @@ -100,7 +100,7 @@ class CsvWriterTest(Csv, WriterTest, TestCase): @incontext() def test_nofields_multiple_args(self, context): # multiple args are iterated onto and flattened in output - context.write_sync((L1, L2), (L3, L4)) + context.write_sync(L1, L2, L3, L4) context.stop() assert self.readlines() == ("a,hey", "b,bee", "c,see", "d,dee") @@ -111,18 +111,10 @@ class CsvWriterTest(Csv, WriterTest, TestCase): with pytest.raises(TypeError): context.write_sync((L1, L2), (L3,)) - @incontext() - def test_nofields_single_arg(self, context): - # single args are just dumped, shapes can vary. - context.write_sync((L1,), (LL,), (L3,)) - context.stop() - - assert self.readlines() == ("a,hey", "i,have,more,values", "c,see") - @incontext() def test_nofields_empty_args(self, context): # empty calls are ignored context.write_sync(EMPTY, EMPTY, EMPTY) context.stop() - assert self.readlines() == () + assert self.readlines() == ('', '', '') From 949940cc27f68bc8e2e17347783322d4a3086dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kry=C5=A1tof=20Piln=C3=A1=C4=8Dek?= Date: Sun, 28 Oct 2018 12:06:18 +0100 Subject: [PATCH 5/9] Move logo to a separate file --- MANIFEST.in | 1 + bonobo/__init__.py | 6 ++++-- bonobo/bonobo.svg | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 bonobo/bonobo.svg diff --git a/MANIFEST.in b/MANIFEST.in index f0e3359..6277daa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include *.txt +include bonobo/bonobo.svg recursive-include bonobo *.py-tpl diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 35dc5c9..76f96a3 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -6,6 +6,7 @@ # Licensed under Apache License 2.0, read the LICENSE file in the root of the source tree. import sys +from pathlib import Path if sys.version_info < (3, 5): raise RuntimeError("Python 3.5+ is required to use Bonobo.") @@ -52,7 +53,8 @@ from bonobo._api import ( from bonobo._version import __version__ __all__ = ["__version__"] + __all__ -__logo__ = '' +with (Path(__file__).parent / "bonobo.svg").open() as f: + __logo__ = f.read() __doc__ = __doc__ __version__ = __version__ @@ -69,4 +71,4 @@ def _repr_html_(): ).format(__logo__, "
".join(get_versions(all=True))) -del sys +del sys, Path, f diff --git a/bonobo/bonobo.svg b/bonobo/bonobo.svg new file mode 100644 index 0000000..c81b1b4 --- /dev/null +++ b/bonobo/bonobo.svg @@ -0,0 +1 @@ + From b95efadf23667d4837cf35fea750ca4fbae30c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kry=C5=A1tof=20Piln=C3=A1=C4=8Dek?= Date: Sun, 28 Oct 2018 13:11:14 +0100 Subject: [PATCH 6/9] Fix deprecation warnings in 3.7 --- bonobo/config/processors.py | 2 +- bonobo/util/environ.py | 2 +- bonobo/util/errors.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py index 6e9b466..3a6a184 100644 --- a/bonobo/config/processors.py +++ b/bonobo/config/processors.py @@ -1,4 +1,4 @@ -from collections import Iterable +from collections.abc import Iterable from contextlib import contextmanager from functools import partial from inspect import signature diff --git a/bonobo/util/environ.py b/bonobo/util/environ.py index b1a6635..bba0734 100644 --- a/bonobo/util/environ.py +++ b/bonobo/util/environ.py @@ -6,7 +6,7 @@ import warnings from contextlib import contextmanager __escape_decoder = codecs.getdecoder("unicode_escape") -__posix_variable = re.compile("\$\{[^\}]*\}") +__posix_variable = re.compile(r"\$\{[^\}]*\}") def parse_var(var): diff --git a/bonobo/util/errors.py b/bonobo/util/errors.py index a14ebdd..cc72928 100644 --- a/bonobo/util/errors.py +++ b/bonobo/util/errors.py @@ -24,7 +24,7 @@ def sweeten_errors(): length = len(pre_re.sub("\\1\\2\\3", arg)) arg = pre_re.sub(w("\\1") + term.bold("\\2") + w("\\3"), arg) - arg = re.sub("^ \$ (.*)", term.lightblack(" $ ") + term.reset("\\1"), arg) + arg = re.sub(r"^ \$ (.*)", term.lightblack(" $ ") + term.reset("\\1"), arg) return (arg, length) From aef12e36ccb50332aa8c1f36f514dc996ac345ab Mon Sep 17 00:00:00 2001 From: Jostein Leira Date: Sun, 28 Oct 2018 21:50:23 +0100 Subject: [PATCH 7/9] Update install.rst Fix some spelling. --- docs/install.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index e220ab0..1876e49 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -89,7 +89,7 @@ of your python interpreter. .. note:: Once again, we use `develop` here. New features should go to `develop`, while bugfixes can go to `master`. -If you can't find the "source" directory, try trunning this: +If you can't find the "source" directory, try running this: .. code-block:: shell-session @@ -122,7 +122,7 @@ Preview versions ---------------- Sometimes, there are pre-versions available (before a major release, for example). By default, pip does not target -pre-versions to avoid accidental upgrades to a potentially instable software, but you can easily opt-in: +pre-versions to avoid accidental upgrades to a potentially unstable version, but you can easily opt-in: .. code-block:: shell-session @@ -135,10 +135,10 @@ Supported platforms Linux, OSX and other Unixes --------------------------- -Bonobo test suite runs continuously on Linux, and core developpers use both OSX and Linux machines. Also, there are jobs +Bonobo test suite runs continuously on Linux, and core developers use both OSX and Linux machines. Also, there are jobs running on production linux machines everyday, so the support for those platforms should be quite excellent. -If you're using some esotheric UNIX machine, there can be surprises (although we're not aware, yet). We do not support +If you're using some esoteric UNIX machine, there can be surprises (although we're not aware, yet). We do not support officially those platforms, but if you can actually fix the problems on those systems, we'll be glad to integrate your patches (as long as it is tested, for both existing linux environments and your strange systems). From 721bdb4318df61cacab38c8626e4b892be1471e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kry=C5=A1tof=20Piln=C3=A1=C4=8Dek?= Date: Sun, 28 Oct 2018 23:50:15 +0100 Subject: [PATCH 8/9] Add MapFields transformation --- bonobo/__init__.py | 1 + bonobo/_api.py | 1 + bonobo/errors.py | 4 ++++ bonobo/nodes/basics.py | 36 ++++++++++++++++++++++++++++++++++++ tests/nodes/test_basics.py | 24 ++++++++++++++++++++++++ 5 files changed, 66 insertions(+) diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 76f96a3..2487780 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -29,6 +29,7 @@ from bonobo._api import ( LdjsonReader, LdjsonWriter, Limit, + MapFields, OrderFields, PickleReader, PickleWriter, diff --git a/bonobo/_api.py b/bonobo/_api.py index ccd2724..ca0d664 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -149,6 +149,7 @@ api.register_group( LdjsonReader, LdjsonWriter, Limit, + MapFields, OrderFields, PickleReader, PickleWriter, diff --git a/bonobo/errors.py b/bonobo/errors.py index c0fb3fa..09995d7 100644 --- a/bonobo/errors.py +++ b/bonobo/errors.py @@ -48,6 +48,10 @@ class UnrecoverableTypeError(UnrecoverableError, TypeError): pass +class UnrecoverableAttributeError(UnrecoverableError, AttributeError): + pass + + class UnrecoverableValueError(UnrecoverableError, ValueError): pass diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index e55f393..ade6b9d 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -10,6 +10,7 @@ from bonobo.config import Configurable, Method, Option, use_context, use_no_inpu from bonobo.config.functools import transformation_factory from bonobo.config.processors import ContextProcessor, use_context_processor from bonobo.constants import NOT_MODIFIED +from bonobo.errors import UnrecoverableAttributeError from bonobo.util.objects import ValueHolder from bonobo.util.term import CLEAR_EOL @@ -18,6 +19,7 @@ __all__ = [ "Format", "Limit", "OrderFields", + "MapFields", "PrettyPrinter", "Rename", "SetFields", @@ -314,6 +316,40 @@ def Format(**formats): return _Format +@transformation_factory +def MapFields(function, key=True): + """ + Transformation factory that maps `function` on the values of a row. + It can be applied either to + 1. all columns (`key=True`), + 2. no column (`key=False`), or + 3. a subset of columns by passing a callable, which takes column name and returns `bool` + (same as the parameter `function` in `filter`). + + :param function: callable + :param key: bool or callable + :return: callable + """ + @use_raw_input + def _MapFields(bag): + try: + factory = type(bag)._make + except AttributeError as e: + raise UnrecoverableAttributeError('This transformation works only on objects with named' + ' fields (namedtuple, BagType, ...).') from e + + if callable(key): + return factory( + function(value) if key(key_) else value for key_, value in zip(bag._fields, bag) + ) + elif key: + return factory(function(value) for value in bag) + else: + return NOT_MODIFIED + + return _MapFields + + def _count(self, context): counter = yield ValueHolder(0) context.send(counter.get()) diff --git a/tests/nodes/test_basics.py b/tests/nodes/test_basics.py index d977653..48660af 100644 --- a/tests/nodes/test_basics.py +++ b/tests/nodes/test_basics.py @@ -7,6 +7,7 @@ import pytest import bonobo from bonobo.constants import EMPTY, NOT_MODIFIED from bonobo.util import ValueHolder, ensure_tuple +from bonobo.util.bags import BagType from bonobo.util.testing import BufferingNodeExecutionContext, ConfigurableNodeTest, StaticNodeTest @@ -113,3 +114,26 @@ def test_methodcaller(): with BufferingNodeExecutionContext(methodcaller("zfill", 5)) as context: context.write_sync("a", "bb", "ccc") assert context.get_buffer() == list(map(ensure_tuple, ["0000a", "000bb", "00ccc"])) + + +MyBag = BagType("MyBag", ["a", "b", "c"]) + + +@pytest.mark.parametrize("input_, key, expected", [ + (MyBag(1, 2, 3), True, MyBag(1, 4, 9)), + (MyBag(1, 2, 3), False, MyBag(1, 2, 3)), + (MyBag(1, 2, 3), lambda x: x == 'c', MyBag(1, 2, 9)), +]) +def test_map_fields(input_, key, expected): + with BufferingNodeExecutionContext(bonobo.MapFields(lambda x: x**2, key)) as context: + context.write_sync(input_) + assert context.status == '-' + [got] = context.get_buffer() + assert expected == got + + +def test_map_fields_error(): + with BufferingNodeExecutionContext(bonobo.MapFields(lambda x: x**2)) as context: + context.write_sync(tuple()) + assert context.status == '!' + assert context.defunct From f93f6bb62eb62695317761dfa298c0fa8abcd00d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kry=C5=A1tof=20Piln=C3=A1=C4=8Dek?= Date: Mon, 29 Oct 2018 13:52:20 +0100 Subject: [PATCH 9/9] fixup! Add MapFields transformation --- bonobo/nodes/basics.py | 14 ++++++++++---- tests/nodes/test_basics.py | 4 +++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index ade6b9d..b0663c2 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -334,13 +334,19 @@ def MapFields(function, key=True): def _MapFields(bag): try: factory = type(bag)._make - except AttributeError as e: - raise UnrecoverableAttributeError('This transformation works only on objects with named' - ' fields (namedtuple, BagType, ...).') from e + except AttributeError: + factory = type(bag) if callable(key): + try: + fields = bag._fields + except AttributeError as e: + raise UnrecoverableAttributeError( + 'This transformation works only on objects with named' + ' fields (namedtuple, BagType, ...).') from e + return factory( - function(value) if key(key_) else value for key_, value in zip(bag._fields, bag) + function(value) if key(key_) else value for key_, value in zip(fields, bag) ) elif key: return factory(function(value) for value in bag) diff --git a/tests/nodes/test_basics.py b/tests/nodes/test_basics.py index 48660af..9decb8e 100644 --- a/tests/nodes/test_basics.py +++ b/tests/nodes/test_basics.py @@ -123,6 +123,8 @@ MyBag = BagType("MyBag", ["a", "b", "c"]) (MyBag(1, 2, 3), True, MyBag(1, 4, 9)), (MyBag(1, 2, 3), False, MyBag(1, 2, 3)), (MyBag(1, 2, 3), lambda x: x == 'c', MyBag(1, 2, 9)), + ((1, 2, 3), True, (1, 4, 9)), + ((1, 2, 3), False, (1, 2, 3)), ]) def test_map_fields(input_, key, expected): with BufferingNodeExecutionContext(bonobo.MapFields(lambda x: x**2, key)) as context: @@ -133,7 +135,7 @@ def test_map_fields(input_, key, expected): def test_map_fields_error(): - with BufferingNodeExecutionContext(bonobo.MapFields(lambda x: x**2)) as context: + with BufferingNodeExecutionContext(bonobo.MapFields(lambda x: x**2, lambda x: x == 'c')) as context: context.write_sync(tuple()) assert context.status == '!' assert context.defunct