From 4154d7e3d839456c047fe9b8ce109cfbc6663715 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 18:27:27 +0200 Subject: [PATCH 1/8] Starting to work on #47 a.k.a the context processor mess. This is a first implementation removing all the uggly function calls, but further work must work on less things called "context", as it is bad for readability. Concerns better separated now. --- bonobo/basics.py | 5 +- bonobo/config/configurables.py | 21 +++-- bonobo/config/processors.py | 79 ++++++++++++++++--- bonobo/execution/base.py | 55 +++++-------- bonobo/execution/node.py | 2 +- bonobo/ext/opendatasoft.py | 1 - bonobo/io/csv.py | 2 - bonobo/io/file.py | 2 - bonobo/io/json.py | 1 - tests/test_basics.py | 27 +++++++ ...rocessors.py => test_config_processors.py} | 27 ++++--- tests/test_execution.py | 5 +- 12 files changed, 152 insertions(+), 75 deletions(-) create mode 100644 tests/test_basics.py rename tests/{context/test_processors.py => test_config_processors.py} (62%) diff --git a/bonobo/basics.py b/bonobo/basics.py index 97282ab..03a1232 100644 --- a/bonobo/basics.py +++ b/bonobo/basics.py @@ -3,7 +3,7 @@ from pprint import pprint as _pprint from colorama import Fore, Style -from bonobo.config.processors import contextual +from bonobo.config.processors import ContextProcessor from bonobo.structs.bags import Bag from bonobo.util.objects import ValueHolder from bonobo.util.term import CLEAR_EOL @@ -49,12 +49,11 @@ def Tee(f): return wrapped -@contextual def count(counter, *args, **kwargs): counter += 1 -@count.add_context_processor +@ContextProcessor.decorate(count) def _count_counter(self, context): counter = ValueHolder(0) yield counter diff --git a/bonobo/config/configurables.py b/bonobo/config/configurables.py index 6be2b29..c1486ee 100644 --- a/bonobo/config/configurables.py +++ b/bonobo/config/configurables.py @@ -1,3 +1,4 @@ +from bonobo.config.processors import ContextProcessor from bonobo.config.options import Option __all__ = [ @@ -15,16 +16,24 @@ class ConfigurableMeta(type): super().__init__(what, bases, dict) cls.__options__ = {} cls.__positional_options__ = [] + cls.__processors__ = [] for typ in cls.__mro__: for name, value in typ.__dict__.items(): if isinstance(value, Option): - if not value.name: - value.name = name - if not name in cls.__options__: - cls.__options__[name] = value - if value.positional: - cls.__positional_options__.append(name) + if isinstance(value, ContextProcessor): + cls.__processors__.append(value) + else: + if not value.name: + value.name = name + if not name in cls.__options__: + cls.__options__[name] = value + if value.positional: + cls.__positional_options__.append(name) + + # This can be done before, more efficiently. Not so bad neither as this is only done at type() creation time + # (aka class Xxx(...) time) and there should not be hundreds of processors. Still not very elegant. + cls.__processors__ = sorted(cls.__processors__, key=lambda v: v._creation_counter) class Configurable(metaclass=ConfigurableMeta): diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py index 9dac6d0..2d3de8f 100644 --- a/bonobo/config/processors.py +++ b/bonobo/config/processors.py @@ -2,24 +2,23 @@ import functools import types -from bonobo.util.compat import deprecated_alias +from bonobo.util.compat import deprecated_alias, deprecated + +from bonobo.config.options import Option +from bonobo.util.iterators import ensure_tuple _CONTEXT_PROCESSORS_ATTR = '__processors__' -class ContextProcessor: - _creation_counter = 0 - +class ContextProcessor(Option): @property def __name__(self): return self.func.__name__ def __init__(self, func): self.func = func - - # This hack is necessary for python3.5 - self._creation_counter = ContextProcessor._creation_counter - ContextProcessor._creation_counter += 1 + super(ContextProcessor, self).__init__(required=False, default=self.__name__) + self.name = self.__name__ def __repr__(self): return repr(self.func).replace(' int: return i**2 -@contextual def push_result(results, i: int): results.append(i) -@push_result.__processors__.append +@ContextProcessor.decorate(push_result) def results(f, context): results = [] yield results From 743de17c4c7a27e6c64ed1203b50c4d06361e0cb Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 18:35:12 +0200 Subject: [PATCH 2/8] mock is in unittest --- tests/test_basicusage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_basicusage.py b/tests/test_basicusage.py index 1cd84a2..a43831d 100644 --- a/tests/test_basicusage.py +++ b/tests/test_basicusage.py @@ -2,7 +2,7 @@ import pytest import bonobo from bonobo.execution import GraphExecutionContext -from mock import patch +from unittest.mock import patch @pytest.mark.timeout(2) From 5111b2932e19f92ccc5d9092f6accf7af4e3ed62 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 18:37:23 +0200 Subject: [PATCH 3/8] mocks work this way under py3.5, apparently. --- tests/test_basics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_basics.py b/tests/test_basics.py index f6dccfa..4243cf2 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -1,4 +1,3 @@ -import pprint from unittest.mock import MagicMock import bonobo @@ -19,7 +18,7 @@ def test_count(): currified() currified.teardown() - context.send.assert_called_once() + assert len(context.method_calls) == 1 bag = context.send.call_args[0][0] assert isinstance(bag, bonobo.Bag) assert 0 == len(bag.kwargs) From da4ba2603c25e4e658e3ba2026c71df85f176e27 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 18:50:06 +0200 Subject: [PATCH 4/8] Update basic tests. --- tests/test_basics.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_basics.py b/tests/test_basics.py index 4243cf2..7a9f317 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock import bonobo import pytest from bonobo.config.processors import ContextCurrifier +from bonobo.constants import NOT_MODIFIED def test_count(): @@ -24,3 +25,38 @@ def test_count(): assert 0 == len(bag.kwargs) assert 1 == len(bag.args) assert bag.args[0] == 42 + + +def test_identity(): + assert bonobo.identity(42) == 42 + + +def test_limit(): + limit = bonobo.Limit(2) + results = [] + for i in range(42): + results += list(limit()) + assert results == [NOT_MODIFIED] * 2 + + +def test_limit_not_there(): + limit = bonobo.Limit(42) + results = [] + for i in range(10): + results += list(limit()) + assert results == [NOT_MODIFIED] * 10 + + +def test_tee(): + inner = MagicMock(side_effect=bonobo.identity) + tee = bonobo.Tee(inner) + results = [] + for i in range(10): + results.append(tee('foo')) + + assert results == [NOT_MODIFIED] * 10 + assert len(inner.mock_calls) == 10 + + +def test_noop(): + assert bonobo.noop(1, 2, 3, 4, foo='bar') == NOT_MODIFIED From 06e7875c89ab6b936a9e7b86b01fde72b6294a49 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 18:51:46 +0200 Subject: [PATCH 5/8] Removes dead code. My pleasure. --- bonobo/util/lifecycle.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 bonobo/util/lifecycle.py diff --git a/bonobo/util/lifecycle.py b/bonobo/util/lifecycle.py deleted file mode 100644 index c497f41..0000000 --- a/bonobo/util/lifecycle.py +++ /dev/null @@ -1,34 +0,0 @@ -from bonobo.basics import noop - - -def _create_lifecycle_functions(noun, verb): - getter = lambda c: getattr(c, verb, noop) - getter.__name__ = 'get_' + noun - - def setter(f): - nonlocal noun, verb - assert callable(f), 'You must provide a callable to decorate with {}.'.format(noun) - - def wrapper(c): - nonlocal verb, f - setattr(f, verb, c) - return f - - return wrapper - - setter.__name__ = 'set_' + noun - - return getter, setter - - -get_initializer, set_initializer = _create_lifecycle_functions('initializer', 'initialize') -get_finalizer, set_finalizer = _create_lifecycle_functions('finalizer', 'finalize') - - -class Contextual: - _with_context = True - - -def with_context(cls_or_func): - cls_or_func._with_context = True - return cls_or_func From 4f9b2081ca5a978a49ed464dbee36522ae29c44f Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 19:03:56 +0200 Subject: [PATCH 6/8] draft of release manual --- docs/contribute/release.rst | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/contribute/release.rst diff --git a/docs/contribute/release.rst b/docs/contribute/release.rst new file mode 100644 index 0000000..8e40f08 --- /dev/null +++ b/docs/contribute/release.rst @@ -0,0 +1,38 @@ +Releases +======== + +WORK IN PROGRESS, THIS DOC IS UNFINISHED AND JUST RAW NOTES TO HELP ME RELEASING STUFF. + +How to make a patch release? +:::::::::::::::::::::::::::: + +For now, reference at http://rdc.li/r + +Additional checklist: + +* make format + +How to make a minor or major release? +::::::::::::::::::::::::::::::::::::: + +Releases above patch level are more complex, because we did not find a way not to hardcode the version number in a bunch +of files, and because a few dependant services (source control, continuous integration, code coverage, documentation +builder ...) also depends on version numbers. + +Checklist: + +* Files +* Github + + +Recipes +::::::: + +Get current minor:: + + git semver | python -c 'import sys; print(".".join(sys.stdin.read().strip().split(".")[0:2]))' + +Open git with all files containing current minor:: + + ack `git semver | python -c 'import sys; print("\\\\.".join(sys.stdin.read().strip().split(".")[0:2]))'` | vim - + From 3d43cad4433f96261be9f665bee844e03f1e7051 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 19:05:33 +0200 Subject: [PATCH 7/8] Removing codacy as it is impossible to configure correctly. --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index 83846f4..a111622 100644 --- a/README.rst +++ b/README.rst @@ -23,10 +23,6 @@ Data-processing. By monkeys. For humans. :target: https://travis-ci.org/python-bonobo/bonobo :alt: Continuous Integration -.. image:: https://api.codacy.com/project/badge/Grade/60aa1ba3ee7b4b4ebd71ca659736c0af - :target: https://www.codacy.com/app/hartym/bonobo - :alt: Code Health from codacy - .. image:: https://landscape.io/github/python-bonobo/bonobo/0.2/landscape.svg?style=flat :target: https://landscape.io/github/python-bonobo/bonobo/0.2 :alt: Code Health from landscape From 96d712d8b965580e587954ec3aea5316ef1a4292 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 19:09:59 +0200 Subject: [PATCH 8/8] Increment minor version number. --- README.rst | 12 ++++++------ docs/contribute/index.rst | 2 +- docs/faq.rst | 2 +- docs/install.rst | 4 ++-- docs/tutorial/tut02.rst | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index a111622..0be6e72 100644 --- a/README.rst +++ b/README.rst @@ -15,20 +15,20 @@ Data-processing. By monkeys. For humans. :target: https://pypi.python.org/pypi/bonobo :alt: Versions -.. image:: https://readthedocs.org/projects/bonobo/badge/?version=0.2 +.. image:: https://readthedocs.org/projects/bonobo/badge/?version=0.3 :target: http://docs.bonobo-project.org/ :alt: Documentation -.. image:: https://travis-ci.org/python-bonobo/bonobo.svg?branch=0.2 +.. image:: https://travis-ci.org/python-bonobo/bonobo.svg?branch=0.3 :target: https://travis-ci.org/python-bonobo/bonobo :alt: Continuous Integration -.. image:: https://landscape.io/github/python-bonobo/bonobo/0.2/landscape.svg?style=flat - :target: https://landscape.io/github/python-bonobo/bonobo/0.2 +.. image:: https://landscape.io/github/python-bonobo/bonobo/0.3/landscape.svg?style=flat + :target: https://landscape.io/github/python-bonobo/bonobo/0.3 :alt: Code Health from landscape -.. image:: https://img.shields.io/coveralls/python-bonobo/bonobo/0.2.svg - :target: https://coveralls.io/github/python-bonobo/bonobo?branch=0.2 +.. image:: https://img.shields.io/coveralls/python-bonobo/bonobo/0.3.svg + :target: https://coveralls.io/github/python-bonobo/bonobo?branch=0.3 :alt: Coverage Bonobo is a data-processing library for python 3.5+ that emphasises writing diff --git a/docs/contribute/index.rst b/docs/contribute/index.rst index 6ee7e31..3c3f6e9 100644 --- a/docs/contribute/index.rst +++ b/docs/contribute/index.rst @@ -58,7 +58,7 @@ Guidelines License ::::::: -`Bonobo is released under the apache license `_. +`Bonobo is released under the apache license `_. License for non lawyers ::::::::::::::::::::::: diff --git a/docs/faq.rst b/docs/faq.rst index d529a7c..6938dcd 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -41,7 +41,7 @@ context. The API may evolve a bit though, because I feel it's a bit hackish, as it is. The concept will stay the same, but we need to find a better way to apply it. -To understand how it works today, look at https://github.com/python-bonobo/bonobo/blob/0.2/bonobo/io/csv.py#L63 and class hierarchy. +To understand how it works today, look at https://github.com/python-bonobo/bonobo/blob/0.3/bonobo/io/csv.py#L63 and class hierarchy. What is a plugin? Do I need to write one? ----------------------------------------- diff --git a/docs/install.rst b/docs/install.rst index 36c7024..ec617fb 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -17,7 +17,7 @@ but editable installs (see below). .. code-block:: shell-session - $ pip install git+https://github.com/python-bonobo/bonobo.git@0.2#egg=bonobo + $ pip install git+https://github.com/python-bonobo/bonobo.git@0.3#egg=bonobo Editable install :::::::::::::::: @@ -28,7 +28,7 @@ python interpreter. .. code-block:: shell-session - $ pip install --editable git+https://github.com/python-bonobo/bonobo.git@0.2#egg=bonobo + $ pip install --editable git+https://github.com/python-bonobo/bonobo.git@0.3#egg=bonobo .. note:: You can also use the `-e` flag instead of the long version. diff --git a/docs/tutorial/tut02.rst b/docs/tutorial/tut02.rst index 1f3f782..0c96a40 100644 --- a/docs/tutorial/tut02.rst +++ b/docs/tutorial/tut02.rst @@ -46,7 +46,7 @@ We'll use a text file that was generated using Bonobo from the "liste-des-cafes- Mairie de Paris under the Open Database License (ODbL). You can `explore the original dataset `_. -You'll need the `example dataset `_, +You'll need the `example dataset `_, available in **Bonobo**'s repository. .. literalinclude:: ../../bonobo/examples/tutorials/tut02_01_read.py @@ -75,7 +75,7 @@ containerization capabilities are provided by an optional and separate python pa It also change a bit the way you can configure service dependencies. The CLI won't run the `if __name__ == '__main__'` block, and thus it won't get the configured services passed to :func:`bonobo.run`. Instead, one option to configure services is to define a `get_services()` function in a -`_services.py `_ file. +`_services.py `_ file. There will be more options using the CLI or environment to override things soon.