diff --git a/README.rst b/README.rst index a111622..0be6e72 100644 --- a/README.rst +++ b/README.rst @@ -15,20 +15,20 @@ Data-processing. By monkeys. For humans. :target: https://pypi.python.org/pypi/bonobo :alt: Versions -.. image:: https://readthedocs.org/projects/bonobo/badge/?version=0.2 +.. image:: https://readthedocs.org/projects/bonobo/badge/?version=0.3 :target: http://docs.bonobo-project.org/ :alt: Documentation -.. image:: https://travis-ci.org/python-bonobo/bonobo.svg?branch=0.2 +.. image:: https://travis-ci.org/python-bonobo/bonobo.svg?branch=0.3 :target: https://travis-ci.org/python-bonobo/bonobo :alt: Continuous Integration -.. image:: https://landscape.io/github/python-bonobo/bonobo/0.2/landscape.svg?style=flat - :target: https://landscape.io/github/python-bonobo/bonobo/0.2 +.. image:: https://landscape.io/github/python-bonobo/bonobo/0.3/landscape.svg?style=flat + :target: https://landscape.io/github/python-bonobo/bonobo/0.3 :alt: Code Health from landscape -.. image:: https://img.shields.io/coveralls/python-bonobo/bonobo/0.2.svg - :target: https://coveralls.io/github/python-bonobo/bonobo?branch=0.2 +.. image:: https://img.shields.io/coveralls/python-bonobo/bonobo/0.3.svg + :target: https://coveralls.io/github/python-bonobo/bonobo?branch=0.3 :alt: Coverage Bonobo is a data-processing library for python 3.5+ that emphasises writing diff --git a/bonobo/basics.py b/bonobo/basics.py index 97282ab..03a1232 100644 --- a/bonobo/basics.py +++ b/bonobo/basics.py @@ -3,7 +3,7 @@ from pprint import pprint as _pprint from colorama import Fore, Style -from bonobo.config.processors import contextual +from bonobo.config.processors import ContextProcessor from bonobo.structs.bags import Bag from bonobo.util.objects import ValueHolder from bonobo.util.term import CLEAR_EOL @@ -49,12 +49,11 @@ def Tee(f): return wrapped -@contextual def count(counter, *args, **kwargs): counter += 1 -@count.add_context_processor +@ContextProcessor.decorate(count) def _count_counter(self, context): counter = ValueHolder(0) yield counter diff --git a/bonobo/config/configurables.py b/bonobo/config/configurables.py index 6be2b29..c1486ee 100644 --- a/bonobo/config/configurables.py +++ b/bonobo/config/configurables.py @@ -1,3 +1,4 @@ +from bonobo.config.processors import ContextProcessor from bonobo.config.options import Option __all__ = [ @@ -15,16 +16,24 @@ class ConfigurableMeta(type): super().__init__(what, bases, dict) cls.__options__ = {} cls.__positional_options__ = [] + cls.__processors__ = [] for typ in cls.__mro__: for name, value in typ.__dict__.items(): if isinstance(value, Option): - if not value.name: - value.name = name - if not name in cls.__options__: - cls.__options__[name] = value - if value.positional: - cls.__positional_options__.append(name) + if isinstance(value, ContextProcessor): + cls.__processors__.append(value) + else: + if not value.name: + value.name = name + if not name in cls.__options__: + cls.__options__[name] = value + if value.positional: + cls.__positional_options__.append(name) + + # This can be done before, more efficiently. Not so bad neither as this is only done at type() creation time + # (aka class Xxx(...) time) and there should not be hundreds of processors. Still not very elegant. + cls.__processors__ = sorted(cls.__processors__, key=lambda v: v._creation_counter) class Configurable(metaclass=ConfigurableMeta): diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py index 9dac6d0..2d3de8f 100644 --- a/bonobo/config/processors.py +++ b/bonobo/config/processors.py @@ -2,24 +2,23 @@ import functools import types -from bonobo.util.compat import deprecated_alias +from bonobo.util.compat import deprecated_alias, deprecated + +from bonobo.config.options import Option +from bonobo.util.iterators import ensure_tuple _CONTEXT_PROCESSORS_ATTR = '__processors__' -class ContextProcessor: - _creation_counter = 0 - +class ContextProcessor(Option): @property def __name__(self): return self.func.__name__ def __init__(self, func): self.func = func - - # This hack is necessary for python3.5 - self._creation_counter = ContextProcessor._creation_counter - ContextProcessor._creation_counter += 1 + super(ContextProcessor, self).__init__(required=False, default=self.__name__) + self.name = self.__name__ def __repr__(self): return repr(self.func).replace('`_. +`Bonobo is released under the apache license `_. License for non lawyers ::::::::::::::::::::::: diff --git a/docs/contribute/release.rst b/docs/contribute/release.rst new file mode 100644 index 0000000..8e40f08 --- /dev/null +++ b/docs/contribute/release.rst @@ -0,0 +1,38 @@ +Releases +======== + +WORK IN PROGRESS, THIS DOC IS UNFINISHED AND JUST RAW NOTES TO HELP ME RELEASING STUFF. + +How to make a patch release? +:::::::::::::::::::::::::::: + +For now, reference at http://rdc.li/r + +Additional checklist: + +* make format + +How to make a minor or major release? +::::::::::::::::::::::::::::::::::::: + +Releases above patch level are more complex, because we did not find a way not to hardcode the version number in a bunch +of files, and because a few dependant services (source control, continuous integration, code coverage, documentation +builder ...) also depends on version numbers. + +Checklist: + +* Files +* Github + + +Recipes +::::::: + +Get current minor:: + + git semver | python -c 'import sys; print(".".join(sys.stdin.read().strip().split(".")[0:2]))' + +Open git with all files containing current minor:: + + ack `git semver | python -c 'import sys; print("\\\\.".join(sys.stdin.read().strip().split(".")[0:2]))'` | vim - + diff --git a/docs/faq.rst b/docs/faq.rst index d529a7c..6938dcd 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -41,7 +41,7 @@ context. The API may evolve a bit though, because I feel it's a bit hackish, as it is. The concept will stay the same, but we need to find a better way to apply it. -To understand how it works today, look at https://github.com/python-bonobo/bonobo/blob/0.2/bonobo/io/csv.py#L63 and class hierarchy. +To understand how it works today, look at https://github.com/python-bonobo/bonobo/blob/0.3/bonobo/io/csv.py#L63 and class hierarchy. What is a plugin? Do I need to write one? ----------------------------------------- diff --git a/docs/install.rst b/docs/install.rst index 36c7024..ec617fb 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -17,7 +17,7 @@ but editable installs (see below). .. code-block:: shell-session - $ pip install git+https://github.com/python-bonobo/bonobo.git@0.2#egg=bonobo + $ pip install git+https://github.com/python-bonobo/bonobo.git@0.3#egg=bonobo Editable install :::::::::::::::: @@ -28,7 +28,7 @@ python interpreter. .. code-block:: shell-session - $ pip install --editable git+https://github.com/python-bonobo/bonobo.git@0.2#egg=bonobo + $ pip install --editable git+https://github.com/python-bonobo/bonobo.git@0.3#egg=bonobo .. note:: You can also use the `-e` flag instead of the long version. diff --git a/docs/tutorial/tut02.rst b/docs/tutorial/tut02.rst index 1f3f782..0c96a40 100644 --- a/docs/tutorial/tut02.rst +++ b/docs/tutorial/tut02.rst @@ -46,7 +46,7 @@ We'll use a text file that was generated using Bonobo from the "liste-des-cafes- Mairie de Paris under the Open Database License (ODbL). You can `explore the original dataset `_. -You'll need the `example dataset `_, +You'll need the `example dataset `_, available in **Bonobo**'s repository. .. literalinclude:: ../../bonobo/examples/tutorials/tut02_01_read.py @@ -75,7 +75,7 @@ containerization capabilities are provided by an optional and separate python pa It also change a bit the way you can configure service dependencies. The CLI won't run the `if __name__ == '__main__'` block, and thus it won't get the configured services passed to :func:`bonobo.run`. Instead, one option to configure services is to define a `get_services()` function in a -`_services.py `_ file. +`_services.py `_ file. There will be more options using the CLI or environment to override things soon. diff --git a/tests/test_basics.py b/tests/test_basics.py new file mode 100644 index 0000000..7a9f317 --- /dev/null +++ b/tests/test_basics.py @@ -0,0 +1,62 @@ +from unittest.mock import MagicMock + +import bonobo +import pytest +from bonobo.config.processors import ContextCurrifier +from bonobo.constants import NOT_MODIFIED + + +def test_count(): + with pytest.raises(TypeError): + bonobo.count() + + context = MagicMock() + + currified = ContextCurrifier(bonobo.count) + currified.setup(context) + + for i in range(42): + currified() + currified.teardown() + + assert len(context.method_calls) == 1 + bag = context.send.call_args[0][0] + assert isinstance(bag, bonobo.Bag) + assert 0 == len(bag.kwargs) + assert 1 == len(bag.args) + assert bag.args[0] == 42 + + +def test_identity(): + assert bonobo.identity(42) == 42 + + +def test_limit(): + limit = bonobo.Limit(2) + results = [] + for i in range(42): + results += list(limit()) + assert results == [NOT_MODIFIED] * 2 + + +def test_limit_not_there(): + limit = bonobo.Limit(42) + results = [] + for i in range(10): + results += list(limit()) + assert results == [NOT_MODIFIED] * 10 + + +def test_tee(): + inner = MagicMock(side_effect=bonobo.identity) + tee = bonobo.Tee(inner) + results = [] + for i in range(10): + results.append(tee('foo')) + + assert results == [NOT_MODIFIED] * 10 + assert len(inner.mock_calls) == 10 + + +def test_noop(): + assert bonobo.noop(1, 2, 3, 4, foo='bar') == NOT_MODIFIED diff --git a/tests/test_basicusage.py b/tests/test_basicusage.py index 1cd84a2..a43831d 100644 --- a/tests/test_basicusage.py +++ b/tests/test_basicusage.py @@ -2,7 +2,7 @@ import pytest import bonobo from bonobo.execution import GraphExecutionContext -from mock import patch +from unittest.mock import patch @pytest.mark.timeout(2) diff --git a/tests/context/test_processors.py b/tests/test_config_processors.py similarity index 62% rename from tests/context/test_processors.py rename to tests/test_config_processors.py index bf62500..3377925 100644 --- a/tests/context/test_processors.py +++ b/tests/test_config_processors.py @@ -1,24 +1,26 @@ from operator import attrgetter -from bonobo.config.processors import ContextProcessor, contextual, resolve_processors +from bonobo.config import Configurable +from bonobo.config.processors import ContextProcessor, resolve_processors, ContextCurrifier -@contextual -class CP1: +class CP1(Configurable): @ContextProcessor def c(self): - pass + yield @ContextProcessor def a(self): - pass + yield 'this is A' @ContextProcessor - def b(self): - pass + def b(self, a): + yield a.upper()[:-1] + 'b' + + def __call__(self, a, b): + return a, b -@contextual class CP2(CP1): @ContextProcessor def f(self): @@ -33,7 +35,6 @@ class CP2(CP1): pass -@contextual class CP3(CP2): @ContextProcessor def c(self): @@ -52,3 +53,11 @@ def test_inheritance_and_ordering(): assert get_all_processors_names(CP1) == ['c', 'a', 'b'] assert get_all_processors_names(CP2) == ['c', 'a', 'b', 'f', 'e', 'd'] assert get_all_processors_names(CP3) == ['c', 'a', 'b', 'f', 'e', 'd', 'c', 'b'] + + +def test_setup_teardown(): + o = CP1() + stack = ContextCurrifier(o) + stack.setup() + assert o(*stack.context) == ('this is A', 'THIS IS b') + stack.teardown() diff --git a/tests/test_execution.py b/tests/test_execution.py index 5f9067b..5194903 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -1,4 +1,4 @@ -from bonobo.config.processors import contextual +from bonobo.config.processors import ContextProcessor from bonobo.constants import BEGIN, END from bonobo.execution.graph import GraphExecutionContext from bonobo.strategies import NaiveStrategy @@ -13,12 +13,11 @@ def square(i: int) -> int: return i**2 -@contextual def push_result(results, i: int): results.append(i) -@push_result.__processors__.append +@ContextProcessor.decorate(push_result) def results(f, context): results = [] yield results