From 2b8397f32ed2b81ee945e4b8869b2f5eb98beca7 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 11 Jun 2017 09:50:20 +0200 Subject: [PATCH 01/24] [logging] Adds logging alias for easier imports. --- bonobo/logging.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bonobo/logging.py b/bonobo/logging.py index 1561089..4626243 100644 --- a/bonobo/logging.py +++ b/bonobo/logging.py @@ -70,6 +70,9 @@ def set_level(level): def get_logger(name='bonobo'): return logging.getLogger(name) +# Compatibility with python logging +getLogger = get_logger + # Setup formating and level. setup(level=settings.LOGGING_LEVEL) From f832f8e2bcd2840cbec16a7da3d20b8276b393b0 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 11 Jun 2017 10:44:01 +0200 Subject: [PATCH 02/24] release: 0.4.1 --- bonobo/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bonobo/_version.py b/bonobo/_version.py index abeeedb..f0ede3d 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = '0.4.0' +__version__ = '0.4.1' From 079ad57a615a05f6943d85739efe5032c26e8436 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 11 Jun 2017 17:15:17 +0200 Subject: [PATCH 03/24] Update fs from 2.0.3 to 2.0.4 --- requirements-docker.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docker.txt b/requirements-docker.txt index fc84888..457488e 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -7,7 +7,7 @@ colorama==0.3.9 docker-pycreds==0.2.1 docker==2.3.0 enum34==1.1.6 -fs==2.0.3 +fs==2.0.4 idna==2.5 pbr==3.0.1 psutil==5.2.2 From 6e373a583d9494190696ef342b80cec931200751 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 11 Jun 2017 17:15:19 +0200 Subject: [PATCH 04/24] Update fs from 2.0.3 to 2.0.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ab50600..8ee6330 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9 enum34==1.1.6 -fs==2.0.3 +fs==2.0.4 idna==2.5 packaging==16.8 pbr==3.0.1 From 735ae43fc6447cabaeb53f946c195ba86a52bda5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 11 Jun 2017 22:14:16 +0200 Subject: [PATCH 05/24] Update bonobo-docker from 0.2.6 to 0.2.8 --- requirements-docker.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docker.txt b/requirements-docker.txt index fc84888..1443f7f 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,6 +1,6 @@ -e .[docker] appdirs==1.4.3 -bonobo-docker==0.2.6 +bonobo-docker==0.2.8 certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9 From 6a0334dcdf1bc987e852cdf0d9953b0b5a9811ce Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 12 Jun 2017 16:24:38 +0200 Subject: [PATCH 06/24] Create index.html --- docs/_templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/index.html b/docs/_templates/index.html index a2ee73b..b778554 100644 --- a/docs/_templates/index.html +++ b/docs/_templates/index.html @@ -49,7 +49,7 @@ -

{% trans %} +

{% trans %} Cookbook{% endtrans %}
{% trans %}examples and recipes{% endtrans %}

From dbb6491e7f2cffa429a92dc5af8012d36b1cf7ea Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 14 Jun 2017 21:38:35 +0200 Subject: [PATCH 07/24] Update requests from 2.17.3 to 2.18.1 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c9c94bb..6c33b15 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,7 +24,7 @@ pytest-timeout==1.2.0 pytest==3.1.2 python-dateutil==2.6.0 pytz==2017.2 -requests==2.17.3 +requests==2.18.1 six==1.10.0 snowballstemmer==1.2.1 sphinx==1.6.2 From c42520987f191b93866a8a88d0c30a84ab271b78 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 14 Jun 2017 21:38:37 +0200 Subject: [PATCH 08/24] Update requests from 2.17.3 to 2.18.1 --- requirements-docker.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docker.txt b/requirements-docker.txt index 21a0cac..b004174 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -12,7 +12,7 @@ idna==2.5 pbr==3.0.1 psutil==5.2.2 pytz==2017.2 -requests==2.17.3 +requests==2.18.1 six==1.10.0 stevedore==1.23.0 urllib3==1.21.1 From e8ab8a8b576a051af061ffb272b3ab7e4cf1796d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 14 Jun 2017 21:38:38 +0200 Subject: [PATCH 09/24] Update requests from 2.17.3 to 2.18.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8ee6330..478879e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ pbr==3.0.1 psutil==5.2.2 pyparsing==2.2.0 pytz==2017.2 -requests==2.17.3 +requests==2.18.1 six==1.10.0 stevedore==1.23.0 urllib3==1.21.1 From 8c898afc273e74e9773bde0a8a489a3defaa5474 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 17 Jun 2017 08:02:29 +0200 Subject: [PATCH 10/24] [config] Implements a "requires()" service injection decorator for functions (api may change). --- bonobo/config/__init__.py | 5 +++-- bonobo/config/services.py | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/bonobo/config/__init__.py b/bonobo/config/__init__.py index 8e662c4..08be544 100644 --- a/bonobo/config/__init__.py +++ b/bonobo/config/__init__.py @@ -1,7 +1,7 @@ from bonobo.config.configurables import Configurable -from bonobo.config.options import Option, Method +from bonobo.config.options import Method, Option from bonobo.config.processors import ContextProcessor -from bonobo.config.services import Container, Service, Exclusive +from bonobo.config.services import Container, Exclusive, Service, requires # bonobo.config public programming interface __all__ = [ @@ -12,4 +12,5 @@ __all__ = [ 'Method', 'Option', 'Service', + 'requires', ] diff --git a/bonobo/config/services.py b/bonobo/config/services.py index 4a91668..d792175 100644 --- a/bonobo/config/services.py +++ b/bonobo/config/services.py @@ -56,7 +56,11 @@ class Service(Option): inst.__options_values__[self.name] = validate_service_name(value) def resolve(self, inst, services): - return services.get(getattr(inst, self.name)) + try: + name = getattr(inst, self.name) + except AttributeError: + name = self.name + return services.get(name) class Container(dict): @@ -126,3 +130,19 @@ class Exclusive(ContextDecorator): def __exit__(self, *exc): self.get_lock().release() + + +def requires(*service_names): + def decorate(mixed): + try: + options = mixed.__options__ + except AttributeError: + mixed.__options__ = options = {} + + for service_name in service_names: + service = Service(service_name) + service.name = service_name + options[service_name] = service + return mixed + + return decorate From a65ca635cf953ba8ac4b4a1d9fb7378e53bcce27 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 17 Jun 2017 10:08:06 +0200 Subject: [PATCH 11/24] [fs] adds a defaut to current working directory in open_fs(...). --- bonobo/_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bonobo/_api.py b/bonobo/_api.py index 9317e31..cf28a33 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -68,7 +68,7 @@ register_api(create_strategy) # Shortcut to filesystem2's open_fs, that we make available there for convenience. @register_api -def open_fs(fs_url, *args, **kwargs): +def open_fs(fs_url=None, *args, **kwargs): """ Wraps :func:`fs.open_fs` function with a few candies. @@ -83,6 +83,10 @@ def open_fs(fs_url, *args, **kwargs): """ from fs import open_fs as _open_fs from os.path import expanduser + from os import getcwd + + if fs_url is None: + fs_url = getcwd() return _open_fs(expanduser(str(fs_url)), *args, **kwargs) From 3c4010f9c3113ac27c4f36f482b0d4c3c9604fb6 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 17 Jun 2017 10:17:42 +0200 Subject: [PATCH 12/24] [core] Execution contexts are now context managers. --- bonobo/execution/base.py | 7 +++++++ bonobo/nodes/io/csv.py | 2 +- bonobo/strategies/executor.py | 14 +++++--------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/bonobo/execution/base.py b/bonobo/execution/base.py index 779f212..641d761 100644 --- a/bonobo/execution/base.py +++ b/bonobo/execution/base.py @@ -58,6 +58,13 @@ class LoopingExecutionContext(Wrapper): # XXX enhancers self._enhancers = get_enhancers(self.wrapped) + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type=None, exc_val=None, exc_tb=None): + self.stop() + def start(self): if self.started: raise RuntimeError('Cannot start a node twice ({}).'.format(get_name(self))) diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py index da70444..bf3872d 100644 --- a/bonobo/nodes/io/csv.py +++ b/bonobo/nodes/io/csv.py @@ -59,7 +59,7 @@ class CsvReader(CsvHandler, FileReader): for row in reader: if len(row) != field_count: - raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count, )) + raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count,)) yield self.get_output(dict(zip(_headers, row))) diff --git a/bonobo/strategies/executor.py b/bonobo/strategies/executor.py index 26b810b..d2cdcbe 100644 --- a/bonobo/strategies/executor.py +++ b/bonobo/strategies/executor.py @@ -1,6 +1,5 @@ import time import traceback - from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor from bonobo.constants import BEGIN, END @@ -29,19 +28,16 @@ class ExecutorStrategy(Strategy): futures = [] for plugin_context in context.plugins: - def _runner(plugin_context=plugin_context): - try: - plugin_context.start() - plugin_context.loop() - plugin_context.stop() - except Exception as exc: - print_error(exc, traceback.format_exc(), context=plugin_context) + with plugin_context: + try: + plugin_context.loop() + except Exception as exc: + print_error(exc, traceback.format_exc(), context=plugin_context) futures.append(executor.submit(_runner)) for node_context in context.nodes: - def _runner(node_context=node_context): try: node_context.start() From 67b42274366e021768f52afb1c7bc2454d915778 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 17 Jun 2017 10:37:17 +0200 Subject: [PATCH 13/24] [stdlib] Fix I/O related nodes (especially json), there were bad bugs with ioformat. --- .gitignore | 9 +---- bonobo/logging.py | 2 +- bonobo/nodes/io/base.py | 82 +++++++++++++++++++++++++++++++++++++++ bonobo/nodes/io/csv.py | 9 +++-- bonobo/nodes/io/file.py | 82 +++------------------------------------ bonobo/nodes/io/json.py | 17 +++++--- bonobo/nodes/io/pickle.py | 10 ++--- bonobo/util/testing.py | 18 +++++++++ config/__init__.py | 0 tests/io/test_csv.py | 78 ++++++++++++++++++++----------------- tests/io/test_file.py | 60 ++++++++++++---------------- tests/io/test_json.py | 56 +++++++++++++------------- tests/io/test_pickle.py | 39 ++++++++----------- 13 files changed, 243 insertions(+), 219 deletions(-) create mode 100644 bonobo/nodes/io/base.py delete mode 100644 config/__init__.py diff --git a/.gitignore b/.gitignore index d48b40b..f16c58c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *,cover *.egg *.egg-info/ +*.iml *.log *.manifest *.mo @@ -20,25 +21,17 @@ .installed.cfg .ipynb_checkpoints .python-version -.tox/ -.webassets-cache /.idea /.release -/bonobo.iml /bonobo/examples/work_in_progress/ /bonobo/ext/jupyter/js/node_modules/ /build/ /coverage.xml -/develop-eggs/ /dist/ /docs/_build/ -/downloads/ /eggs/ /examples/private -/htmlcov/ /sdist/ /tags -celerybeat-schedule -parts/ pip-delete-this-directory.txt pip-log.txt diff --git a/bonobo/logging.py b/bonobo/logging.py index 4626243..17bdeb7 100644 --- a/bonobo/logging.py +++ b/bonobo/logging.py @@ -70,9 +70,9 @@ def set_level(level): def get_logger(name='bonobo'): return logging.getLogger(name) + # Compatibility with python logging getLogger = get_logger - # Setup formating and level. setup(level=settings.LOGGING_LEVEL) diff --git a/bonobo/nodes/io/base.py b/bonobo/nodes/io/base.py new file mode 100644 index 0000000..d9b3212 --- /dev/null +++ b/bonobo/nodes/io/base.py @@ -0,0 +1,82 @@ +from bonobo import settings +from bonobo.config import Configurable, ContextProcessor, Option, Service +from bonobo.structs.bags import Bag + + +class IOFormatEnabled(Configurable): + ioformat = Option(default=settings.IOFORMAT.get) + + def get_input(self, *args, **kwargs): + if self.ioformat == settings.IOFORMAT_ARG0: + if len(args) != 1 or len(kwargs): + raise ValueError( + 'Wrong input formating: IOFORMAT=ARG0 implies one arg and no kwargs, got args={!r} and kwargs={!r}.'. + format(args, kwargs) + ) + return args[0] + + if self.ioformat == settings.IOFORMAT_KWARGS: + if len(args) or not len(kwargs): + raise ValueError( + 'Wrong input formating: IOFORMAT=KWARGS ioformat implies no arg, got args={!r} and kwargs={!r}.'. + format(args, kwargs) + ) + return kwargs + + raise NotImplementedError('Unsupported format.') + + def get_output(self, row): + if self.ioformat == settings.IOFORMAT_ARG0: + return row + + if self.ioformat == settings.IOFORMAT_KWARGS: + return Bag(**row) + + raise NotImplementedError('Unsupported format.') + + +class FileHandler(Configurable): + """Abstract component factory for file-related components. + + Args: + path (str): which path to use within the provided filesystem. + eol (str): which character to use to separate lines. + mode (str): which mode to use when opening the file. + fs (str): service name to use for filesystem. + """ + + path = Option(str, required=True, positional=True) # type: str + eol = Option(str, default='\n') # type: str + mode = Option(str) # type: str + encoding = Option(str, default='utf-8') # type: str + fs = Service('fs') # type: str + + @ContextProcessor + def file(self, context, fs): + with self.open(fs) as file: + yield file + + def open(self, fs): + return fs.open(self.path, self.mode, encoding=self.encoding) + + +class Reader: + """Abstract component factory for readers. + """ + + def __call__(self, *args, **kwargs): + yield from self.read(*args, **kwargs) + + def read(self, *args, **kwargs): + raise NotImplementedError('Abstract.') + + +class Writer: + """Abstract component factory for writers. + """ + + def __call__(self, *args, **kwargs): + return self.write(*args, **kwargs) + + def write(self, *args, **kwargs): + raise NotImplementedError('Abstract.') diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py index bf3872d..ae68bd0 100644 --- a/bonobo/nodes/io/csv.py +++ b/bonobo/nodes/io/csv.py @@ -3,7 +3,8 @@ import csv from bonobo.config import Option from bonobo.config.processors import ContextProcessor from bonobo.constants import NOT_MODIFIED -from bonobo.nodes.io.file import FileHandler, FileReader, FileWriter +from bonobo.nodes.io.file import FileReader, FileWriter +from bonobo.nodes.io.base import FileHandler, IOFormatEnabled from bonobo.util.objects import ValueHolder @@ -28,7 +29,7 @@ class CsvHandler(FileHandler): headers = Option(tuple) -class CsvReader(CsvHandler, FileReader): +class CsvReader(IOFormatEnabled, FileReader, CsvHandler): """ Reads a CSV and yield the values as dicts. @@ -59,12 +60,12 @@ class CsvReader(CsvHandler, FileReader): for row in reader: if len(row) != field_count: - raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count,)) + raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count, )) yield self.get_output(dict(zip(_headers, row))) -class CsvWriter(CsvHandler, FileWriter): +class CsvWriter(IOFormatEnabled, FileWriter, CsvHandler): @ContextProcessor def writer(self, context, fs, file, lineno): writer = csv.writer(file, delimiter=self.delimiter, quotechar=self.quotechar, lineterminator=self.eol) diff --git a/bonobo/nodes/io/file.py b/bonobo/nodes/io/file.py index 53ba138..e49d6de 100644 --- a/bonobo/nodes/io/file.py +++ b/bonobo/nodes/io/file.py @@ -1,81 +1,11 @@ -from bonobo import settings -from bonobo.config import Option, Service -from bonobo.config.configurables import Configurable +from bonobo.config import Option from bonobo.config.processors import ContextProcessor from bonobo.constants import NOT_MODIFIED -from bonobo.structs.bags import Bag +from bonobo.nodes.io.base import FileHandler, Reader, Writer from bonobo.util.objects import ValueHolder -class FileHandler(Configurable): - """Abstract component factory for file-related components. - - Args: - path (str): which path to use within the provided filesystem. - eol (str): which character to use to separate lines. - mode (str): which mode to use when opening the file. - fs (str): service name to use for filesystem. - """ - - path = Option(str, required=True, positional=True) # type: str - eol = Option(str, default='\n') # type: str - mode = Option(str) # type: str - encoding = Option(str, default='utf-8') # type: str - fs = Service('fs') # type: str - ioformat = Option(default=settings.IOFORMAT.get) - - @ContextProcessor - def file(self, context, fs): - with self.open(fs) as file: - yield file - - def open(self, fs): - return fs.open(self.path, self.mode, encoding=self.encoding) - - def get_input(self, *args, **kwargs): - if self.ioformat == settings.IOFORMAT_ARG0: - assert len(args) == 1 and not len(kwargs), 'ARG0 format implies one arg and no kwargs.' - return args[0] - - if self.ioformat == settings.IOFORMAT_KWARGS: - assert len(args) == 0 and len(kwargs), 'KWARGS format implies no arg.' - return kwargs - - raise NotImplementedError('Unsupported format.') - - def get_output(self, row): - if self.ioformat == settings.IOFORMAT_ARG0: - return row - - if self.ioformat == settings.IOFORMAT_KWARGS: - return Bag(**row) - - raise NotImplementedError('Unsupported format.') - - -class Reader(FileHandler): - """Abstract component factory for readers. - """ - - def __call__(self, *args, **kwargs): - yield from self.read(*args, **kwargs) - - def read(self, *args, **kwargs): - raise NotImplementedError('Abstract.') - - -class Writer(FileHandler): - """Abstract component factory for writers. - """ - - def __call__(self, *args, **kwargs): - return self.write(*args) - - def write(self, *args, **kwargs): - raise NotImplementedError('Abstract.') - - -class FileReader(Reader): +class FileReader(Reader, FileHandler): """Component factory for file-like readers. On its own, it can be used to read a file and yield one row per line, trimming the "eol" character at the end if @@ -93,7 +23,7 @@ class FileReader(Reader): yield line.rstrip(self.eol) -class FileWriter(Writer): +class FileWriter(Writer, FileHandler): """Component factory for file or file-like writers. On its own, it can be used to write in a file one line per row that comes into this component. Extending it is @@ -107,11 +37,11 @@ class FileWriter(Writer): lineno = ValueHolder(0) yield lineno - def write(self, fs, file, lineno, row): + def write(self, fs, file, lineno, line): """ Write a row on the next line of opened file in context. """ - self._write_line(file, (self.eol if lineno.value else '') + row) + self._write_line(file, (self.eol if lineno.value else '') + line) lineno += 1 return NOT_MODIFIED diff --git a/bonobo/nodes/io/json.py b/bonobo/nodes/io/json.py index f355c02..c6d9bf5 100644 --- a/bonobo/nodes/io/json.py +++ b/bonobo/nodes/io/json.py @@ -1,15 +1,17 @@ import json from bonobo.config.processors import ContextProcessor -from bonobo.nodes.io.file import FileWriter, FileReader +from bonobo.constants import NOT_MODIFIED +from bonobo.nodes.io.base import FileHandler, IOFormatEnabled +from bonobo.nodes.io.file import FileReader, FileWriter -class JsonHandler(): +class JsonHandler(FileHandler): eol = ',\n' prefix, suffix = '[', ']' -class JsonReader(JsonHandler, FileReader): +class JsonReader(IOFormatEnabled, FileReader, JsonHandler): loader = staticmethod(json.load) def read(self, fs, file): @@ -17,18 +19,21 @@ class JsonReader(JsonHandler, FileReader): yield self.get_output(line) -class JsonWriter(JsonHandler, FileWriter): +class JsonWriter(IOFormatEnabled, FileWriter, JsonHandler): @ContextProcessor def envelope(self, context, fs, file, lineno): file.write(self.prefix) yield file.write(self.suffix) - def write(self, fs, file, lineno, row): + def write(self, fs, file, lineno, *args, **kwargs): """ Write a json row on the next line of file pointed by ctx.file. :param ctx: :param row: """ - return super().write(fs, file, lineno, json.dumps(row)) + row = self.get_input(*args, **kwargs) + self._write_line(file, (self.eol if lineno.value else '') + json.dumps(row)) + lineno += 1 + return NOT_MODIFIED diff --git a/bonobo/nodes/io/pickle.py b/bonobo/nodes/io/pickle.py index c603e91..e94f94a 100644 --- a/bonobo/nodes/io/pickle.py +++ b/bonobo/nodes/io/pickle.py @@ -1,10 +1,11 @@ import pickle -from bonobo.config.processors import ContextProcessor from bonobo.config import Option +from bonobo.config.processors import ContextProcessor from bonobo.constants import NOT_MODIFIED +from bonobo.nodes.io.base import FileHandler, IOFormatEnabled +from bonobo.nodes.io.file import FileReader, FileWriter from bonobo.util.objects import ValueHolder -from .file import FileReader, FileWriter, FileHandler class PickleHandler(FileHandler): @@ -19,7 +20,7 @@ class PickleHandler(FileHandler): item_names = Option(tuple) -class PickleReader(PickleHandler, FileReader): +class PickleReader(IOFormatEnabled, FileReader, PickleHandler): """ Reads a Python pickle object and yields the items in dicts. """ @@ -56,8 +57,7 @@ class PickleReader(PickleHandler, FileReader): yield self.get_output(dict(zip(i)) if is_dict else dict(zip(pickle_headers.value, i))) -class PickleWriter(PickleHandler, FileWriter): - +class PickleWriter(IOFormatEnabled, FileWriter, PickleHandler): mode = Option(str, default='wb') def write(self, fs, file, lineno, item): diff --git a/bonobo/util/testing.py b/bonobo/util/testing.py index d5b6cc8..7c07256 100644 --- a/bonobo/util/testing.py +++ b/bonobo/util/testing.py @@ -1,6 +1,7 @@ from contextlib import contextmanager from unittest.mock import MagicMock +from bonobo import open_fs from bonobo.execution.node import NodeExecutionContext @@ -17,3 +18,20 @@ def optional_contextmanager(cm, *, ignore=False): else: with cm: yield + + +class FilesystemTester: + def __init__(self, extension='txt', mode='w'): + self.extension = extension + self.input_data = '' + self.mode = mode + + def get_services_for_reader(self, tmpdir): + fs, filename = open_fs(tmpdir), 'input.' + self.extension + with fs.open(filename, self.mode) as fp: + fp.write(self.input_data) + return fs, filename, {'fs': fs} + + def get_services_for_writer(self, tmpdir): + fs, filename = open_fs(tmpdir), 'output.' + self.extension + return fs, filename, {'fs': fs} diff --git a/config/__init__.py b/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/io/test_csv.py b/tests/io/test_csv.py index 47e641b..dc6f71c 100644 --- a/tests/io/test_csv.py +++ b/tests/io/test_csv.py @@ -1,23 +1,21 @@ import pytest -from bonobo import Bag, CsvReader, CsvWriter, open_fs, settings +from bonobo import Bag, CsvReader, CsvWriter, settings from bonobo.constants import BEGIN, END from bonobo.execution.node import NodeExecutionContext -from bonobo.util.testing import CapturingNodeExecutionContext +from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester + +csv_tester = FilesystemTester('csv') +csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar' -def test_write_csv_to_file(tmpdir): - fs, filename = open_fs(tmpdir), 'output.csv' +def test_write_csv_to_file_arg0(tmpdir): + fs, filename, services = csv_tester.get_services_for_writer(tmpdir) - writer = CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0) - context = NodeExecutionContext(writer, services={'fs': fs}) - - context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END) - - context.start() - context.step() - context.step() - context.stop() + with NodeExecutionContext(CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context: + context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END) + context.step() + context.step() with fs.open(filename) as fp: assert fp.read() == 'foo\nbar\nbaz\n' @@ -26,19 +24,33 @@ def test_write_csv_to_file(tmpdir): getattr(context, 'file') -def test_read_csv_from_file(tmpdir): - fs, filename = open_fs(tmpdir), 'input.csv' - with fs.open(filename, 'w') as fp: - fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar') +@pytest.mark.parametrize('add_kwargs', ({}, { + 'ioformat': settings.IOFORMAT_KWARGS, +},)) +def test_write_csv_to_file_kwargs(tmpdir, add_kwargs): + fs, filename, services = csv_tester.get_services_for_writer(tmpdir) - reader = CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0) + with NodeExecutionContext(CsvWriter(path=filename, **add_kwargs), services=services) as context: + context.write(BEGIN, Bag(**{'foo': 'bar'}), Bag(**{'foo': 'baz', 'ignore': 'this'}), END) + context.step() + context.step() - context = CapturingNodeExecutionContext(reader, services={'fs': fs}) + with fs.open(filename) as fp: + assert fp.read() == 'foo\nbar\nbaz\n' - context.start() - context.write(BEGIN, Bag(), END) - context.step() - context.stop() + with pytest.raises(AttributeError): + getattr(context, 'file') + + +def test_read_csv_from_file_arg0(tmpdir): + fs, filename, services = csv_tester.get_services_for_reader(tmpdir) + + with CapturingNodeExecutionContext( + CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0), + services=services, + ) as context: + context.write(BEGIN, Bag(), END) + context.step() assert len(context.send.mock_calls) == 2 @@ -59,19 +71,15 @@ def test_read_csv_from_file(tmpdir): } -def test_read_csv_kwargs_output_formater(tmpdir): - fs, filename = open_fs(tmpdir), 'input.csv' - with fs.open(filename, 'w') as fp: - fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar') +def test_read_csv_from_file_kwargs(tmpdir): + fs, filename, services = csv_tester.get_services_for_reader(tmpdir) - reader = CsvReader(path=filename, delimiter=',') - - context = CapturingNodeExecutionContext(reader, services={'fs': fs}) - - context.start() - context.write(BEGIN, Bag(), END) - context.step() - context.stop() + with CapturingNodeExecutionContext( + CsvReader(path=filename, delimiter=','), + services=services, + ) as context: + context.write(BEGIN, Bag(), END) + context.step() assert len(context.send.mock_calls) == 2 diff --git a/tests/io/test_file.py b/tests/io/test_file.py index 1566b39..07a15eb 100644 --- a/tests/io/test_file.py +++ b/tests/io/test_file.py @@ -1,9 +1,22 @@ import pytest -from bonobo import Bag, FileReader, FileWriter, open_fs +from bonobo import Bag, FileReader, FileWriter from bonobo.constants import BEGIN, END from bonobo.execution.node import NodeExecutionContext -from bonobo.util.testing import CapturingNodeExecutionContext +from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester + +txt_tester = FilesystemTester('txt') +txt_tester.input_data = 'Hello\nWorld\n' + + +def test_file_writer_contextless(tmpdir): + fs, filename, services = txt_tester.get_services_for_writer(tmpdir) + + with FileWriter(path=filename).open(fs) as fp: + fp.write('Yosh!') + + with fs.open(filename) as fp: + assert fp.read() == 'Yosh!' @pytest.mark.parametrize( @@ -14,46 +27,23 @@ from bonobo.util.testing import CapturingNodeExecutionContext ] ) def test_file_writer_in_context(tmpdir, lines, output): - fs, filename = open_fs(tmpdir), 'output.txt' + fs, filename, services = txt_tester.get_services_for_writer(tmpdir) - writer = FileWriter(path=filename) - context = NodeExecutionContext(writer, services={'fs': fs}) - - context.start() - context.write(BEGIN, *map(Bag, lines), END) - for _ in range(len(lines)): - context.step() - context.stop() + with NodeExecutionContext(FileWriter(path=filename), services=services) as context: + context.write(BEGIN, *map(Bag, lines), END) + for _ in range(len(lines)): + context.step() with fs.open(filename) as fp: assert fp.read() == output -def test_file_writer_out_of_context(tmpdir): - fs, filename = open_fs(tmpdir), 'output.txt' +def test_file_reader(tmpdir): + fs, filename, services = txt_tester.get_services_for_reader(tmpdir) - writer = FileWriter(path=filename) - - with writer.open(fs) as fp: - fp.write('Yosh!') - - with fs.open(filename) as fp: - assert fp.read() == 'Yosh!' - - -def test_file_reader_in_context(tmpdir): - fs, filename = open_fs(tmpdir), 'input.txt' - - with fs.open(filename, 'w') as fp: - fp.write('Hello\nWorld\n') - - reader = FileReader(path=filename) - context = CapturingNodeExecutionContext(reader, services={'fs': fs}) - - context.start() - context.write(BEGIN, Bag(), END) - context.step() - context.stop() + with CapturingNodeExecutionContext(FileReader(path=filename), services=services) as context: + context.write(BEGIN, Bag(), END) + context.step() assert len(context.send.mock_calls) == 2 diff --git a/tests/io/test_json.py b/tests/io/test_json.py index 15d9e7e..75350ce 100644 --- a/tests/io/test_json.py +++ b/tests/io/test_json.py @@ -1,44 +1,48 @@ import pytest -from bonobo import Bag, JsonReader, JsonWriter, open_fs, settings +from bonobo import Bag, JsonReader, JsonWriter, settings from bonobo.constants import BEGIN, END from bonobo.execution.node import NodeExecutionContext -from bonobo.util.testing import CapturingNodeExecutionContext +from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester + +json_tester = FilesystemTester('json') +json_tester.input_data = '''[{"x": "foo"},{"x": "bar"}]''' -def test_write_json_to_file(tmpdir): - fs, filename = open_fs(tmpdir), 'output.json' +def test_write_json_arg0(tmpdir): + fs, filename, services = json_tester.get_services_for_writer(tmpdir) - writer = JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0) - context = NodeExecutionContext(writer, services={'fs': fs}) - - context.start() - context.write(BEGIN, Bag({'foo': 'bar'}), END) - context.step() - context.stop() + with NodeExecutionContext(JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context: + context.write(BEGIN, Bag({'foo': 'bar'}), END) + context.step() with fs.open(filename) as fp: assert fp.read() == '[{"foo": "bar"}]' - with pytest.raises(AttributeError): - getattr(context, 'file') - with pytest.raises(AttributeError): - getattr(context, 'first') +@pytest.mark.parametrize('add_kwargs', ({}, { + 'ioformat': settings.IOFORMAT_KWARGS, +}, )) +def test_write_json_kwargs(tmpdir, add_kwargs): + fs, filename, services = json_tester.get_services_for_writer(tmpdir) + + with NodeExecutionContext(JsonWriter(filename, **add_kwargs), services=services) as context: + context.write(BEGIN, Bag(**{'foo': 'bar'}), END) + context.step() + + with fs.open(filename) as fp: + assert fp.read() == '[{"foo": "bar"}]' -def test_read_json_from_file(tmpdir): - fs, filename = open_fs(tmpdir), 'input.json' - with fs.open(filename, 'w') as fp: - fp.write('[{"x": "foo"},{"x": "bar"}]') - reader = JsonReader(filename, ioformat=settings.IOFORMAT_ARG0) +def test_read_json_arg0(tmpdir): + fs, filename, services = json_tester.get_services_for_reader(tmpdir) - context = CapturingNodeExecutionContext(reader, services={'fs': fs}) - - context.start() - context.write(BEGIN, Bag(), END) - context.step() - context.stop() + with CapturingNodeExecutionContext( + JsonReader(filename, ioformat=settings.IOFORMAT_ARG0), + services=services, + ) as context: + context.write(BEGIN, Bag(), END) + context.step() assert len(context.send.mock_calls) == 2 diff --git a/tests/io/test_pickle.py b/tests/io/test_pickle.py index 1709b16..aff7796 100644 --- a/tests/io/test_pickle.py +++ b/tests/io/test_pickle.py @@ -2,24 +2,22 @@ import pickle import pytest -from bonobo import Bag, PickleReader, PickleWriter, open_fs, settings +from bonobo import Bag, PickleReader, PickleWriter, settings from bonobo.constants import BEGIN, END from bonobo.execution.node import NodeExecutionContext -from bonobo.util.testing import CapturingNodeExecutionContext +from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester + +pickle_tester = FilesystemTester('pkl', mode='wb') +pickle_tester.input_data = pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']]) def test_write_pickled_dict_to_file(tmpdir): - fs, filename = open_fs(tmpdir), 'output.pkl' + fs, filename, services = pickle_tester.get_services_for_writer(tmpdir) - writer = PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0) - context = NodeExecutionContext(writer, services={'fs': fs}) - - context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END) - - context.start() - context.step() - context.step() - context.stop() + with NodeExecutionContext(PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context: + context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END) + context.step() + context.step() with fs.open(filename, 'rb') as fp: assert pickle.loads(fp.read()) == {'foo': 'bar'} @@ -29,18 +27,13 @@ def test_write_pickled_dict_to_file(tmpdir): def test_read_pickled_list_from_file(tmpdir): - fs, filename = open_fs(tmpdir), 'input.pkl' - with fs.open(filename, 'wb') as fp: - fp.write(pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']])) + fs, filename, services = pickle_tester.get_services_for_reader(tmpdir) - reader = PickleReader(filename, ioformat=settings.IOFORMAT_ARG0) - - context = CapturingNodeExecutionContext(reader, services={'fs': fs}) - - context.start() - context.write(BEGIN, Bag(), END) - context.step() - context.stop() + with CapturingNodeExecutionContext( + PickleReader(filename, ioformat=settings.IOFORMAT_ARG0), services=services + ) as context: + context.write(BEGIN, Bag(), END) + context.step() assert len(context.send.mock_calls) == 2 From b1db4263d42838a45927857871743ec270ba734c Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 18 Jun 2017 21:30:19 +0200 Subject: [PATCH 14/24] Update dependencies. --- Makefile | 4 ++-- requirements-docker.txt | 5 +++-- requirements.txt | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f782b7e..881f553 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-06-10 15:15:34.093885 +# Updated at 2017-06-18 21:29:55.255815 PACKAGE ?= bonobo PYTHON ?= $(shell which python) @@ -18,7 +18,7 @@ SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build SPHINX_OPTIONS ?= SPHINX_SOURCEDIR ?= docs SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build -YAPF ?= $(PYTHON_DIRNAME)/yapf +YAPF ?= $(PYTHON) -m yapf YAPF_OPTIONS ?= -rip VERSION ?= $(shell git describe 2>/dev/null || echo dev) diff --git a/requirements-docker.txt b/requirements-docker.txt index b004174..dadce09 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -6,14 +6,15 @@ chardet==3.0.4 colorama==0.3.9 docker-pycreds==0.2.1 docker==2.3.0 -enum34==1.1.6 fs==2.0.4 idna==2.5 +packaging==16.8 pbr==3.0.1 psutil==5.2.2 +pyparsing==2.2.0 pytz==2017.2 requests==2.18.1 six==1.10.0 stevedore==1.23.0 urllib3==1.21.1 -websocket-client==0.40.0 +websocket-client==0.42.1 diff --git a/requirements.txt b/requirements.txt index 478879e..1b284bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ appdirs==1.4.3 certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9 -enum34==1.1.6 fs==2.0.4 idna==2.5 packaging==16.8 From 1d273767851e75e3cc92096515410e92594831d9 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 18 Jun 2017 21:35:30 +0200 Subject: [PATCH 15/24] release: 0.4.2 --- bonobo/_version.py | 2 +- bonobo/strategies/executor.py | 2 ++ docs/changelog.rst | 17 +++++++++++++++++ tests/io/test_csv.py | 10 +++++----- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/bonobo/_version.py b/bonobo/_version.py index f0ede3d..a987347 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = '0.4.1' +__version__ = '0.4.2' diff --git a/bonobo/strategies/executor.py b/bonobo/strategies/executor.py index d2cdcbe..44d206e 100644 --- a/bonobo/strategies/executor.py +++ b/bonobo/strategies/executor.py @@ -28,6 +28,7 @@ class ExecutorStrategy(Strategy): futures = [] for plugin_context in context.plugins: + def _runner(plugin_context=plugin_context): with plugin_context: try: @@ -38,6 +39,7 @@ class ExecutorStrategy(Strategy): futures.append(executor.submit(_runner)) for node_context in context.nodes: + def _runner(node_context=node_context): try: node_context.start() diff --git a/docs/changelog.rst b/docs/changelog.rst index f447fc9..ebd2fac 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,23 @@ Changelog ========= +v.0.4.2 - 18 june 2017 +:::::::::::::::::::::: + +* [config] Implements a "requires()" service injection decorator for functions (api may change). +* [core] Execution contexts are now context managers. +* [fs] adds a defaut to current working directory in open_fs(...). +* [logging] Adds logging alias for easier imports. +* [stdlib] Fix I/O related nodes (especially json), there were bad bugs with ioformat. + +Dependency updates +------------------ + +* Update bonobo-docker from 0.2.6 to 0.2.8 +* Update dependencies. +* Update fs from 2.0.3 to 2.0.4 +* Update requests from 2.17.3 to 2.18.1 + v.0.4.0 - 10 june 2017 :::::::::::::::::::::: diff --git a/tests/io/test_csv.py b/tests/io/test_csv.py index dc6f71c..9a9480c 100644 --- a/tests/io/test_csv.py +++ b/tests/io/test_csv.py @@ -26,7 +26,7 @@ def test_write_csv_to_file_arg0(tmpdir): @pytest.mark.parametrize('add_kwargs', ({}, { 'ioformat': settings.IOFORMAT_KWARGS, -},)) +}, )) def test_write_csv_to_file_kwargs(tmpdir, add_kwargs): fs, filename, services = csv_tester.get_services_for_writer(tmpdir) @@ -46,8 +46,8 @@ def test_read_csv_from_file_arg0(tmpdir): fs, filename, services = csv_tester.get_services_for_reader(tmpdir) with CapturingNodeExecutionContext( - CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0), - services=services, + CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0), + services=services, ) as context: context.write(BEGIN, Bag(), END) context.step() @@ -75,8 +75,8 @@ def test_read_csv_from_file_kwargs(tmpdir): fs, filename, services = csv_tester.get_services_for_reader(tmpdir) with CapturingNodeExecutionContext( - CsvReader(path=filename, delimiter=','), - services=services, + CsvReader(path=filename, delimiter=','), + services=services, ) as context: context.write(BEGIN, Bag(), END) context.step() From d0345880913ab7de1f3bc5c014be1a18d008d9a3 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Tue, 4 Jul 2017 10:51:33 +0200 Subject: [PATCH 16/24] Update dependencies. --- Makefile | 2 +- requirements-dev.txt | 2 +- requirements-docker.txt | 6 +++--- requirements-jupyter.txt | 4 ++-- requirements.txt | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 881f553..10094af 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-06-18 21:29:55.255815 +# Updated at 2017-07-04 10:50:55.775681 PACKAGE ?= bonobo PYTHON ?= $(shell which python) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6c33b15..55ba71c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -27,7 +27,7 @@ pytz==2017.2 requests==2.18.1 six==1.10.0 snowballstemmer==1.2.1 -sphinx==1.6.2 +sphinx==1.6.3 sphinxcontrib-websupport==1.0.1 termcolor==1.1.0 urllib3==1.21.1 diff --git a/requirements-docker.txt b/requirements-docker.txt index dadce09..0a39353 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,6 +1,6 @@ -e .[docker] appdirs==1.4.3 -bonobo-docker==0.2.8 +bonobo-docker==0.2.9 certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9 @@ -9,7 +9,7 @@ docker==2.3.0 fs==2.0.4 idna==2.5 packaging==16.8 -pbr==3.0.1 +pbr==3.1.1 psutil==5.2.2 pyparsing==2.2.0 pytz==2017.2 @@ -17,4 +17,4 @@ requests==2.18.1 six==1.10.0 stevedore==1.23.0 urllib3==1.21.1 -websocket-client==0.42.1 +websocket-client==0.44.0 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 7db4a67..d6a6fdb 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -11,7 +11,7 @@ ipywidgets==6.0.0 jedi==0.10.2 jinja2==2.9.6 jsonschema==2.6.0 -jupyter-client==5.0.1 +jupyter-client==5.1.0 jupyter-console==5.1.0 jupyter-core==4.3.0 jupyter==1.0.0 @@ -24,7 +24,7 @@ pandocfilters==1.4.1 pexpect==4.2.1 pickleshare==0.7.4 prompt-toolkit==1.0.14 -ptyprocess==0.5.1 +ptyprocess==0.5.2 pygments==2.2.0 python-dateutil==2.6.0 pyzmq==16.0.2 diff --git a/requirements.txt b/requirements.txt index 1b284bc..093a6a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ colorama==0.3.9 fs==2.0.4 idna==2.5 packaging==16.8 -pbr==3.0.1 +pbr==3.1.1 psutil==5.2.2 pyparsing==2.2.0 pytz==2017.2 From 896bc79abe443075cfdb27c56bb64be5dad0215f Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Tue, 4 Jul 2017 10:55:45 +0200 Subject: [PATCH 17/24] release: 0.4.3 --- bonobo/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bonobo/_version.py b/bonobo/_version.py index a987347..908c0bb 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = '0.4.2' +__version__ = '0.4.3' From 5062221e7866e707cd2c42073f93d929c2059ef5 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 5 Jul 2017 11:15:03 +0200 Subject: [PATCH 18/24] [config] Refactoring of configurables, allowing partially configured objects. Configurables did not allow more than one "method" option, and mixed scenarios (options+methods+...) were sometimes flaky, forcing the user to know what order was the right one. Now, all options work the same, sharing the same "order" namespace. Backward incompatible change: Options are now required by default, unless a default is provided. Also adds a few candies for debugging/testing, found in the bonobo.util.inspect module. --- bonobo/_api.py | 3 +- bonobo/config/__init__.py | 2 +- bonobo/config/configurables.py | 200 +++++++++++++++++++-------- bonobo/config/options.py | 60 +++++--- bonobo/config/processors.py | 10 +- bonobo/config/services.py | 4 +- bonobo/ext/opendatasoft.py | 4 +- bonobo/nodes/__init__.py | 11 +- bonobo/nodes/basics.py | 14 +- bonobo/nodes/io/csv.py | 2 +- bonobo/nodes/io/pickle.py | 2 +- bonobo/nodes/throttle.py | 55 ++++++++ bonobo/settings.py | 2 +- bonobo/util/collections.py | 6 + bonobo/util/inspect.py | 114 +++++++++++++++ tests/config/test_configurables.py | 57 +++++++- tests/config/test_methods.py | 80 ++++++++--- tests/config/test_methods_partial.py | 66 +++++++++ tests/test_basics.py | 1 + 19 files changed, 573 insertions(+), 120 deletions(-) create mode 100644 bonobo/nodes/throttle.py create mode 100644 bonobo/util/collections.py create mode 100644 bonobo/util/inspect.py create mode 100644 tests/config/test_methods_partial.py diff --git a/bonobo/_api.py b/bonobo/_api.py index cf28a33..89b6d4c 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -1,6 +1,6 @@ from bonobo.structs import Bag, Graph, Token from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \ - PrettyPrinter, PickleWriter, PickleReader, Tee, count, identity, noop, pprint + PrettyPrinter, PickleWriter, PickleReader, RateLimited, Tee, count, identity, noop, pprint from bonobo.strategies import create_strategy from bonobo.util.objects import get_name @@ -104,6 +104,7 @@ register_api_group( PrettyPrinter, PickleReader, PickleWriter, + RateLimited, Tee, count, identity, diff --git a/bonobo/config/__init__.py b/bonobo/config/__init__.py index 08be544..6a4247e 100644 --- a/bonobo/config/__init__.py +++ b/bonobo/config/__init__.py @@ -3,7 +3,7 @@ from bonobo.config.options import Method, Option from bonobo.config.processors import ContextProcessor from bonobo.config.services import Container, Exclusive, Service, requires -# bonobo.config public programming interface +# Bonobo's Config API __all__ = [ 'Configurable', 'Container', diff --git a/bonobo/config/configurables.py b/bonobo/config/configurables.py index 43cb8c2..01db9e0 100644 --- a/bonobo/config/configurables.py +++ b/bonobo/config/configurables.py @@ -1,12 +1,14 @@ -from bonobo.config.options import Method, Option -from bonobo.config.processors import ContextProcessor -from bonobo.errors import ConfigurationError, AbstractError +from bonobo.util.inspect import isoption, iscontextprocessor +from bonobo.errors import AbstractError +from bonobo.util.collections import sortedlist __all__ = [ 'Configurable', 'Option', ] +get_creation_counter = lambda v: v._creation_counter + class ConfigurableMeta(type): """ @@ -15,36 +17,77 @@ class ConfigurableMeta(type): def __init__(cls, what, bases=None, dict=None): super().__init__(what, bases, dict) - cls.__options__ = {} - cls.__positional_options__ = [] - cls.__processors__ = [] - cls.__wrappable__ = None + + cls.__processors = sortedlist() + cls.__methods = sortedlist() + cls.__options = sortedlist() + cls.__names = set() + + # cls.__kwoptions = [] for typ in cls.__mro__: - for name, value in typ.__dict__.items(): - if isinstance(value, Option): - if isinstance(value, ContextProcessor): - cls.__processors__.append(value) - else: - if not value.name: - value.name = name + for name, value in filter(lambda x: isoption(x[1]), typ.__dict__.items()): + if iscontextprocessor(value): + cls.__processors.insort((value._creation_counter, value)) + continue - if isinstance(value, Method): - if cls.__wrappable__: - raise ConfigurationError( - 'Cannot define more than one "Method" option in a configurable. That may change in the future.' - ) - cls.__wrappable__ = name + if not value.name: + value.name = name - if not name in cls.__options__: - cls.__options__[name] = value + if not name in cls.__names: + cls.__names.add(name) + cls.__options.insort((not value.positional, value._creation_counter, name, value)) - if value.positional: - cls.__positional_options__.append(name) + @property + def __options__(cls): + return ((name, option) for _, _, name, option in cls.__options) - # This can be done before, more efficiently. Not so bad neither as this is only done at type() creation time - # (aka class Xxx(...) time) and there should not be hundreds of processors. Still not very elegant. - cls.__processors__ = sorted(cls.__processors__, key=lambda v: v._creation_counter) + @property + def __options_dict__(cls): + return dict(cls.__options__) + + @property + def __processors__(cls): + return (processor for _, processor in cls.__processors) + + def __repr__(self): + return ' '.join(('= position + 1 else None + position += 1 + + return self.__options_values + + def __getattr__(self, item): + _dict = self.func.__options_dict__ + if item in _dict: + return _dict[item].__get__(self, self.func) + return getattr(self.func, item) class Configurable(metaclass=ConfigurableMeta): @@ -54,61 +97,108 @@ class Configurable(metaclass=ConfigurableMeta): """ - def __new__(cls, *args, **kwargs): - if cls.__wrappable__ and len(args) == 1 and hasattr(args[0], '__call__'): - return type(args[0].__name__, (cls, ), {cls.__wrappable__: args[0]}) + def __new__(cls, *args, _final=False, **kwargs): + """ + Custom instance builder. If not all options are fulfilled, will return a :class:`PartiallyConfigured` instance + which is just a :class:`functools.partial` object that behaves like a :class:`Configurable` instance. - return super(Configurable, cls).__new__(cls) - - def __init__(self, *args, **kwargs): - super().__init__() - - # initialize option's value dictionary, used by descriptor implementation (see Option). - self.__options_values__ = {} + The special `_final` argument can be used to force final instance to be created, or an error raised if options + are missing. + :param args: + :param _final: bool + :param kwargs: + :return: Configurable or PartiallyConfigured + """ + options = tuple(cls.__options__) # compute missing options, given the kwargs. missing = set() - for name, option in type(self).__options__.items(): + for name, option in options: if option.required and not option.name in kwargs: missing.add(name) # transform positional arguments in keyword arguments if possible. position = 0 - for positional_option in self.__positional_options__: - if len(args) <= position: - break - kwargs[positional_option] = args[position] - position += 1 - if positional_option in missing: - missing.remove(positional_option) + for name, option in options: + if not option.positional: + break # option orders make all positional options first, job done. - # complain if there are still missing options. - if len(missing): - raise TypeError( - '{}() missing {} required option{}: {}.'.format( - type(self).__name__, - len(missing), 's' if len(missing) > 1 else '', ', '.join(map(repr, sorted(missing))) - ) - ) + if not isoption(getattr(cls, name)): + missing.remove(name) + continue + + if len(args) <= position: + break # no more positional arguments given. + + position += 1 + if name in missing: + missing.remove(name) # complain if there is more options than possible. - extraneous = set(kwargs.keys()) - set(type(self).__options__.keys()) + extraneous = set(kwargs.keys()) - (set(next(zip(*options))) if len(options) else set()) if len(extraneous): raise TypeError( '{}() got {} unexpected option{}: {}.'.format( - type(self).__name__, + cls.__name__, len(extraneous), 's' if len(extraneous) > 1 else '', ', '.join(map(repr, sorted(extraneous))) ) ) + # missing options? we'll return a partial instance to finish the work later, unless we're required to be + # "final". + if len(missing): + if _final: + raise TypeError( + '{}() missing {} required option{}: {}.'.format( + cls.__name__, + len(missing), 's' if len(missing) > 1 else '', ', '.join(map(repr, sorted(missing))) + ) + ) + return PartiallyConfigured(cls, *args, **kwargs) + + return super(Configurable, cls).__new__(cls) + + def __init__(self, *args, **kwargs): + # initialize option's value dictionary, used by descriptor implementation (see Option). + self._options_values = { + **kwargs + } + # set option values. for name, value in kwargs.items(): setattr(self, name, value) + position = 0 + for name, option in self.__options__: + if not option.positional: + break # option orders make all positional options first + + # value was overriden? Skip. + maybe_value = getattr(type(self), name) + if not isoption(maybe_value): + continue + + if len(args) <= position: + break + + if name in self._options_values: + raise ValueError('Already got a value for option {}'.format(name)) + + setattr(self, name, args[position]) + position += 1 + def __call__(self, *args, **kwargs): """ You can implement a configurable callable behaviour by implemenenting the call(...) method. Of course, it is also backward compatible with legacy __call__ override. """ return self.call(*args, **kwargs) + @property + def __options__(self): + return type(self).__options__ + + @property + def __processors__(self): + return type(self).__processors__ + def call(self, *args, **kwargs): raise AbstractError('Not implemented.') diff --git a/bonobo/config/options.py b/bonobo/config/options.py index 51f4a20..82604fb 100644 --- a/bonobo/config/options.py +++ b/bonobo/config/options.py @@ -1,3 +1,6 @@ +from bonobo.util.inspect import istype + + class Option: """ An Option is a descriptor for Configurable's parameters. @@ -14,7 +17,9 @@ class Option: If an option is required, an error will be raised if no value is provided (at runtime). If it is not, option will have the default value if user does not override it at runtime. - (default: False) + Ignored if a default is provided, meaning that the option cannot be required. + + (default: True) .. attribute:: positional @@ -48,10 +53,10 @@ class Option: _creation_counter = 0 - def __init__(self, type=None, *, required=False, positional=False, default=None): + def __init__(self, type=None, *, required=True, positional=False, default=None): self.name = None self.type = type - self.required = required + self.required = required if default is None else False self.positional = positional self.default = default @@ -60,12 +65,27 @@ class Option: Option._creation_counter += 1 def __get__(self, inst, typ): - if not self.name in inst.__options_values__: - inst.__options_values__[self.name] = self.get_default() - return inst.__options_values__[self.name] + # XXX If we call this on the type, then either return overriden value or ... ??? + if inst is None: + return vars(type).get(self.name, self) + + if not self.name in inst._options_values: + inst._options_values[self.name] = self.get_default() + + return inst._options_values[self.name] def __set__(self, inst, value): - inst.__options_values__[self.name] = self.clean(value) + inst._options_values[self.name] = self.clean(value) + + def __repr__(self): + return '<{positional}{typename} {type}{name} default={default!r}{required}>'.format( + typename=type(self).__name__, + type='({})'.format(self.type) if istype(self.type) else '', + name=self.name, + positional='*' if self.positional else '**', + default=self.default, + required=' (required)' if self.required else '', + ) def clean(self, value): return self.type(value) if self.type else value @@ -105,20 +125,18 @@ class Method(Option): """ - def __init__(self): - super().__init__(None, required=False) - - def __get__(self, inst, typ): - if not self.name in inst.__options_values__: - inst.__options_values__[self.name] = getattr(inst, self.name) - return inst.__options_values__[self.name] + def __init__(self, *, required=True, positional=True): + super().__init__(None, required=required, positional=positional) def __set__(self, inst, value): - if isinstance(value, str): - raise ValueError('should be callable') - inst.__options_values__[self.name] = self.type(value) if self.type else value - - def clean(self, value): if not hasattr(value, '__call__'): - raise ValueError('{} value must be callable.'.format(type(self).__name__)) - return value + raise TypeError('Option of type {!r} is expecting a callable value, got {!r} object (which is not).'.format( + type(self).__name__, type(value).__name__)) + inst._options_values[self.name] = self.type(value) if self.type else value + + def __call__(self, *args, **kwargs): + # only here to trick IDEs into thinking this is callable. + raise NotImplementedError('You cannot call the descriptor') + + + diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py index d441b6e..27f8703 100644 --- a/bonobo/config/processors.py +++ b/bonobo/config/processors.py @@ -74,8 +74,7 @@ class ContextCurrifier: def __init__(self, wrapped, *initial_context): self.wrapped = wrapped self.context = tuple(initial_context) - self._stack = [] - self._stack_values = [] + self._stack, self._stack_values = None, None def __iter__(self): yield from self.wrapped @@ -86,8 +85,10 @@ class ContextCurrifier: return self.wrapped(*self.context, *args, **kwargs) def setup(self, *context): - if len(self._stack): + if self._stack is not None: raise RuntimeError('Cannot setup context currification twice.') + + self._stack, self._stack_values = list(), list() for processor in resolve_processors(self.wrapped): _processed = processor(self.wrapped, *context, *self.context) _append_to_context = next(_processed) @@ -97,7 +98,7 @@ class ContextCurrifier: self._stack.append(_processed) def teardown(self): - while len(self._stack): + while self._stack: processor = self._stack.pop() try: # todo yield from ? how to ? @@ -108,6 +109,7 @@ class ContextCurrifier: else: # No error ? We should have had StopIteration ... raise RuntimeError('Context processors should not yield more than once.') + self._stack, self._stack_values = None, None @contextmanager def as_contextmanager(self, *context): diff --git a/bonobo/config/services.py b/bonobo/config/services.py index d792175..1fe066d 100644 --- a/bonobo/config/services.py +++ b/bonobo/config/services.py @@ -53,7 +53,7 @@ class Service(Option): super().__init__(str, required=False, default=name) def __set__(self, inst, value): - inst.__options_values__[self.name] = validate_service_name(value) + inst._options_values[self.name] = validate_service_name(value) def resolve(self, inst, services): try: @@ -75,7 +75,7 @@ class Container(dict): def args_for(self, mixed): try: - options = mixed.__options__ + options = dict(mixed.__options__) except AttributeError: options = {} diff --git a/bonobo/ext/opendatasoft.py b/bonobo/ext/opendatasoft.py index 4be3134..2dc54c0 100644 --- a/bonobo/ext/opendatasoft.py +++ b/bonobo/ext/opendatasoft.py @@ -13,13 +13,13 @@ def path_str(path): class OpenDataSoftAPI(Configurable): - dataset = Option(str, required=True) + dataset = Option(str, positional=True) endpoint = Option(str, default='{scheme}://{netloc}{path}') scheme = Option(str, default='https') netloc = Option(str, default='data.opendatasoft.com') path = Option(path_str, default='/api/records/1.0/search/') rows = Option(int, default=500) - limit = Option(int, default=None) + limit = Option(int, required=False) timezone = Option(str, default='Europe/Paris') kwargs = Option(dict, default=dict) diff --git a/bonobo/nodes/__init__.py b/bonobo/nodes/__init__.py index c25b580..2cdd1e9 100644 --- a/bonobo/nodes/__init__.py +++ b/bonobo/nodes/__init__.py @@ -1,9 +1,8 @@ -from bonobo.nodes.io import __all__ as _all_io -from bonobo.nodes.io import * - -from bonobo.nodes.basics import __all__ as _all_basics from bonobo.nodes.basics import * - +from bonobo.nodes.basics import __all__ as _all_basics from bonobo.nodes.filter import Filter +from bonobo.nodes.io import * +from bonobo.nodes.io import __all__ as _all_io +from bonobo.nodes.throttle import RateLimited -__all__ = _all_basics + _all_io + ['Filter'] +__all__ = _all_basics + _all_io + ['Filter', 'RateLimited'] diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index c21757a..c1ead61 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -1,16 +1,16 @@ import functools -from pprint import pprint as _pprint - import itertools + from colorama import Fore, Style from bonobo import settings from bonobo.config import Configurable, Option from bonobo.config.processors import ContextProcessor +from bonobo.constants import NOT_MODIFIED from bonobo.structs.bags import Bag +from bonobo.util.compat import deprecated from bonobo.util.objects import ValueHolder from bonobo.util.term import CLEAR_EOL -from bonobo.constants import NOT_MODIFIED __all__ = [ 'identity', @@ -87,8 +87,12 @@ class PrettyPrinter(Configurable): ) -pprint = PrettyPrinter() -pprint.__name__ = 'pprint' +_pprint = PrettyPrinter() + + +@deprecated +def pprint(*args, **kwargs): + return _pprint(*args, **kwargs) def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True): diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py index ae68bd0..75fffe8 100644 --- a/bonobo/nodes/io/csv.py +++ b/bonobo/nodes/io/csv.py @@ -26,7 +26,7 @@ class CsvHandler(FileHandler): """ delimiter = Option(str, default=';') quotechar = Option(str, default='"') - headers = Option(tuple) + headers = Option(tuple, required=False) class CsvReader(IOFormatEnabled, FileReader, CsvHandler): diff --git a/bonobo/nodes/io/pickle.py b/bonobo/nodes/io/pickle.py index e94f94a..d9da55f 100644 --- a/bonobo/nodes/io/pickle.py +++ b/bonobo/nodes/io/pickle.py @@ -17,7 +17,7 @@ class PickleHandler(FileHandler): """ - item_names = Option(tuple) + item_names = Option(tuple, required=False) class PickleReader(IOFormatEnabled, FileReader, PickleHandler): diff --git a/bonobo/nodes/throttle.py b/bonobo/nodes/throttle.py new file mode 100644 index 0000000..2f08cd3 --- /dev/null +++ b/bonobo/nodes/throttle.py @@ -0,0 +1,55 @@ +import threading +import time + +from bonobo.config import Configurable, ContextProcessor, Method, Option + + +class RateLimitBucket(threading.Thread): + daemon = True + + @property + def stopped(self): + return self._stop_event.is_set() + + def __init__(self, initial=1, period=1, amount=1): + super(RateLimitBucket, self).__init__() + self.semaphore = threading.BoundedSemaphore(initial) + self.amount = amount + self.period = period + + self._stop_event = threading.Event() + + def stop(self): + self._stop_event.set() + + def run(self): + while not self.stopped: + time.sleep(self.period) + for _ in range(self.amount): + self.semaphore.release() + + def wait(self): + return self.semaphore.acquire() + + +class RateLimited(Configurable): + handler = Method() + + initial = Option(int, positional=True, default=1) + period = Option(int, positional=True, default=1) + amount = Option(int, positional=True, default=1) + + @ContextProcessor + def bucket(self, context): + print(context) + bucket = RateLimitBucket(self.initial, self.amount, self.period) + bucket.start() + print(bucket) + yield bucket + bucket.stop() + bucket.join() + + def call(self, bucket, *args, **kwargs): + print(bucket, args, kwargs) + bucket.wait() + return self.handler(*args, **kwargs) diff --git a/bonobo/settings.py b/bonobo/settings.py index e0e5289..8e8a780 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -27,7 +27,7 @@ class Setting: self.validator = None def __repr__(self): - return ''.format(self.name, self.value) + return ''.format(self.name, self.get()) def set(self, value): if self.validator and not self.validator(value): diff --git a/bonobo/util/collections.py b/bonobo/util/collections.py new file mode 100644 index 0000000..b97630a --- /dev/null +++ b/bonobo/util/collections.py @@ -0,0 +1,6 @@ +import bisect + + +class sortedlist(list): + def insort(self, x): + bisect.insort(self, x) \ No newline at end of file diff --git a/bonobo/util/inspect.py b/bonobo/util/inspect.py new file mode 100644 index 0000000..72fcc7e --- /dev/null +++ b/bonobo/util/inspect.py @@ -0,0 +1,114 @@ +from collections import namedtuple + + +def isconfigurabletype(mixed): + """ + Check if the given argument is an instance of :class:`bonobo.config.ConfigurableMeta`, meaning it has all the + plumbery necessary to build :class:`bonobo.config.Configurable`-like instances. + + :param mixed: + :return: bool + """ + from bonobo.config.configurables import ConfigurableMeta + return isinstance(mixed, ConfigurableMeta) + + +def isconfigurable(mixed): + """ + Check if the given argument is an instance of :class:`bonobo.config.Configurable`. + + :param mixed: + :return: bool + """ + from bonobo.config.configurables import Configurable + return isinstance(mixed, Configurable) + + +def isoption(mixed): + """ + Check if the given argument is an instance of :class:`bonobo.config.Option`. + + :param mixed: + :return: bool + """ + + from bonobo.config.options import Option + return isinstance(mixed, Option) + + +def ismethod(mixed): + """ + Check if the given argument is an instance of :class:`bonobo.config.Method`. + + :param mixed: + :return: bool + """ + from bonobo.config.options import Method + return isinstance(mixed, Method) + + +def iscontextprocessor(x): + """ + Check if the given argument is an instance of :class:`bonobo.config.ContextProcessor`. + + :param mixed: + :return: bool + """ + from bonobo.config.processors import ContextProcessor + return isinstance(x, ContextProcessor) + + +def istype(mixed): + """ + Check if the given argument is a type object. + + :param mixed: + :return: bool + """ + return isinstance(mixed, type) + + +ConfigurableInspection = namedtuple('ConfigurableInspection', + [ + 'type', + 'instance', + 'options', + 'processors', + 'partial', + ]) + +ConfigurableInspection.__enter__ = lambda self: self +ConfigurableInspection.__exit__ = lambda *exc_details: None + + +def inspect_node(mixed, *, _partial=None): + """ + If the given argument is somehow a :class:`bonobo.config.Configurable` object (either a subclass, an instance, or + a partially configured instance), then it will return a :class:`ConfigurableInspection` namedtuple, used to inspect + the configurable metadata (options). If you want to get the option values, you don't need this, it is only usefull + to perform introspection on a configurable. + + If it's not looking like a configurable, it will raise a :class:`TypeError`. + + :param mixed: + :return: ConfigurableInspection + + :raise: TypeError + """ + if isconfigurabletype(mixed): + inst, typ = None, mixed + elif isconfigurable(mixed): + inst, typ = mixed, type(mixed) + elif hasattr(mixed, 'func'): + return inspect_node(mixed.func, _partial=(mixed.args, mixed.keywords)) + else: + raise TypeError( + 'Not a Configurable, nor a Configurable instance and not even a partially configured Configurable. Check your inputs.') + + return ConfigurableInspection( + typ, + inst, + list(typ.__options__), + list(typ.__processors__), + _partial, + ) diff --git a/tests/config/test_configurables.py b/tests/config/test_configurables.py index 178c188..f1c5387 100644 --- a/tests/config/test_configurables.py +++ b/tests/config/test_configurables.py @@ -2,12 +2,17 @@ import pytest from bonobo.config.configurables import Configurable from bonobo.config.options import Option +from bonobo.util.inspect import inspect_node + + +class NoOptConfigurable(Configurable): + pass class MyConfigurable(Configurable): - required_str = Option(str, required=True) + required_str = Option(str) default_str = Option(str, default='foo') - integer = Option(int) + integer = Option(int, required=False) class MyHarderConfigurable(MyConfigurable): @@ -25,14 +30,20 @@ class MyConfigurableUsingPositionalOptions(MyConfigurable): def test_missing_required_option_error(): + with inspect_node(MyConfigurable()) as ni: + assert ni.partial + with pytest.raises(TypeError) as exc: - MyConfigurable() + MyConfigurable(_final=True) assert exc.match('missing 1 required option:') def test_missing_required_options_error(): + with inspect_node(MyHarderConfigurable()) as ni: + assert ni.partial + with pytest.raises(TypeError) as exc: - MyHarderConfigurable() + MyHarderConfigurable(_final=True) assert exc.match('missing 2 required options:') @@ -50,6 +61,10 @@ def test_extraneous_options_error(): def test_defaults(): o = MyConfigurable(required_str='hello') + + with inspect_node(o) as ni: + assert not ni.partial + assert o.required_str == 'hello' assert o.default_str == 'foo' assert o.integer == None @@ -57,6 +72,10 @@ def test_defaults(): def test_str_type_factory(): o = MyConfigurable(required_str=42) + + with inspect_node(o) as ni: + assert not ni.partial + assert o.required_str == '42' assert o.default_str == 'foo' assert o.integer == None @@ -64,6 +83,10 @@ def test_str_type_factory(): def test_int_type_factory(): o = MyConfigurable(required_str='yo', default_str='bar', integer='42') + + with inspect_node(o) as ni: + assert not ni.partial + assert o.required_str == 'yo' assert o.default_str == 'bar' assert o.integer == 42 @@ -71,6 +94,10 @@ def test_int_type_factory(): def test_bool_type_factory(): o = MyHarderConfigurable(required_str='yes', also_required='True') + + with inspect_node(o) as ni: + assert not ni.partial + assert o.required_str == 'yes' assert o.default_str == 'foo' assert o.integer == None @@ -79,6 +106,10 @@ def test_bool_type_factory(): def test_option_resolution_order(): o = MyBetterConfigurable() + + with inspect_node(o) as ni: + assert not ni.partial + assert o.required_str == 'kaboom' assert o.default_str == 'foo' assert o.integer == None @@ -86,3 +117,21 @@ def test_option_resolution_order(): def test_option_positional(): o = MyConfigurableUsingPositionalOptions('1', '2', '3', required_str='hello') + + with inspect_node(o) as ni: + assert not ni.partial + + assert o.first == '1' + assert o.second == '2' + assert o.third == '3' + assert o.required_str == 'hello' + assert o.default_str == 'foo' + assert o.integer is None + + +def test_no_opt_configurable(): + o = NoOptConfigurable() + + with inspect_node(o) as ni: + assert not ni.partial + diff --git a/tests/config/test_methods.py b/tests/config/test_methods.py index 3a5f6a3..a4e4ebb 100644 --- a/tests/config/test_methods.py +++ b/tests/config/test_methods.py @@ -1,7 +1,5 @@ -import pytest - from bonobo.config import Configurable, Method, Option -from bonobo.errors import ConfigurationError +from bonobo.util.inspect import inspect_node class MethodBasedConfigurable(Configurable): @@ -13,22 +11,56 @@ class MethodBasedConfigurable(Configurable): self.handler(*args, **kwargs) -def test_one_wrapper_only(): - with pytest.raises(ConfigurationError): +def test_multiple_wrapper_suppored(): + class TwoMethods(Configurable): + h1 = Method(required=True) + h2 = Method(required=True) - class TwoMethods(Configurable): - h1 = Method() - h2 = Method() + with inspect_node(TwoMethods) as ci: + assert ci.type == TwoMethods + assert not ci.instance + assert len(ci.options) == 2 + assert not len(ci.processors) + assert not ci.partial + + @TwoMethods + def OneMethod(): + pass + + with inspect_node(OneMethod) as ci: + assert ci.type == TwoMethods + assert not ci.instance + assert len(ci.options) == 2 + assert not len(ci.processors) + assert ci.partial + + @OneMethod + def transformation(): + pass + + with inspect_node(transformation) as ci: + assert ci.type == TwoMethods + assert ci.instance + assert len(ci.options) == 2 + assert not len(ci.processors) + assert not ci.partial def test_define_with_decorator(): calls = [] - @MethodBasedConfigurable - def Concrete(self, *args, **kwargs): - calls.append((args, kwargs, )) + def my_handler(*args, **kwargs): + calls.append((args, kwargs,)) + + Concrete = MethodBasedConfigurable(my_handler) assert callable(Concrete.handler) + assert Concrete.handler == my_handler + + with inspect_node(Concrete) as ci: + assert ci.type == MethodBasedConfigurable + assert ci.partial + t = Concrete('foo', bar='baz') assert callable(t.handler) @@ -37,13 +69,29 @@ def test_define_with_decorator(): assert len(calls) == 1 +def test_late_binding_method_decoration(): + calls = [] + + @MethodBasedConfigurable(foo='foo') + def Concrete(*args, **kwargs): + calls.append((args, kwargs,)) + + assert callable(Concrete.handler) + t = Concrete(bar='baz') + + assert callable(t.handler) + assert len(calls) == 0 + t() + assert len(calls) == 1 + + def test_define_with_argument(): calls = [] def concrete_handler(*args, **kwargs): - calls.append((args, kwargs, )) + calls.append((args, kwargs,)) - t = MethodBasedConfigurable('foo', bar='baz', handler=concrete_handler) + t = MethodBasedConfigurable(concrete_handler, 'foo', bar='baz') assert callable(t.handler) assert len(calls) == 0 t() @@ -55,7 +103,7 @@ def test_define_with_inheritance(): class Inheriting(MethodBasedConfigurable): def handler(self, *args, **kwargs): - calls.append((args, kwargs, )) + calls.append((args, kwargs,)) t = Inheriting('foo', bar='baz') assert callable(t.handler) @@ -71,8 +119,8 @@ def test_inheritance_then_decorate(): pass @Inheriting - def Concrete(self, *args, **kwargs): - calls.append((args, kwargs, )) + def Concrete(*args, **kwargs): + calls.append((args, kwargs,)) assert callable(Concrete.handler) t = Concrete('foo', bar='baz') diff --git a/tests/config/test_methods_partial.py b/tests/config/test_methods_partial.py new file mode 100644 index 0000000..fdb1111 --- /dev/null +++ b/tests/config/test_methods_partial.py @@ -0,0 +1,66 @@ +from unittest.mock import MagicMock + +from bonobo.config import Configurable, ContextProcessor, Method, Option +from bonobo.util.inspect import inspect_node + + +class Bobby(Configurable): + handler = Method() + handler2 = Method() + foo = Option(positional=True) + bar = Option(required=False) + + @ContextProcessor + def think(self, context): + yield 'different' + + def call(self, think, *args, **kwargs): + self.handler('1', *args, **kwargs) + self.handler2('2', *args, **kwargs) + + +def test_partial(): + C = Bobby + + # inspect the configurable class + with inspect_node(C) as ci: + assert ci.type == Bobby + assert not ci.instance + assert len(ci.options) == 4 + assert len(ci.processors) == 1 + assert not ci.partial + + # instanciate a partial instance ... + f1 = MagicMock() + C = C(f1) + + with inspect_node(C) as ci: + assert ci.type == Bobby + assert not ci.instance + assert len(ci.options) == 4 + assert len(ci.processors) == 1 + assert ci.partial + assert ci.partial[0] == (f1,) + assert not len(ci.partial[1]) + + # instanciate a more complete partial instance ... + f2 = MagicMock() + C = C(f2) + + with inspect_node(C) as ci: + assert ci.type == Bobby + assert not ci.instance + assert len(ci.options) == 4 + assert len(ci.processors) == 1 + assert ci.partial + assert ci.partial[0] == (f1, f2,) + assert not len(ci.partial[1]) + + c = C('foo') + + with inspect_node(c) as ci: + assert ci.type == Bobby + assert ci.instance + assert len(ci.options) == 4 + assert len(ci.processors) == 1 + assert not ci.partial diff --git a/tests/test_basics.py b/tests/test_basics.py index 283e3d7..5230b0b 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -5,6 +5,7 @@ import pytest import bonobo from bonobo.config.processors import ContextCurrifier from bonobo.constants import NOT_MODIFIED +from bonobo.util.inspect import inspect_node def test_count(): From 2ff19c18a68a69b5e9228739906635d1d1a9ce5e Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 5 Jul 2017 11:34:34 +0200 Subject: [PATCH 19/24] [errors] Implements unrecoverable errors, that will raise and stop the transformation. Used when an invalid ioformat is used. --- bonobo/errors.py | 14 ++ bonobo/examples/datasets/coffeeshops.json | 250 +++++++++++----------- bonobo/examples/datasets/coffeeshops.txt | 248 ++++++++++----------- bonobo/execution/node.py | 5 +- bonobo/nodes/io/base.py | 13 +- 5 files changed, 274 insertions(+), 256 deletions(-) diff --git a/bonobo/errors.py b/bonobo/errors.py index 564950d..8510a50 100644 --- a/bonobo/errors.py +++ b/bonobo/errors.py @@ -60,3 +60,17 @@ class ConfigurationError(Exception): class MissingServiceImplementationError(KeyError): pass + + +class UnrecoverableError(Exception): + """Flag for errors that must interrupt the workflow, either because they will happen for sure on each node run, or + because you know that your transformation has no point continuing runnning after a bad event.""" + +class UnrecoverableValueError(UnrecoverableError, ValueError): + pass + +class UnrecoverableRuntimeError(UnrecoverableError, RuntimeError): + pass + +class UnrecoverableNotImplementedError(UnrecoverableError, NotImplementedError): + pass diff --git a/bonobo/examples/datasets/coffeeshops.json b/bonobo/examples/datasets/coffeeshops.json index 60e89b1..391b5e8 100644 --- a/bonobo/examples/datasets/coffeeshops.json +++ b/bonobo/examples/datasets/coffeeshops.json @@ -1,182 +1,182 @@ -{"Le Reynou": "2 bis quai de la m\u00e9gisserie, 75001 Paris, France", -"les montparnos": "65 boulevard Pasteur, 75015 Paris, France", -"Le Saint Jean": "23 rue des abbesses, 75018 Paris, France", -"Le Felteu": "1 rue Pecquay, 75004 Paris, France", +{"les montparnos": "65 boulevard Pasteur, 75015 Paris, France", +"Coffee Chope": "344Vrue Vaugirard, 75015 Paris, France", +"Caf\u00e9 Lea": "5 rue Claude Bernard, 75005 Paris, France", +"Le Bellerive": "71 quai de Seine, 75019 Paris, France", +"Le drapeau de la fidelit\u00e9": "21 rue Copreaux, 75015 Paris, France", "O q de poule": "53 rue du ruisseau, 75018 Paris, France", +"Le caf\u00e9 des amis": "125 rue Blomet, 75015 Paris, France", "Le chantereine": "51 Rue Victoire, 75009 Paris, France", "Le M\u00fcller": "11 rue Feutrier, 75018 Paris, France", -"La Caravane": "Rue de la Fontaine au Roi, 75011 Paris, France", -"Le Pas Sage": "1 Passage du Grand Cerf, 75002 Paris, France", -"La Renaissance": "112 Rue Championnet, 75018 Paris, France", "Ext\u00e9rieur Quai": "5, rue d'Alsace, 75010 Paris, France", -"Le Sully": "6 Bd henri IV, 75004 Paris, France", -"Le drapeau de la fidelit\u00e9": "21 rue Copreaux, 75015 Paris, France", -"Le caf\u00e9 des amis": "125 rue Blomet, 75015 Paris, France", -"Le Kleemend's": "34 avenue Pierre Mend\u00e8s-France, 75013 Paris, France", -"Assaporare Dix sur Dix": "75, avenue Ledru-Rollin, 75012 Paris, France", -"Caf\u00e9 Pierre": "202 rue du faubourg st antoine, 75012 Paris, France", -"Le Caf\u00e9 Livres": "10 rue Saint Martin, 75004 Paris, France", -"Le Chaumontois": "12 rue Armand Carrel, 75018 Paris, France", -"Le Square": "31 rue Saint-Dominique, 75007 Paris, France", -"Les Arcades": "61 rue de Ponthieu, 75008 Paris, France", +"La Bauloise": "36 rue du hameau, 75015 Paris, France", +"Le Dellac": "14 rue Rougemont, 75009 Paris, France", "Le Bosquet": "46 avenue Bosquet, 75007 Paris, France", +"Le Sully": "6 Bd henri IV, 75004 Paris, France", +"Le Felteu": "1 rue Pecquay, 75004 Paris, France", "Le bistrot de Ma\u00eblle et Augustin": "42 rue coquill\u00e8re, 75001 Paris, France", "D\u00e9d\u00e9 la frite": "52 rue Notre-Dame des Victoires, 75002 Paris, France", "Cardinal Saint-Germain": "11 boulevard Saint-Germain, 75005 Paris, France", +"Le Reynou": "2 bis quai de la m\u00e9gisserie, 75001 Paris, France", +"Aux cadrans": "21 ter boulevard Diderot, 75012 Paris, France", +"Le Saint Jean": "23 rue des abbesses, 75018 Paris, France", +"La Renaissance": "112 Rue Championnet, 75018 Paris, France", +"Le Square": "31 rue Saint-Dominique, 75007 Paris, France", +"Les Arcades": "61 rue de Ponthieu, 75008 Paris, France", +"Le Kleemend's": "34 avenue Pierre Mend\u00e8s-France, 75013 Paris, France", +"Assaporare Dix sur Dix": "75, avenue Ledru-Rollin, 75012 Paris, France", +"Caf\u00e9 Pierre": "202 rue du faubourg st antoine, 75012 Paris, France", "Caf\u00e9 antoine": "17 rue Jean de la Fontaine, 75016 Paris, France", "Au cerceau d'or": "129 boulevard sebastopol, 75002 Paris, France", -"Aux cadrans": "21 ter boulevard Diderot, 75012 Paris, France", -"Caf\u00e9 Lea": "5 rue Claude Bernard, 75005 Paris, France", -"Le Bellerive": "71 quai de Seine, 75019 Paris, France", -"La Bauloise": "36 rue du hameau, 75015 Paris, France", -"Le Dellac": "14 rue Rougemont, 75009 Paris, France", +"La Caravane": "Rue de la Fontaine au Roi, 75011 Paris, France", +"Le Pas Sage": "1 Passage du Grand Cerf, 75002 Paris, France", +"Le Caf\u00e9 Livres": "10 rue Saint Martin, 75004 Paris, France", +"Le Chaumontois": "12 rue Armand Carrel, 75018 Paris, France", +"Drole d'endroit pour une rencontre": "58 rue de Montorgueil, 75002 Paris, France", +"Le pari's caf\u00e9": "104 rue caulaincourt, 75018 Paris, France", +"Le Poulailler": "60 rue saint-sabin, 75011 Paris, France", +"Chai 33": "33 Cour Saint Emilion, 75012 Paris, France", +"L'Assassin": "99 rue Jean-Pierre Timbaud, 75011 Paris, France", +"l'Usine": "1 rue d'Avron, 75020 Paris, France", +"La Bricole": "52 rue Liebniz, 75018 Paris, France", +"le ronsard": "place maubert, 75005 Paris, France", +"Face Bar": "82 rue des archives, 75003 Paris, France", +"American Kitchen": "49 rue bichat, 75010 Paris, France", +"La Marine": "55 bis quai de valmy, 75010 Paris, France", +"Le Bloc": "21 avenue Brochant, 75017 Paris, France", +"La Recoleta au Manoir": "229 avenue Gambetta, 75020 Paris, France", +"Le Pareloup": "80 Rue Saint-Charles, 75015 Paris, France", +"La Brasserie Gait\u00e9": "3 rue de la Gait\u00e9, 75014 Paris, France", +"Caf\u00e9 Zen": "46 rue Victoire, 75009 Paris, France", +"O'Breizh": "27 rue de Penthi\u00e8vre, 75008 Paris, France", +"Le Petit Choiseul": "23 rue saint augustin, 75002 Paris, France", +"Invitez vous chez nous": "7 rue Ep\u00e9e de Bois, 75005 Paris, France", +"La Cordonnerie": "142 Rue Saint-Denis 75002 Paris, 75002 Paris, France", +"Le Supercoin": "3, rue Baudelique, 75018 Paris, France", +"Populettes": "86 bis rue Riquet, 75018 Paris, France", +"Au bon coin": "49 rue des Cloys, 75018 Paris, France", +"Le Couvent": "69 rue Broca, 75013 Paris, France", +"La Br\u00fblerie des Ternes": "111 rue mouffetard, 75005 Paris, France", +"L'\u00c9cir": "59 Boulevard Saint-Jacques, 75014 Paris, France", +"Le Chat bossu": "126, rue du Faubourg Saint Antoine, 75012 Paris, France", +"Denfert caf\u00e9": "58 boulvevard Saint Jacques, 75014 Paris, France", +"Le Caf\u00e9 frapp\u00e9": "95 rue Montmartre, 75002 Paris, France", +"La Perle": "78 rue vieille du temple, 75003 Paris, France", +"Le Descartes": "1 rue Thouin, 75005 Paris, France", +"Bagels & Coffee Corner": "Place de Clichy, 75017 Paris, France", +"Le petit club": "55 rue de la tombe Issoire, 75014 Paris, France", +"Le Plein soleil": "90 avenue Parmentier, 75011 Paris, France", +"Le Relais Haussmann": "146, boulevard Haussmann, 75008 Paris, France", +"Le Malar": "88 rue Saint-Dominique, 75007 Paris, France", +"Au panini de la place": "47 rue Belgrand, 75020 Paris, France", +"Le Village": "182 rue de Courcelles, 75017 Paris, France", +"Pause Caf\u00e9": "41 rue de Charonne, 75011 Paris, France", +"Le Pure caf\u00e9": "14 rue Jean Mac\u00e9, 75011 Paris, France", +"Extra old caf\u00e9": "307 fg saint Antoine, 75011 Paris, France", +"Chez Fafa": "44 rue Vinaigriers, 75010 Paris, France", +"En attendant l'or": "3 rue Faidherbe, 75011 Paris, France", "Br\u00fblerie San Jos\u00e9": "30 rue des Petits-Champs, 75002 Paris, France", "Caf\u00e9 de la Mairie (du VIII)": "rue de Lisbonne, 75008 Paris, France", -"Le General Beuret": "9 Place du General Beuret, 75015 Paris, France", -"Le Cap Bourbon": "1 rue Louis le Grand, 75002 Paris, France", -"En attendant l'or": "3 rue Faidherbe, 75011 Paris, France", "Caf\u00e9 Martin": "2 place Martin Nadaud, 75001 Paris, France", "Etienne": "14 rue Turbigo, Paris, 75001 Paris, France", "L'ing\u00e9nu": "184 bd Voltaire, 75011 Paris, France", -"Le Biz": "18 rue Favart, 75002 Paris, France", "L'Olive": "8 rue L'Olive, 75018 Paris, France", -"Le pari's caf\u00e9": "104 rue caulaincourt, 75018 Paris, France", -"Le Poulailler": "60 rue saint-sabin, 75011 Paris, France", -"La Marine": "55 bis quai de valmy, 75010 Paris, France", -"American Kitchen": "49 rue bichat, 75010 Paris, France", -"Chai 33": "33 Cour Saint Emilion, 75012 Paris, France", -"Face Bar": "82 rue des archives, 75003 Paris, France", -"Le Bloc": "21 avenue Brochant, 75017 Paris, France", -"La Bricole": "52 rue Liebniz, 75018 Paris, France", -"le ronsard": "place maubert, 75005 Paris, France", -"l'Usine": "1 rue d'Avron, 75020 Paris, France", -"La Cordonnerie": "142 Rue Saint-Denis 75002 Paris, 75002 Paris, France", -"Invitez vous chez nous": "7 rue Ep\u00e9e de Bois, 75005 Paris, France", -"Le sully": "13 rue du Faubourg Saint Denis, 75010 Paris, France", -"Le Ragueneau": "202 rue Saint-Honor\u00e9, 75001 Paris, France", +"Le Biz": "18 rue Favart, 75002 Paris, France", +"Le Cap Bourbon": "1 rue Louis le Grand, 75002 Paris, France", +"Le General Beuret": "9 Place du General Beuret, 75015 Paris, France", "Le Germinal": "95 avenue Emile Zola, 75015 Paris, France", +"Le Ragueneau": "202 rue Saint-Honor\u00e9, 75001 Paris, France", "Le refuge": "72 rue lamarck, 75018 Paris, France", -"Drole d'endroit pour une rencontre": "58 rue de Montorgueil, 75002 Paris, France", -"Le Petit Choiseul": "23 rue saint augustin, 75002 Paris, France", -"O'Breizh": "27 rue de Penthi\u00e8vre, 75008 Paris, France", -"Le Supercoin": "3, rue Baudelique, 75018 Paris, France", -"Populettes": "86 bis rue Riquet, 75018 Paris, France", -"La Recoleta au Manoir": "229 avenue Gambetta, 75020 Paris, France", -"L'Assassin": "99 rue Jean-Pierre Timbaud, 75011 Paris, France", -"Le Pareloup": "80 Rue Saint-Charles, 75015 Paris, France", -"Caf\u00e9 Zen": "46 rue Victoire, 75009 Paris, France", -"La Brasserie Gait\u00e9": "3 rue de la Gait\u00e9, 75014 Paris, France", -"Au bon coin": "49 rue des Cloys, 75018 Paris, France", -"La Br\u00fblerie des Ternes": "111 rue mouffetard, 75005 Paris, France", -"Le Chat bossu": "126, rue du Faubourg Saint Antoine, 75012 Paris, France", -"Denfert caf\u00e9": "58 boulvevard Saint Jacques, 75014 Paris, France", -"Le Couvent": "69 rue Broca, 75013 Paris, France", -"Bagels & Coffee Corner": "Place de Clichy, 75017 Paris, France", -"La Perle": "78 rue vieille du temple, 75003 Paris, France", -"Le Caf\u00e9 frapp\u00e9": "95 rue Montmartre, 75002 Paris, France", -"L'\u00c9cir": "59 Boulevard Saint-Jacques, 75014 Paris, France", -"Le Descartes": "1 rue Thouin, 75005 Paris, France", -"Le petit club": "55 rue de la tombe Issoire, 75014 Paris, France", -"Le Relais Haussmann": "146, boulevard Haussmann, 75008 Paris, France", -"Au panini de la place": "47 rue Belgrand, 75020 Paris, France", -"Extra old caf\u00e9": "307 fg saint Antoine, 75011 Paris, France", -"Le Plein soleil": "90 avenue Parmentier, 75011 Paris, France", -"Le Pure caf\u00e9": "14 rue Jean Mac\u00e9, 75011 Paris, France", -"Le Village": "182 rue de Courcelles, 75017 Paris, France", -"Le Malar": "88 rue Saint-Dominique, 75007 Paris, France", -"Pause Caf\u00e9": "41 rue de Charonne, 75011 Paris, France", -"Chez Fafa": "44 rue Vinaigriers, 75010 Paris, France", -"Caf\u00e9 dans l'aerogare Air France Invalides": "2 rue Robert Esnault Pelterie, 75007 Paris, France", -"Le relais de la victoire": "73 rue de la Victoire, 75009 Paris, France", -"Caprice caf\u00e9": "12 avenue Jean Moulin, 75014 Paris, France", -"Caves populaires": "22 rue des Dames, 75017 Paris, France", -"Cafe de grenelle": "188 rue de Grenelle, 75007 Paris, France", -"Chez Prune": "36 rue Beaurepaire, 75010 Paris, France", -"L'anjou": "1 rue de Montholon, 75009 Paris, France", -"Le Brio": "216, rue Marcadet, 75018 Paris, France", -"Tamm Bara": "7 rue Clisson, 75013 Paris, France", -"La chaumi\u00e8re gourmande": "Route de la Muette \u00e0 Neuilly", -"Club hippique du Jardin d\u2019Acclimatation": "75016 Paris, France", -"Les P\u00e8res Populaires": "46 rue de Buzenval, 75020 Paris, France", -"Epicerie Musicale": "55bis quai de Valmy, 75010 Paris, France", -"Le Centenaire": "104 rue amelot, 75011 Paris, France", -"Le Zazabar": "116 Rue de M\u00e9nilmontant, 75020 Paris, France", -"Ragueneau": "202 rue Saint Honor\u00e9, 75001 Paris, France", -"L'In\u00e9vitable": "22 rue Linn\u00e9, 75005 Paris, France", +"Le sully": "13 rue du Faubourg Saint Denis, 75010 Paris, France", "Le Dunois": "77 rue Dunois, 75013 Paris, France", "La Montagne Sans Genevi\u00e8ve": "13 Rue du Pot de Fer, 75005 Paris, France", "Le Caminito": "48 rue du Dessous des Berges, 75013 Paris, France", "Le petit Bretonneau": "Le petit Bretonneau - \u00e0 l'int\u00e9rieur de l'H\u00f4pital, 75018 Paris, France", +"La chaumi\u00e8re gourmande": "Route de la Muette \u00e0 Neuilly", +"Club hippique du Jardin d\u2019Acclimatation": "75016 Paris, France", "Le bal du pirate": "60 rue des bergers, 75015 Paris, France", +"Le Zazabar": "116 Rue de M\u00e9nilmontant, 75020 Paris, France", "L'antre d'eux": "16 rue DE MEZIERES, 75006 Paris, France", "l'orillon bar": "35 rue de l'orillon, 75011 Paris, France", "zic zinc": "95 rue claude decaen, 75012 Paris, France", +"Les P\u00e8res Populaires": "46 rue de Buzenval, 75020 Paris, France", +"Epicerie Musicale": "55bis quai de Valmy, 75010 Paris, France", +"Le relais de la victoire": "73 rue de la Victoire, 75009 Paris, France", +"Le Centenaire": "104 rue amelot, 75011 Paris, France", +"Cafe de grenelle": "188 rue de Grenelle, 75007 Paris, France", +"Ragueneau": "202 rue Saint Honor\u00e9, 75001 Paris, France", "Caf\u00e9 Pistache": "9 rue des petits champs, 75001 Paris, France", "La Cagnotte": "13 Rue Jean-Baptiste Dumay, 75020 Paris, France", -"bistrot les timbr\u00e9s": "14 rue d'alleray, 75015 Paris, France", "Le Killy Jen": "28 bis boulevard Diderot, 75012 Paris, France", "Caf\u00e9 beauveau": "9 rue de Miromesnil, 75008 Paris, France", "le 1 cinq": "172 rue de vaugirard, 75015 Paris, France", -"Au Vin Des Rues": "21 rue Boulard, 75014 Paris, France", "Les Artisans": "106 rue Lecourbe, 75015 Paris, France", "Peperoni": "83 avenue de Wagram, 75001 Paris, France", -"Le BB (Bouchon des Batignolles)": "2 rue Lemercier, 75017 Paris, France", -"La Libert\u00e9": "196 rue du faubourg saint-antoine, 75012 Paris, France", -"Chez Rutabaga": "16 rue des Petits Champs, 75002 Paris, France", -"La cantoche de Paname": "40 Boulevard Beaumarchais, 75011 Paris, France", -"Le Saint Ren\u00e9": "148 Boulevard de Charonne, 75020 Paris, France", -"La Brocante": "10 rue Rossini, 75009 Paris, France", -"Caf\u00e9 Clochette": "16 avenue Richerand, 75010 Paris, France", -"L'europ\u00e9en": "21 Bis Boulevard Diderot, 75012 Paris, France", -"NoMa": "39 rue Notre Dame de Nazareth, 75003 Paris, France", -"O'Paris": "1 Rue des Envierges, 75020 Paris, France", +"Le Brio": "216, rue Marcadet, 75018 Paris, France", +"Tamm Bara": "7 rue Clisson, 75013 Paris, France", +"Caf\u00e9 dans l'aerogare Air France Invalides": "2 rue Robert Esnault Pelterie, 75007 Paris, France", +"bistrot les timbr\u00e9s": "14 rue d'alleray, 75015 Paris, France", +"Caprice caf\u00e9": "12 avenue Jean Moulin, 75014 Paris, France", +"Caves populaires": "22 rue des Dames, 75017 Paris, France", +"Au Vin Des Rues": "21 rue Boulard, 75014 Paris, France", +"Chez Prune": "36 rue Beaurepaire, 75010 Paris, France", +"L'In\u00e9vitable": "22 rue Linn\u00e9, 75005 Paris, France", +"L'anjou": "1 rue de Montholon, 75009 Paris, France", "Botak cafe": "1 rue Paul albert, 75018 Paris, France", -"La cantine de Zo\u00e9": "136 rue du Faubourg poissonni\u00e8re, 75010 Paris, France", -"Les caves populaires": "22 rue des Dames, 75017 Paris, France", -"Le Plomb du cantal": "3 rue Ga\u00eet\u00e9, 75014 Paris, France", -"Trois pi\u00e8ces cuisine": "101 rue des dames, 75017 Paris, France", -"Le Zinc": "61 avenue de la Motte Picquet, 75015 Paris, France", -"L'avant comptoir": "3 carrefour de l'Od\u00e9on, 75006 Paris, France", -"Les Vendangeurs": "6/8 rue Stanislas, 75006 Paris, France", -"Chez Luna": "108 rue de M\u00e9nilmontant, 75020 Paris, France", -"Le bar Fleuri": "1 rue du Plateau, 75019 Paris, France", "Bistrot Saint-Antoine": "58 rue du Fbg Saint-Antoine, 75012 Paris, France", "Chez Oscar": "11/13 boulevard Beaumarchais, 75004 Paris, France", "Le Piquet": "48 avenue de la Motte Picquet, 75015 Paris, France", +"L'avant comptoir": "3 carrefour de l'Od\u00e9on, 75006 Paris, France", "le chateau d'eau": "67 rue du Ch\u00e2teau d'eau, 75010 Paris, France", +"Les Vendangeurs": "6/8 rue Stanislas, 75006 Paris, France", "maison du vin": "52 rue des plantes, 75014 Paris, France", "Le Tournebride": "104 rue Mouffetard, 75005 Paris, France", "Le Fronton": "63 rue de Ponthieu, 75008 Paris, France", +"Le BB (Bouchon des Batignolles)": "2 rue Lemercier, 75017 Paris, France", +"La cantine de Zo\u00e9": "136 rue du Faubourg poissonni\u00e8re, 75010 Paris, France", +"Chez Rutabaga": "16 rue des Petits Champs, 75002 Paris, France", +"Les caves populaires": "22 rue des Dames, 75017 Paris, France", +"Le Plomb du cantal": "3 rue Ga\u00eet\u00e9, 75014 Paris, France", +"Trois pi\u00e8ces cuisine": "101 rue des dames, 75017 Paris, France", +"La Brocante": "10 rue Rossini, 75009 Paris, France", +"Le Zinc": "61 avenue de la Motte Picquet, 75015 Paris, France", +"Chez Luna": "108 rue de M\u00e9nilmontant, 75020 Paris, France", +"Le bar Fleuri": "1 rue du Plateau, 75019 Paris, France", +"La Libert\u00e9": "196 rue du faubourg saint-antoine, 75012 Paris, France", +"La cantoche de Paname": "40 Boulevard Beaumarchais, 75011 Paris, France", +"Le Saint Ren\u00e9": "148 Boulevard de Charonne, 75020 Paris, France", +"Caf\u00e9 Clochette": "16 avenue Richerand, 75010 Paris, France", +"L'europ\u00e9en": "21 Bis Boulevard Diderot, 75012 Paris, France", +"NoMa": "39 rue Notre Dame de Nazareth, 75003 Paris, France", "le lutece": "380 rue de vaugirard, 75015 Paris, France", +"O'Paris": "1 Rue des Envierges, 75020 Paris, France", "Rivolux": "16 rue de Rivoli, 75004 Paris, France", "Brasiloja": "16 rue Ganneron, 75018 Paris, France", -"Le caf\u00e9 Monde et M\u00e9dias": "Place de la R\u00e9publique, 75003 Paris, France", -"L'entrep\u00f4t": "157 rue Bercy 75012 Paris, 75012 Paris, France", -"Coffee Chope": "344Vrue Vaugirard, 75015 Paris, France", -"l'El\u00e9phant du nil": "125 Rue Saint-Antoine, 75004 Paris, France", -"Le Parc Vaugirard": "358 rue de Vaugirard, 75015 Paris, France", -"Pari's Caf\u00e9": "174 avenue de Clichy, 75017 Paris, France", -"Le Comptoir": "354 bis rue Vaugirard, 75015 Paris, France", -"Caf\u00e9 Varenne": "36 rue de Varenne, 75007 Paris, France", -"Melting Pot": "3 rue de Lagny, 75020 Paris, France", -"le Zango": "58 rue Daguerre, 75014 Paris, France", -"Chez Miamophile": "6 rue M\u00e9lingue, 75019 Paris, France", "Institut des Cultures d'Islam": "19-23 rue L\u00e9on, 75018 Paris, France", "Canopy Caf\u00e9 associatif": "19 rue Pajol, 75018 Paris, France", -"Caf\u00e9 rallye tournelles": "11 Quai de la Tournelle, 75005 Paris, France", "Petits Freres des Pauvres": "47 rue de Batignolles, 75017 Paris, France", -"Brasserie le Morvan": "61 rue du ch\u00e2teau d'eau, 75010 Paris, France", +"Le Lucernaire": "53 rue Notre-Dame des Champs, 75006 Paris, France", "L'Angle": "28 rue de Ponthieu, 75008 Paris, France", +"Le Caf\u00e9 d'avant": "35 rue Claude Bernard, 75005 Paris, France", "Caf\u00e9 Dupont": "198 rue de la Convention, 75015 Paris, France", +"Le S\u00e9vign\u00e9": "15 rue du Parc Royal, 75003 Paris, France", "L'Entracte": "place de l'opera, 75002 Paris, France", "Panem": "18 rue de Crussol, 75011 Paris, France", "Au pays de Vannes": "34 bis rue de Wattignies, 75012 Paris, France", -"Le Lucernaire": "53 rue Notre-Dame des Champs, 75006 Paris, France", -"Le Caf\u00e9 d'avant": "35 rue Claude Bernard, 75005 Paris, France", +"l'El\u00e9phant du nil": "125 Rue Saint-Antoine, 75004 Paris, France", "L'\u00e2ge d'or": "26 rue du Docteur Magnan, 75013 Paris, France", -"Le S\u00e9vign\u00e9": "15 rue du Parc Royal, 75003 Paris, France", +"Le Comptoir": "354 bis rue Vaugirard, 75015 Paris, France", "L'horizon": "93, rue de la Roquette, 75011 Paris, France", "L'empreinte": "54, avenue Daumesnil, 75012 Paris, France", "Caf\u00e9 Victor": "10 boulevard Victor, 75015 Paris, France", +"Caf\u00e9 Varenne": "36 rue de Varenne, 75007 Paris, France", "Le Brigadier": "12 rue Blanche, 75009 Paris, France", -"Waikiki": "10 rue d\"Ulm, 75005 Paris, France"} \ No newline at end of file +"Waikiki": "10 rue d\"Ulm, 75005 Paris, France", +"Le Parc Vaugirard": "358 rue de Vaugirard, 75015 Paris, France", +"Pari's Caf\u00e9": "174 avenue de Clichy, 75017 Paris, France", +"Melting Pot": "3 rue de Lagny, 75020 Paris, France", +"le Zango": "58 rue Daguerre, 75014 Paris, France", +"Chez Miamophile": "6 rue M\u00e9lingue, 75019 Paris, France", +"Le caf\u00e9 Monde et M\u00e9dias": "Place de la R\u00e9publique, 75003 Paris, France", +"Caf\u00e9 rallye tournelles": "11 Quai de la Tournelle, 75005 Paris, France", +"Brasserie le Morvan": "61 rue du ch\u00e2teau d'eau, 75010 Paris, France", +"L'entrep\u00f4t": "157 rue Bercy 75012 Paris, 75012 Paris, France"} \ No newline at end of file diff --git a/bonobo/examples/datasets/coffeeshops.txt b/bonobo/examples/datasets/coffeeshops.txt index 5fe1ef6..b87eacb 100644 --- a/bonobo/examples/datasets/coffeeshops.txt +++ b/bonobo/examples/datasets/coffeeshops.txt @@ -1,182 +1,182 @@ -Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France les montparnos, 65 boulevard Pasteur, 75015 Paris, France -Le Saint Jean, 23 rue des abbesses, 75018 Paris, France -Le Felteu, 1 rue Pecquay, 75004 Paris, France +Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France +Café Lea, 5 rue Claude Bernard, 75005 Paris, France +Le Bellerive, 71 quai de Seine, 75019 Paris, France +Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France O q de poule, 53 rue du ruisseau, 75018 Paris, France +Le café des amis, 125 rue Blomet, 75015 Paris, France Le chantereine, 51 Rue Victoire, 75009 Paris, France Le Müller, 11 rue Feutrier, 75018 Paris, France -La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France -Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France -La Renaissance, 112 Rue Championnet, 75018 Paris, France Extérieur Quai, 5, rue d'Alsace, 75010 Paris, France -Le Sully, 6 Bd henri IV, 75004 Paris, France -Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France -Le café des amis, 125 rue Blomet, 75015 Paris, France -Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France -Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France -Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France -Le Café Livres, 10 rue Saint Martin, 75004 Paris, France -Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France -Le Square, 31 rue Saint-Dominique, 75007 Paris, France -Les Arcades, 61 rue de Ponthieu, 75008 Paris, France +La Bauloise, 36 rue du hameau, 75015 Paris, France +Le Dellac, 14 rue Rougemont, 75009 Paris, France Le Bosquet, 46 avenue Bosquet, 75007 Paris, France +Le Sully, 6 Bd henri IV, 75004 Paris, France +Le Felteu, 1 rue Pecquay, 75004 Paris, France Le bistrot de Maëlle et Augustin, 42 rue coquillère, 75001 Paris, France Dédé la frite, 52 rue Notre-Dame des Victoires, 75002 Paris, France Cardinal Saint-Germain, 11 boulevard Saint-Germain, 75005 Paris, France +Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France +Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France +Le Saint Jean, 23 rue des abbesses, 75018 Paris, France +La Renaissance, 112 Rue Championnet, 75018 Paris, France +Le Square, 31 rue Saint-Dominique, 75007 Paris, France +Les Arcades, 61 rue de Ponthieu, 75008 Paris, France +Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France +Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France +Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France Café antoine, 17 rue Jean de la Fontaine, 75016 Paris, France Au cerceau d'or, 129 boulevard sebastopol, 75002 Paris, France -Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France -Café Lea, 5 rue Claude Bernard, 75005 Paris, France -Le Bellerive, 71 quai de Seine, 75019 Paris, France -La Bauloise, 36 rue du hameau, 75015 Paris, France -Le Dellac, 14 rue Rougemont, 75009 Paris, France +La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France +Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France +Le Café Livres, 10 rue Saint Martin, 75004 Paris, France +Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France +Drole d'endroit pour une rencontre, 58 rue de Montorgueil, 75002 Paris, France +Le pari's café, 104 rue caulaincourt, 75018 Paris, France +Le Poulailler, 60 rue saint-sabin, 75011 Paris, France +Chai 33, 33 Cour Saint Emilion, 75012 Paris, France +L'Assassin, 99 rue Jean-Pierre Timbaud, 75011 Paris, France +l'Usine, 1 rue d'Avron, 75020 Paris, France +La Bricole, 52 rue Liebniz, 75018 Paris, France +le ronsard, place maubert, 75005 Paris, France +Face Bar, 82 rue des archives, 75003 Paris, France +American Kitchen, 49 rue bichat, 75010 Paris, France +La Marine, 55 bis quai de valmy, 75010 Paris, France +Le Bloc, 21 avenue Brochant, 75017 Paris, France +La Recoleta au Manoir, 229 avenue Gambetta, 75020 Paris, France +Le Pareloup, 80 Rue Saint-Charles, 75015 Paris, France +La Brasserie Gaité, 3 rue de la Gaité, 75014 Paris, France +Café Zen, 46 rue Victoire, 75009 Paris, France +O'Breizh, 27 rue de Penthièvre, 75008 Paris, France +Le Petit Choiseul, 23 rue saint augustin, 75002 Paris, France +Invitez vous chez nous, 7 rue Epée de Bois, 75005 Paris, France +La Cordonnerie, 142 Rue Saint-Denis 75002 Paris, 75002 Paris, France +Le Supercoin, 3, rue Baudelique, 75018 Paris, France +Populettes, 86 bis rue Riquet, 75018 Paris, France +Au bon coin, 49 rue des Cloys, 75018 Paris, France +Le Couvent, 69 rue Broca, 75013 Paris, France +La Brûlerie des Ternes, 111 rue mouffetard, 75005 Paris, France +L'Écir, 59 Boulevard Saint-Jacques, 75014 Paris, France +Le Chat bossu, 126, rue du Faubourg Saint Antoine, 75012 Paris, France +Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France +Le Café frappé, 95 rue Montmartre, 75002 Paris, France +La Perle, 78 rue vieille du temple, 75003 Paris, France +Le Descartes, 1 rue Thouin, 75005 Paris, France +Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France +Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France +Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France +Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France +Le Malar, 88 rue Saint-Dominique, 75007 Paris, France +Au panini de la place, 47 rue Belgrand, 75020 Paris, France +Le Village, 182 rue de Courcelles, 75017 Paris, France +Pause Café, 41 rue de Charonne, 75011 Paris, France +Le Pure café, 14 rue Jean Macé, 75011 Paris, France +Extra old café, 307 fg saint Antoine, 75011 Paris, France +Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France +En attendant l'or, 3 rue Faidherbe, 75011 Paris, France Brûlerie San José, 30 rue des Petits-Champs, 75002 Paris, France Café de la Mairie (du VIII), rue de Lisbonne, 75008 Paris, France -Le General Beuret, 9 Place du General Beuret, 75015 Paris, France -Le Cap Bourbon, 1 rue Louis le Grand, 75002 Paris, France -En attendant l'or, 3 rue Faidherbe, 75011 Paris, France Café Martin, 2 place Martin Nadaud, 75001 Paris, France Etienne, 14 rue Turbigo, Paris, 75001 Paris, France L'ingénu, 184 bd Voltaire, 75011 Paris, France -Le Biz, 18 rue Favart, 75002 Paris, France L'Olive, 8 rue L'Olive, 75018 Paris, France -Le pari's café, 104 rue caulaincourt, 75018 Paris, France -Le Poulailler, 60 rue saint-sabin, 75011 Paris, France -La Marine, 55 bis quai de valmy, 75010 Paris, France -American Kitchen, 49 rue bichat, 75010 Paris, France -Chai 33, 33 Cour Saint Emilion, 75012 Paris, France -Face Bar, 82 rue des archives, 75003 Paris, France -Le Bloc, 21 avenue Brochant, 75017 Paris, France -La Bricole, 52 rue Liebniz, 75018 Paris, France -le ronsard, place maubert, 75005 Paris, France -l'Usine, 1 rue d'Avron, 75020 Paris, France -La Cordonnerie, 142 Rue Saint-Denis 75002 Paris, 75002 Paris, France -Invitez vous chez nous, 7 rue Epée de Bois, 75005 Paris, France -Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France -Le Ragueneau, 202 rue Saint-Honoré, 75001 Paris, France +Le Biz, 18 rue Favart, 75002 Paris, France +Le Cap Bourbon, 1 rue Louis le Grand, 75002 Paris, France +Le General Beuret, 9 Place du General Beuret, 75015 Paris, France Le Germinal, 95 avenue Emile Zola, 75015 Paris, France +Le Ragueneau, 202 rue Saint-Honoré, 75001 Paris, France Le refuge, 72 rue lamarck, 75018 Paris, France -Drole d'endroit pour une rencontre, 58 rue de Montorgueil, 75002 Paris, France -Le Petit Choiseul, 23 rue saint augustin, 75002 Paris, France -O'Breizh, 27 rue de Penthièvre, 75008 Paris, France -Le Supercoin, 3, rue Baudelique, 75018 Paris, France -Populettes, 86 bis rue Riquet, 75018 Paris, France -La Recoleta au Manoir, 229 avenue Gambetta, 75020 Paris, France -L'Assassin, 99 rue Jean-Pierre Timbaud, 75011 Paris, France -Le Pareloup, 80 Rue Saint-Charles, 75015 Paris, France -Café Zen, 46 rue Victoire, 75009 Paris, France -La Brasserie Gaité, 3 rue de la Gaité, 75014 Paris, France -Au bon coin, 49 rue des Cloys, 75018 Paris, France -La Brûlerie des Ternes, 111 rue mouffetard, 75005 Paris, France -Le Chat bossu, 126, rue du Faubourg Saint Antoine, 75012 Paris, France -Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France -Le Couvent, 69 rue Broca, 75013 Paris, France -Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France -La Perle, 78 rue vieille du temple, 75003 Paris, France -Le Café frappé, 95 rue Montmartre, 75002 Paris, France -L'Écir, 59 Boulevard Saint-Jacques, 75014 Paris, France -Le Descartes, 1 rue Thouin, 75005 Paris, France -Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France -Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France -Au panini de la place, 47 rue Belgrand, 75020 Paris, France -Extra old café, 307 fg saint Antoine, 75011 Paris, France -Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France -Le Pure café, 14 rue Jean Macé, 75011 Paris, France -Le Village, 182 rue de Courcelles, 75017 Paris, France -Le Malar, 88 rue Saint-Dominique, 75007 Paris, France -Pause Café, 41 rue de Charonne, 75011 Paris, France -Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France -Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France -Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France -Caprice café, 12 avenue Jean Moulin, 75014 Paris, France -Caves populaires, 22 rue des Dames, 75017 Paris, France -Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France -Chez Prune, 36 rue Beaurepaire, 75010 Paris, France -L'anjou, 1 rue de Montholon, 75009 Paris, France -Le Brio, 216, rue Marcadet, 75018 Paris, France -Tamm Bara, 7 rue Clisson, 75013 Paris, France -La chaumière gourmande, Route de la Muette à Neuilly -Club hippique du Jardin d’Acclimatation, 75016 Paris, France -Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France -Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France -Le Centenaire, 104 rue amelot, 75011 Paris, France -Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France -Ragueneau, 202 rue Saint Honoré, 75001 Paris, France -L'Inévitable, 22 rue Linné, 75005 Paris, France +Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France Le Dunois, 77 rue Dunois, 75013 Paris, France La Montagne Sans Geneviève, 13 Rue du Pot de Fer, 75005 Paris, France Le Caminito, 48 rue du Dessous des Berges, 75013 Paris, France Le petit Bretonneau, Le petit Bretonneau - à l'intérieur de l'Hôpital, 75018 Paris, France +La chaumière gourmande, Route de la Muette à Neuilly +Club hippique du Jardin d’Acclimatation, 75016 Paris, France Le bal du pirate, 60 rue des bergers, 75015 Paris, France +Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France L'antre d'eux, 16 rue DE MEZIERES, 75006 Paris, France l'orillon bar, 35 rue de l'orillon, 75011 Paris, France zic zinc, 95 rue claude decaen, 75012 Paris, France +Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France +Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France +Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France +Le Centenaire, 104 rue amelot, 75011 Paris, France +Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France +Ragueneau, 202 rue Saint Honoré, 75001 Paris, France Café Pistache, 9 rue des petits champs, 75001 Paris, France La Cagnotte, 13 Rue Jean-Baptiste Dumay, 75020 Paris, France -bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France Café beauveau, 9 rue de Miromesnil, 75008 Paris, France le 1 cinq, 172 rue de vaugirard, 75015 Paris, France -Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France Les Artisans, 106 rue Lecourbe, 75015 Paris, France Peperoni, 83 avenue de Wagram, 75001 Paris, France -Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France -La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France -Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France -La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France -Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France -La Brocante, 10 rue Rossini, 75009 Paris, France -Café Clochette, 16 avenue Richerand, 75010 Paris, France -L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France -NoMa, 39 rue Notre Dame de Nazareth, 75003 Paris, France -O'Paris, 1 Rue des Envierges, 75020 Paris, France +Le Brio, 216, rue Marcadet, 75018 Paris, France +Tamm Bara, 7 rue Clisson, 75013 Paris, France +Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France +bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France +Caprice café, 12 avenue Jean Moulin, 75014 Paris, France +Caves populaires, 22 rue des Dames, 75017 Paris, France +Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France +Chez Prune, 36 rue Beaurepaire, 75010 Paris, France +L'Inévitable, 22 rue Linné, 75005 Paris, France +L'anjou, 1 rue de Montholon, 75009 Paris, France Botak cafe, 1 rue Paul albert, 75018 Paris, France -La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France -Les caves populaires, 22 rue des Dames, 75017 Paris, France -Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France -Trois pièces cuisine, 101 rue des dames, 75017 Paris, France -Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France -L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France -Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France -Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France -Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France +L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France +Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France maison du vin, 52 rue des plantes, 75014 Paris, France Le Tournebride, 104 rue Mouffetard, 75005 Paris, France Le Fronton, 63 rue de Ponthieu, 75008 Paris, France +Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France +La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France +Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France +Les caves populaires, 22 rue des Dames, 75017 Paris, France +Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France +Trois pièces cuisine, 101 rue des dames, 75017 Paris, France +La Brocante, 10 rue Rossini, 75009 Paris, France +Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France +Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France +Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France +La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France +La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France +Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France +Café Clochette, 16 avenue Richerand, 75010 Paris, France +L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France +NoMa, 39 rue Notre Dame de Nazareth, 75003 Paris, France le lutece, 380 rue de vaugirard, 75015 Paris, France +O'Paris, 1 Rue des Envierges, 75020 Paris, France Rivolux, 16 rue de Rivoli, 75004 Paris, France Brasiloja, 16 rue Ganneron, 75018 Paris, France -Le café Monde et Médias, Place de la République, 75003 Paris, France -L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France -Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France -l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France -Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France -Pari's Café, 174 avenue de Clichy, 75017 Paris, France -Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France -Café Varenne, 36 rue de Varenne, 75007 Paris, France -Melting Pot, 3 rue de Lagny, 75020 Paris, France -le Zango, 58 rue Daguerre, 75014 Paris, France -Chez Miamophile, 6 rue Mélingue, 75019 Paris, France Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France Canopy Café associatif, 19 rue Pajol, 75018 Paris, France -Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France -Brasserie le Morvan, 61 rue du château d'eau, 75010 Paris, France +Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France L'Angle, 28 rue de Ponthieu, 75008 Paris, France +Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France Café Dupont, 198 rue de la Convention, 75015 Paris, France +Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France L'Entracte, place de l'opera, 75002 Paris, France Panem, 18 rue de Crussol, 75011 Paris, France Au pays de Vannes, 34 bis rue de Wattignies, 75012 Paris, France -Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France -Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France +l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France L'âge d'or, 26 rue du Docteur Magnan, 75013 Paris, France -Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France +Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France L'horizon, 93, rue de la Roquette, 75011 Paris, France L'empreinte, 54, avenue Daumesnil, 75012 Paris, France Café Victor, 10 boulevard Victor, 75015 Paris, France +Café Varenne, 36 rue de Varenne, 75007 Paris, France Le Brigadier, 12 rue Blanche, 75009 Paris, France -Waikiki, 10 rue d"Ulm, 75005 Paris, France \ No newline at end of file +Waikiki, 10 rue d"Ulm, 75005 Paris, France +Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France +Pari's Café, 174 avenue de Clichy, 75017 Paris, France +Melting Pot, 3 rue de Lagny, 75020 Paris, France +le Zango, 58 rue Daguerre, 75014 Paris, France +Chez Miamophile, 6 rue Mélingue, 75019 Paris, France +Le café Monde et Médias, Place de la République, 75003 Paris, France +Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France +Brasserie le Morvan, 61 rue du château d'eau, 75010 Paris, France +L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France \ No newline at end of file diff --git a/bonobo/execution/node.py b/bonobo/execution/node.py index 635068e..4edb75e 100644 --- a/bonobo/execution/node.py +++ b/bonobo/execution/node.py @@ -3,7 +3,7 @@ from queue import Empty from time import sleep from bonobo.constants import INHERIT_INPUT, NOT_MODIFIED -from bonobo.errors import InactiveReadableError +from bonobo.errors import InactiveReadableError, UnrecoverableError from bonobo.execution.base import LoopingExecutionContext from bonobo.structs.bags import Bag from bonobo.structs.inputs import Input @@ -93,6 +93,9 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext): except Empty: sleep(self.PERIOD) continue + except UnrecoverableError as exc: + self.handle_error(exc, traceback.format_exc()) + break except Exception as exc: # pylint: disable=broad-except self.handle_error(exc, traceback.format_exc()) diff --git a/bonobo/nodes/io/base.py b/bonobo/nodes/io/base.py index d9b3212..58088d0 100644 --- a/bonobo/nodes/io/base.py +++ b/bonobo/nodes/io/base.py @@ -1,5 +1,6 @@ from bonobo import settings from bonobo.config import Configurable, ContextProcessor, Option, Service +from bonobo.errors import UnrecoverableValueError, UnrecoverableNotImplementedError from bonobo.structs.bags import Bag @@ -9,21 +10,21 @@ class IOFormatEnabled(Configurable): def get_input(self, *args, **kwargs): if self.ioformat == settings.IOFORMAT_ARG0: if len(args) != 1 or len(kwargs): - raise ValueError( + raise UnrecoverableValueError( 'Wrong input formating: IOFORMAT=ARG0 implies one arg and no kwargs, got args={!r} and kwargs={!r}.'. - format(args, kwargs) + format(args, kwargs) ) return args[0] if self.ioformat == settings.IOFORMAT_KWARGS: if len(args) or not len(kwargs): - raise ValueError( + raise UnrecoverableValueError( 'Wrong input formating: IOFORMAT=KWARGS ioformat implies no arg, got args={!r} and kwargs={!r}.'. - format(args, kwargs) + format(args, kwargs) ) return kwargs - raise NotImplementedError('Unsupported format.') + raise UnrecoverableNotImplementedError('Unsupported format.') def get_output(self, row): if self.ioformat == settings.IOFORMAT_ARG0: @@ -32,7 +33,7 @@ class IOFormatEnabled(Configurable): if self.ioformat == settings.IOFORMAT_KWARGS: return Bag(**row) - raise NotImplementedError('Unsupported format.') + raise UnrecoverableNotImplementedError('Unsupported format.') class FileHandler(Configurable): From 0bcdbd70ab948c03480f04f7f99a931dfd703c2d Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 5 Jul 2017 11:52:47 +0200 Subject: [PATCH 20/24] [nodes] Removes old pretty printers (bonobo.pprint, bonobo.PrettyPrint) in favor of simpler bonobo.PrettyPrinter implementation. /!\ BC break /!\ --- bonobo/_api.py | 3 +-- bonobo/nodes/basics.py | 49 ------------------------------------------ 2 files changed, 1 insertion(+), 51 deletions(-) diff --git a/bonobo/_api.py b/bonobo/_api.py index 89b6d4c..6cf328c 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -1,6 +1,6 @@ from bonobo.structs import Bag, Graph, Token from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \ - PrettyPrinter, PickleWriter, PickleReader, RateLimited, Tee, count, identity, noop, pprint + PrettyPrinter, PickleWriter, PickleReader, RateLimited, Tee, count, identity, noop from bonobo.strategies import create_strategy from bonobo.util.objects import get_name @@ -109,7 +109,6 @@ register_api_group( count, identity, noop, - pprint, ) diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index c1ead61..85b2114 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -1,14 +1,11 @@ import functools import itertools -from colorama import Fore, Style - from bonobo import settings from bonobo.config import Configurable, Option from bonobo.config.processors import ContextProcessor from bonobo.constants import NOT_MODIFIED from bonobo.structs.bags import Bag -from bonobo.util.compat import deprecated from bonobo.util.objects import ValueHolder from bonobo.util.term import CLEAR_EOL @@ -17,7 +14,6 @@ __all__ = [ 'Limit', 'Tee', 'count', - 'pprint', 'PrettyPrinter', 'noop', ] @@ -87,51 +83,6 @@ class PrettyPrinter(Configurable): ) -_pprint = PrettyPrinter() - - -@deprecated -def pprint(*args, **kwargs): - return _pprint(*args, **kwargs) - - -def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True): - from bonobo.constants import NOT_MODIFIED - - def _pprint(*args, **kwargs): - nonlocal title_keys, sort, print_values - - row = args[0] - for key in title_keys: - if key in row: - print(Style.BRIGHT, row.get(key), Style.RESET_ALL, sep='') - break - - if print_values: - for k in sorted(row) if sort else row: - print( - ' • ', - Fore.BLUE, - k, - Style.RESET_ALL, - ' : ', - Fore.BLACK, - '(', - type(row[k]).__name__, - ')', - Style.RESET_ALL, - ' ', - repr(row[k]), - CLEAR_EOL, - ) - - yield NOT_MODIFIED - - _pprint.__name__ = 'pprint' - - return _pprint - - def noop(*args, **kwargs): # pylint: disable=unused-argument from bonobo.constants import NOT_MODIFIED return NOT_MODIFIED From 6ef25deac9a624f115459e3fbc7cdfb2320d1b2a Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 5 Jul 2017 12:01:37 +0200 Subject: [PATCH 21/24] [config] Adds test for requires() decorator. --- bonobo/errors.py | 11 +++++++---- tests/config/test_services.py | 22 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/bonobo/errors.py b/bonobo/errors.py index 8510a50..08b97d4 100644 --- a/bonobo/errors.py +++ b/bonobo/errors.py @@ -58,19 +58,22 @@ class ConfigurationError(Exception): pass -class MissingServiceImplementationError(KeyError): - pass - - class UnrecoverableError(Exception): """Flag for errors that must interrupt the workflow, either because they will happen for sure on each node run, or because you know that your transformation has no point continuing runnning after a bad event.""" + class UnrecoverableValueError(UnrecoverableError, ValueError): pass + class UnrecoverableRuntimeError(UnrecoverableError, RuntimeError): pass + class UnrecoverableNotImplementedError(UnrecoverableError, NotImplementedError): pass + + +class MissingServiceImplementationError(UnrecoverableError, KeyError): + pass diff --git a/tests/config/test_services.py b/tests/config/test_services.py index b762dbe..ff81e82 100644 --- a/tests/config/test_services.py +++ b/tests/config/test_services.py @@ -3,7 +3,7 @@ import time import pytest -from bonobo.config import Configurable, Container, Exclusive, Service +from bonobo.config import Configurable, Container, Exclusive, Service, requires from bonobo.config.services import validate_service_name @@ -94,3 +94,23 @@ def test_exclusive(): 'hello', '0 0', '0 1', '0 2', '0 3', '0 4', '1 0', '1 1', '1 2', '1 3', '1 4', '2 0', '2 1', '2 2', '2 3', '2 4', '3 0', '3 1', '3 2', '3 3', '3 4', '4 0', '4 1', '4 2', '4 3', '4 4' ] + + +def test_requires(): + vcr = VCR() + + services = Container( + output=vcr.append + ) + + @requires('output') + def append(out, x): + out(x) + + svcargs = services.args_for(append) + assert len(svcargs) == 1 + assert svcargs[0] == vcr.append + + + + From 9801c75720ceb5715e805c26d4be07c734155cac Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 5 Jul 2017 12:41:14 +0200 Subject: [PATCH 22/24] [settings] Better impl. of Setting class, tests for it and refactor hardcoded settings to use it. --- bonobo/_api.py | 2 +- bonobo/commands/__init__.py | 6 ++-- bonobo/commands/run.py | 4 +-- bonobo/ext/console.py | 4 +-- bonobo/logging.py | 2 +- bonobo/nodes/basics.py | 2 +- bonobo/settings.py | 53 ++++++++++++++++++++++++------- tests/test_settings.py | 63 +++++++++++++++++++++++++++++++++++++ 8 files changed, 115 insertions(+), 21 deletions(-) create mode 100644 tests/test_settings.py diff --git a/bonobo/_api.py b/bonobo/_api.py index 6cf328c..ab890c6 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -45,7 +45,7 @@ def run(graph, strategy=None, plugins=None, services=None): from bonobo import settings settings.check() - if not settings.QUIET: # pragma: no cover + if not settings.QUIET.get(): # pragma: no cover if _is_interactive_console(): from bonobo.ext.console import ConsoleOutputPlugin if ConsoleOutputPlugin not in plugins: diff --git a/bonobo/commands/__init__.py b/bonobo/commands/__init__.py index 59e6dfb..4e183a3 100644 --- a/bonobo/commands/__init__.py +++ b/bonobo/commands/__init__.py @@ -27,9 +27,9 @@ def entrypoint(args=None): args = parser.parse_args(args).__dict__ if args.pop('debug', False): - settings.DEBUG = True - settings.LOGGING_LEVEL = logging.DEBUG - logging.set_level(settings.LOGGING_LEVEL) + settings.DEBUG.set(True) + settings.LOGGING_LEVEL.set(logging.DEBUG) + logging.set_level(settings.LOGGING_LEVEL.get()) logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args)) commands[args.pop('command')](**args) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 7f29d3f..6de6bf6 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -31,10 +31,10 @@ def execute(filename, module, install=False, quiet=False, verbose=False): from bonobo import Graph, run, settings if quiet: - settings.QUIET = True + settings.QUIET.set(True) if verbose: - settings.DEBUG = True + settings.DEBUG.set(True) if filename: if os.path.isdir(filename): diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index f30fae0..acf464b 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -65,7 +65,7 @@ class ConsoleOutputPlugin(Plugin): for i in context.graph.topologically_sorted_indexes: node = context[i] - name_suffix = '({})'.format(i) if settings.DEBUG else '' + name_suffix = '({})'.format(i) if settings.DEBUG.get() else '' if node.alive: _line = ''.join( ( @@ -100,7 +100,7 @@ class ConsoleOutputPlugin(Plugin): print(MOVE_CURSOR_UP(t_cnt + 2), file=sys.stderr) def _write(self, graph_context, rewind): - if settings.PROFILE: + if settings.PROFILE.get(): if self.counter % 10 and self._append_cache: append = self._append_cache else: diff --git a/bonobo/logging.py b/bonobo/logging.py index 17bdeb7..3784600 100644 --- a/bonobo/logging.py +++ b/bonobo/logging.py @@ -75,4 +75,4 @@ def get_logger(name='bonobo'): getLogger = get_logger # Setup formating and level. -setup(level=settings.LOGGING_LEVEL) +setup(level=settings.LOGGING_LEVEL.get()) diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index 85b2114..164eeb1 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -69,7 +69,7 @@ def _count_counter(self, context): class PrettyPrinter(Configurable): def call(self, *args, **kwargs): - formater = self._format_quiet if settings.QUIET else self._format_console + formater = self._format_quiet if settings.QUIET.get() else self._format_console for i, (item, value) in enumerate(itertools.chain(enumerate(args), kwargs.items())): print(formater(i, item, value)) diff --git a/bonobo/settings.py b/bonobo/settings.py index 8e8a780..e5edd83 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -5,6 +5,10 @@ from bonobo.errors import ValidationError def to_bool(s): + if s is None: + return False + if type(s) is bool: + return s if len(s): if s.lower() in ('f', 'false', 'n', 'no', '0'): return False @@ -13,7 +17,18 @@ def to_bool(s): class Setting: - def __init__(self, name, default=None, validator=None): + __all__ = {} + + @classmethod + def clear_all(cls): + for setting in Setting.__all__.values(): + setting.clear() + + def __new__(cls, name, *args, **kwargs): + Setting.__all__[name] = super().__new__(cls) + return Setting.__all__[name] + + def __init__(self, name, default=None, validator=None, formatter=None): self.name = name if default: @@ -21,15 +36,14 @@ class Setting: else: self.default = lambda: None - if validator: - self.validator = validator - else: - self.validator = None + self.validator = validator + self.formatter = formatter def __repr__(self): return ''.format(self.name, self.get()) def set(self, value): + value = self.formatter(value) if self.formatter else value if self.validator and not self.validator(value): raise ValidationError('Invalid value {!r} for setting {}.'.format(value, self.name)) self.value = value @@ -38,21 +52,35 @@ class Setting: try: return self.value except AttributeError: - self.value = self.default() + value = os.environ.get(self.name, None) + if value is None: + value = self.default() + self.set(value) return self.value + def clear(self): + try: + del self.value + except AttributeError: + pass + # Debug/verbose mode. -DEBUG = to_bool(os.environ.get('DEBUG', 'f')) +DEBUG = Setting('DEBUG', formatter=to_bool, default=False) # Profile mode. -PROFILE = to_bool(os.environ.get('PROFILE', 'f')) +PROFILE = Setting('PROFILE', formatter=to_bool, default=False) # Quiet mode. -QUIET = to_bool(os.environ.get('QUIET', 'f')) +QUIET = Setting('QUIET', formatter=to_bool, default=False) # Logging level. -LOGGING_LEVEL = logging.DEBUG if DEBUG else logging.INFO +LOGGING_LEVEL = Setting( + 'LOGGING_LEVEL', + formatter=logging._checkLevel, + validator=logging._checkLevel, + default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO +) # Input/Output format for transformations IOFORMAT_ARG0 = 'arg0' @@ -67,5 +95,8 @@ IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__co def check(): - if DEBUG and QUIET: + if DEBUG.get() and QUIET.get(): raise RuntimeError('I cannot be verbose and quiet at the same time.') + + +clear_all = Setting.clear_all diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..c8313c5 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,63 @@ +import logging +from os import environ +from unittest.mock import patch + +import pytest + +from bonobo import settings + +TEST_SETTING = 'TEST_SETTING' + + +def test_to_bool(): + assert not settings.to_bool('') + assert not settings.to_bool('FALSE') + assert not settings.to_bool('NO') + assert not settings.to_bool('0') + + assert settings.to_bool('yup') + assert settings.to_bool('True') + assert settings.to_bool('yes') + assert settings.to_bool('1') + + +def test_setting(): + s = settings.Setting(TEST_SETTING) + assert s.get() is None + + with patch.dict(environ, {TEST_SETTING: 'hello'}): + assert s.get() is None + s.clear() + assert s.get() == 'hello' + + s = settings.Setting(TEST_SETTING, default='nope') + assert s.get() is 'nope' + + with patch.dict(environ, {TEST_SETTING: 'hello'}): + assert s.get() == 'nope' + s.clear() + assert s.get() == 'hello' + + +def test_default_settings(): + settings.clear_all() + + assert settings.DEBUG.get() == False + assert settings.PROFILE.get() == False + assert settings.QUIET.get() == False + assert settings.LOGGING_LEVEL.get() == logging._checkLevel('INFO') + + with patch.dict(environ, {'DEBUG': 't'}): + settings.clear_all() + assert settings.LOGGING_LEVEL.get() == logging._checkLevel('DEBUG') + + settings.clear_all() + + +def test_check(): + settings.check() + with patch.dict(environ, {'DEBUG': 't', 'PROFILE': 't', 'QUIET': 't'}): + settings.clear_all() + with pytest.raises(RuntimeError): + settings.check() + settings.clear_all() From 8de6f50523d74ff08d81a0764d19259393c705ae Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 5 Jul 2017 13:08:53 +0200 Subject: [PATCH 23/24] [examples] Fix examples, fix termination bug with unrecoverable errors. --- bonobo/examples/datasets/fablabs.py | 8 ++++---- bonobo/examples/nodes/filter.py | 7 +++++-- bonobo/examples/nodes/slow.py | 3 +++ bonobo/examples/tutorials/tut02e02_write.py | 2 +- bonobo/execution/node.py | 1 + bonobo/structs/inputs.py | 18 +++++++++++------- 6 files changed, 25 insertions(+), 14 deletions(-) diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index be95fe1..33ed91c 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -73,15 +73,15 @@ def display(row): print( ' - {}address{}: {address}'. - format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address)) + format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address)) ) print( ' - {}links{}: {links}'. - format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links'])) + format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links'])) ) print( ' - {}geometry{}: {geometry}'. - format(Fore.BLUE, Style.RESET_ALL, **row) + format(Fore.BLUE, Style.RESET_ALL, **row) ) print( ' - {}source{}: {source}'.format( @@ -96,8 +96,8 @@ graph = bonobo.Graph( ), normalize, filter_france, + bonobo.JsonWriter(path='fablabs.txt', ioformat='arg0'), bonobo.Tee(display), - bonobo.JsonWriter(path='fablabs.txt'), ) if __name__ == '__main__': diff --git a/bonobo/examples/nodes/filter.py b/bonobo/examples/nodes/filter.py index bf390e9..4f7219a 100644 --- a/bonobo/examples/nodes/filter.py +++ b/bonobo/examples/nodes/filter.py @@ -9,13 +9,16 @@ class OddOnlyFilter(Filter): @Filter -def MultiplesOfThreeOnlyFilter(self, i): +def multiples_of_three(i): return not (i % 3) graph = bonobo.Graph( lambda: tuple(range(50)), OddOnlyFilter(), - MultiplesOfThreeOnlyFilter(), + multiples_of_three, print, ) + +if __name__ == '__main__': + bonobo.run(graph) diff --git a/bonobo/examples/nodes/slow.py b/bonobo/examples/nodes/slow.py index b9623af..ecaaf44 100644 --- a/bonobo/examples/nodes/slow.py +++ b/bonobo/examples/nodes/slow.py @@ -14,3 +14,6 @@ graph = bonobo.Graph( pause, print, ) + +if __name__ == '__main__': + bonobo.run(graph) diff --git a/bonobo/examples/tutorials/tut02e02_write.py b/bonobo/examples/tutorials/tut02e02_write.py index 1d41ac2..664bca6 100644 --- a/bonobo/examples/tutorials/tut02e02_write.py +++ b/bonobo/examples/tutorials/tut02e02_write.py @@ -8,7 +8,7 @@ def split_one(line): graph = bonobo.Graph( bonobo.FileReader('coffeeshops.txt'), split_one, - bonobo.JsonWriter('coffeeshops.json'), + bonobo.JsonWriter('coffeeshops.json', ioformat='arg0'), ) if __name__ == '__main__': diff --git a/bonobo/execution/node.py b/bonobo/execution/node.py index 4edb75e..45691a6 100644 --- a/bonobo/execution/node.py +++ b/bonobo/execution/node.py @@ -95,6 +95,7 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext): continue except UnrecoverableError as exc: self.handle_error(exc, traceback.format_exc()) + self.input.shutdown() break except Exception as exc: # pylint: disable=broad-except self.handle_error(exc, traceback.format_exc()) diff --git a/bonobo/structs/inputs.py b/bonobo/structs/inputs.py index cf9a6ec..7cfe12f 100644 --- a/bonobo/structs/inputs.py +++ b/bonobo/structs/inputs.py @@ -77,6 +77,12 @@ class Input(Queue, Readable, Writable): return Queue.put(self, data, block, timeout) + def _decrement_runlevel(self): + if self._runlevel == 1: + self.on_finalize() + self._runlevel -= 1 + self.on_end() + def get(self, block=True, timeout=None): if not self.alive: raise InactiveReadableError('Cannot get() on an inactive {}.'.format(Readable.__name__)) @@ -84,13 +90,7 @@ class Input(Queue, Readable, Writable): data = Queue.get(self, block, timeout) if data == END: - if self._runlevel == 1: - self.on_finalize() - - self._runlevel -= 1 - - # callback - self.on_end() + self._decrement_runlevel() if not self.alive: raise InactiveReadableError( @@ -100,6 +100,10 @@ class Input(Queue, Readable, Writable): return data + def shutdown(self): + while self._runlevel >= 1: + self._decrement_runlevel() + def empty(self): self.mutex.acquire() while self._qsize() and self.queue[0] == END: From 4a2c7280d6f8b54f55fa3402dd992a14bed04ddb Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 5 Jul 2017 13:09:46 +0200 Subject: [PATCH 24/24] [misc] Fixes formatting. --- bonobo/config/configurables.py | 7 +++---- bonobo/config/options.py | 9 ++++----- bonobo/examples/datasets/fablabs.py | 6 +++--- bonobo/nodes/io/base.py | 4 ++-- bonobo/util/inspect.py | 20 +++++++++++--------- tests/config/test_configurables.py | 1 - tests/config/test_methods.py | 10 +++++----- tests/config/test_methods_partial.py | 4 ++-- tests/config/test_services.py | 8 +------- 9 files changed, 31 insertions(+), 38 deletions(-) diff --git a/bonobo/config/configurables.py b/bonobo/config/configurables.py index 01db9e0..7b40303 100644 --- a/bonobo/config/configurables.py +++ b/bonobo/config/configurables.py @@ -51,7 +51,7 @@ class ConfigurableMeta(type): return (processor for _, processor in cls.__processors) def __repr__(self): - return ' '.join(('