From 361b0b7ecc98c963c705fd0cd4e73a12bd4d6867 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 16 May 2019 10:19:47 +0100 Subject: [PATCH 1/3] Fixes unused return variable, restore stdout/stderr even if there is an error. --- bonobo/contrib/django/commands.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bonobo/contrib/django/commands.py b/bonobo/contrib/django/commands.py index f2a8a15..4974540 100644 --- a/bonobo/contrib/django/commands.py +++ b/bonobo/contrib/django/commands.py @@ -77,6 +77,7 @@ class ETLCommand(BaseCommand): self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr, ending=CLEAR_EOL + "\n") self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + "!" + Style.RESET_ALL + " " + x - results = self.run(*args, **options) - - self.stdout, self.stderr = _stdout_backup, _stderr_backup + try: + return self.run(*args, **options) + finally: + self.stdout, self.stderr = _stdout_backup, _stderr_backup From e17dbc7134c4d4ea8561b10dca91ad53a03ebce3 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 16 May 2019 11:50:38 +0200 Subject: [PATCH 2/3] Fixes code smell and ignore some useless (in context) lgtm rules. --- bonobo/__init__.py | 4 ++-- bonobo/_api.py | 7 +++---- bonobo/execution/contexts/base.py | 2 +- bonobo/execution/contexts/node.py | 4 +++- bonobo/util/collections.py | 13 ++++++++++--- bonobo/util/statistics.py | 2 +- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 2487780..829aa6e 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -56,8 +56,8 @@ from bonobo._version import __version__ __all__ = ["__version__"] + __all__ with (Path(__file__).parent / "bonobo.svg").open() as f: __logo__ = f.read() -__doc__ = __doc__ -__version__ = __version__ +__doc__ = __doc__ # lgtm [py/redundant-assignment] +__version__ = __version__ # lgtm [py/redundant-assignment] def _repr_html_(): diff --git a/bonobo/_api.py b/bonobo/_api.py index 78ed72f..8432eee 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -125,14 +125,13 @@ def open_fs(fs_url=None, *args, **kwargs): :param str default_protocol: The protocol to use if one is not supplied in the FS URL (defaults to ``"osfs"``). :returns: :class:`fs.base.FS` object """ + import os from fs.opener import open_fs as _open_fs - from os.path import expanduser - from os import getcwd if fs_url is None: - fs_url = getcwd() + fs_url = os.getcwd() - return _open_fs(expanduser(str(fs_url)), *args, **kwargs) + return _open_fs(os.path.expanduser(str(fs_url)), *args, **kwargs) # standard transformations diff --git a/bonobo/execution/contexts/base.py b/bonobo/execution/contexts/base.py index 9399d3f..6f3fa71 100644 --- a/bonobo/execution/contexts/base.py +++ b/bonobo/execution/contexts/base.py @@ -74,7 +74,7 @@ class Lifecycle: self.start() return self - def __exit__(self, exc_type=None, exc_val=None, exc_tb=None): + def __exit__(self, exc_type=None, exc_val=None, exc_tb=None): # lgtm [py/special-method-wrong-signature] self.stop() def get_flags_as_string(self): diff --git a/bonobo/execution/contexts/node.py b/bonobo/execution/contexts/node.py index 30b0d5b..2a4294c 100644 --- a/bonobo/execution/contexts/node.py +++ b/bonobo/execution/contexts/node.py @@ -259,7 +259,9 @@ class NodeExecutionContext(BaseContext, WithStatistics): """ for message in messages: if not isinstance(message, Token): - message = ensure_tuple(message, cls=self._input_type, length=self._input_length) + message = ensure_tuple( + message, cls=self._input_type, length=self._input_length + ) # lgtm [py/call/wrong-named-argument] if self._input_length is None: self._input_length = len(message) self.input.put(message) diff --git a/bonobo/util/collections.py b/bonobo/util/collections.py index 9e903ec..bd00954 100644 --- a/bonobo/util/collections.py +++ b/bonobo/util/collections.py @@ -34,6 +34,10 @@ def _with_length_check(f): def tuple_or_const(tuple_or_mixed, *, consts=(None, False), **kwargs): + """ + Like ensure_tuple, but also accept as valid outputs a list of constants. + """ + if tuple_or_mixed in consts: return tuple_or_mixed if isinstance(tuple_or_mixed, str): @@ -49,10 +53,13 @@ def ensure_tuple(tuple_or_mixed, *, cls=None): If it's not a tuple, let's make a tuple of one item. Otherwise, not changed. - :param tuple_or_mixed: - :return: tuple - + :param tuple_or_mixed: material to work on. + :param cls: type of the resulting tuple, or `tuple` if not provided. + :param length: provided by `_with_length_check` decorator, if specified, make sure that the tuple is of this + length (and raise a `TypeError` if not), otherwise, do nothing. + :return: tuple (or something of type `cls`, if provided) """ + if cls is None: cls = tuple diff --git a/bonobo/util/statistics.py b/bonobo/util/statistics.py index 62a5075..23c7ccd 100644 --- a/bonobo/util/statistics.py +++ b/bonobo/util/statistics.py @@ -26,7 +26,7 @@ class Timer: self.__start = time.time() return self - def __exit__(self, type=None, value=None, traceback=None): + def __exit__(self, type=None, value=None, traceback=None): # lgtm [py/special-method-wrong-signature] # Error handling here self.__finish = time.time() From 3bc28aa0c22750f51836b4660adac8c632155183 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 16 May 2019 11:53:21 +0200 Subject: [PATCH 3/3] Update requirements, fixes packaging version problem. --- Makefile | 2 +- Projectfile | 2 +- bonobo/__init__.py | 49 ++++--------------------- bonobo/commands/__init__.py | 1 + bonobo/config/__init__.py | 6 +-- bonobo/contrib/django/__init__.py | 2 +- bonobo/contrib/django/commands.py | 4 +- bonobo/contrib/google/__init__.py | 7 ++-- bonobo/contrib/opendatasoft/__init__.py | 2 +- bonobo/examples/__init__.py | 2 +- bonobo/execution/strategies/__init__.py | 4 +- bonobo/util/__init__.py | 12 +----- requirements-dev.txt | 3 +- requirements-docker.txt | 1 - requirements-jupyter.txt | 6 +-- requirements-sqlalchemy.txt | 1 - requirements.txt | 3 +- setup.py | 4 +- 18 files changed, 33 insertions(+), 78 deletions(-) diff --git a/Makefile b/Makefile index 0cb2978..841d9a2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Generated by Medikit 0.7.1 on 2019-05-08. +# Generated by Medikit 0.7.1 on 2019-05-16. # All changes will be overriden. # Edit Projectfile and run “make update” (or “medikit update”) to regenerate. diff --git a/Projectfile b/Projectfile index 590f670..d6ab52b 100644 --- a/Projectfile +++ b/Projectfile @@ -47,7 +47,7 @@ python.add_requirements( 'graphviz >=0.8,<0.9', 'jinja2 ~=2.9', 'mondrian ~=0.8', - 'packaging ~=17.0', + 'packaging ~=19.0', 'psutil ~=5.4', 'python-slugify ~=1.2.0', 'requests ~=2.0', diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 829aa6e..ee146fd 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -8,50 +8,17 @@ import sys from pathlib import Path +from bonobo._api import ( + CsvReader, CsvWriter, FileReader, FileWriter, Filter, FixedWindow, Format, Graph, JsonReader, JsonWriter, + LdjsonReader, LdjsonWriter, Limit, MapFields, OrderFields, PickleReader, PickleWriter, PrettyPrinter, RateLimited, + Rename, SetFields, Tee, UnpackItems, __all__, __doc__, count, create_reader, create_strategy, create_writer, + get_argument_parser, get_examples_path, identity, inspect, noop, open_examples_fs, open_fs, parse_args, run +) +from bonobo._version import __version__ + if sys.version_info < (3, 5): raise RuntimeError("Python 3.5+ is required to use Bonobo.") -from bonobo._api import ( - run, - inspect, - Graph, - create_strategy, - open_fs, - CsvReader, - CsvWriter, - FileReader, - FileWriter, - Filter, - FixedWindow, - Format, - JsonReader, - JsonWriter, - LdjsonReader, - LdjsonWriter, - Limit, - MapFields, - OrderFields, - PickleReader, - PickleWriter, - PrettyPrinter, - RateLimited, - Rename, - SetFields, - Tee, - UnpackItems, - count, - identity, - noop, - create_reader, - create_writer, - get_examples_path, - open_examples_fs, - get_argument_parser, - parse_args, - __all__, - __doc__, -) -from bonobo._version import __version__ __all__ = ["__version__"] + __all__ with (Path(__file__).parent / "bonobo.svg").open() as f: diff --git a/bonobo/commands/__init__.py b/bonobo/commands/__init__.py index 7568ae3..e66c85f 100644 --- a/bonobo/commands/__init__.py +++ b/bonobo/commands/__init__.py @@ -2,6 +2,7 @@ import argparse import logging import mondrian + from bonobo import settings from bonobo.commands.base import BaseCommand, BaseGraphCommand diff --git a/bonobo/config/__init__.py b/bonobo/config/__init__.py index 3e2f3bf..243dc99 100644 --- a/bonobo/config/__init__.py +++ b/bonobo/config/__init__.py @@ -5,10 +5,10 @@ configurable transformations, either class-based or function-based. """ from bonobo.config.configurables import Configurable -from bonobo.config.functools import transformation_factory, partial +from bonobo.config.functools import partial, transformation_factory from bonobo.config.options import Method, Option -from bonobo.config.processors import ContextProcessor, use_context, use_context_processor, use_raw_input, use_no_input -from bonobo.config.services import Container, Exclusive, Service, use, create_container +from bonobo.config.processors import ContextProcessor, use_context, use_context_processor, use_no_input, use_raw_input +from bonobo.config.services import Container, Exclusive, Service, create_container, use from bonobo.util import deprecated_alias requires = deprecated_alias("requires", use) diff --git a/bonobo/contrib/django/__init__.py b/bonobo/contrib/django/__init__.py index 24fe68d..459a6c8 100644 --- a/bonobo/contrib/django/__init__.py +++ b/bonobo/contrib/django/__init__.py @@ -6,7 +6,7 @@ This module contains all tools for Bonobo and Django to interract nicely. """ -from .utils import create_or_update from .commands import ETLCommand +from .utils import create_or_update __all__ = ["ETLCommand", "create_or_update"] diff --git a/bonobo/contrib/django/commands.py b/bonobo/contrib/django/commands.py index 4974540..e96eac4 100644 --- a/bonobo/contrib/django/commands.py +++ b/bonobo/contrib/django/commands.py @@ -2,13 +2,13 @@ from logging import getLogger from types import GeneratorType from colorama import Back, Fore, Style -from django.core.management import BaseCommand -from django.core.management.base import OutputWrapper from mondrian import term import bonobo from bonobo.plugins.console import ConsoleOutputPlugin from bonobo.util.term import CLEAR_EOL +from django.core.management import BaseCommand +from django.core.management.base import OutputWrapper from .utils import create_or_update diff --git a/bonobo/contrib/google/__init__.py b/bonobo/contrib/google/__init__.py index 126a468..449b58f 100644 --- a/bonobo/contrib/google/__init__.py +++ b/bonobo/contrib/google/__init__.py @@ -1,14 +1,15 @@ import os -# https://developers.google.com/api-client-library/python/guide/aaa_oauth -# pip install google-api-python-client (1.6.4) - import httplib2 from apiclient import discovery from oauth2client import client, tools from oauth2client.file import Storage from oauth2client.tools import argparser +# https://developers.google.com/api-client-library/python/guide/aaa_oauth +# pip install google-api-python-client (1.6.4) + + HOME_DIR = os.path.expanduser("~") GOOGLE_SECRETS = os.path.join(HOME_DIR, ".cache/secrets/client_secrets.json") diff --git a/bonobo/contrib/opendatasoft/__init__.py b/bonobo/contrib/opendatasoft/__init__.py index 0d00a57..d7377e2 100644 --- a/bonobo/contrib/opendatasoft/__init__.py +++ b/bonobo/contrib/opendatasoft/__init__.py @@ -3,8 +3,8 @@ from urllib.parse import urlencode import requests # todo: make this a service so we can substitute it ? from bonobo.config import Option -from bonobo.config.processors import ContextProcessor from bonobo.config.configurables import Configurable +from bonobo.config.processors import ContextProcessor from bonobo.util.objects import ValueHolder diff --git a/bonobo/examples/__init__.py b/bonobo/examples/__init__.py index ed8a873..3d5cc05 100644 --- a/bonobo/examples/__init__.py +++ b/bonobo/examples/__init__.py @@ -1,7 +1,7 @@ import os import bonobo -from bonobo.execution.strategies import STRATEGIES, DEFAULT_STRATEGY +from bonobo.execution.strategies import DEFAULT_STRATEGY, STRATEGIES from bonobo.util.statistics import Timer diff --git a/bonobo/execution/strategies/__init__.py b/bonobo/execution/strategies/__init__.py index 1d66554..5995bf0 100644 --- a/bonobo/execution/strategies/__init__.py +++ b/bonobo/execution/strategies/__init__.py @@ -7,9 +7,7 @@ at home if you want to give it a shot. """ from bonobo.execution.strategies.executor import ( - ProcessPoolExecutorStrategy, - ThreadPoolExecutorStrategy, - AsyncThreadPoolExecutorStrategy, + AsyncThreadPoolExecutorStrategy, ProcessPoolExecutorStrategy, ThreadPoolExecutorStrategy ) from bonobo.execution.strategies.naive import NaiveStrategy diff --git a/bonobo/util/__init__.py b/bonobo/util/__init__.py index 881fada..c3c33e2 100644 --- a/bonobo/util/__init__.py +++ b/bonobo/util/__init__.py @@ -6,17 +6,9 @@ and inspect transformations, graphs, and nodes. from bonobo.util.collections import cast, ensure_tuple, sortedlist, tuplize from bonobo.util.compat import deprecated, deprecated_alias from bonobo.util.inspect import ( - inspect_node, - isconfigurable, - isconfigurabletype, - iscontextprocessor, - isdict, - ismethod, - isoption, - istuple, - istype, + inspect_node, isconfigurable, isconfigurabletype, iscontextprocessor, isdict, ismethod, isoption, istuple, istype ) -from bonobo.util.objects import get_name, get_attribute_or_create, ValueHolder +from bonobo.util.objects import ValueHolder, get_attribute_or_create, get_name # Bonobo's util API __all__ = [ diff --git a/requirements-dev.txt b/requirements-dev.txt index eb8e9c5..4cc73b8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,11 +20,10 @@ jinja2==2.10.1 markupsafe==1.1.1 more-itertools==7.0.0 packaging==19.0 -pathlib2==2.3.3 pluggy==0.11.0 poyo==0.4.2 py==1.8.0 -pygments==2.3.1 +pygments==2.4.0 pyparsing==2.4.0 pytest-cov==2.7.1 pytest-timeout==1.3.3 diff --git a/requirements-docker.txt b/requirements-docker.txt index b29ec7f..223cc84 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -24,7 +24,6 @@ requests==2.21.0 semantic-version==2.6.0 six==1.12.0 stevedore==1.30.1 -typing==3.6.6 unidecode==1.0.23 urllib3==1.24.3 websocket-client==0.56.0 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 59cf1bd..f8b23f3 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -7,7 +7,7 @@ bleach==3.1.0 decorator==4.4.0 defusedxml==0.6.0 entrypoints==0.3 -ipykernel==5.1.0 +ipykernel==5.1.1 ipython-genutils==0.2.0 ipython==7.5.0 ipywidgets==6.0.1 @@ -30,8 +30,8 @@ pickleshare==0.7.5 prometheus-client==0.6.0 prompt-toolkit==2.0.9 ptyprocess==0.6.0 -pygments==2.3.1 -pyrsistent==0.15.1 +pygments==2.4.0 +pyrsistent==0.15.2 python-dateutil==2.8.0 pyzmq==18.0.1 qtconsole==4.4.4 diff --git a/requirements-sqlalchemy.txt b/requirements-sqlalchemy.txt index d297067..3e386b6 100644 --- a/requirements-sqlalchemy.txt +++ b/requirements-sqlalchemy.txt @@ -22,7 +22,6 @@ requests==2.21.0 six==1.12.0 sqlalchemy==1.3.3 stevedore==1.30.1 -typing==3.6.6 unidecode==1.0.23 urllib3==1.24.3 whistle==1.0.1 diff --git a/requirements.txt b/requirements.txt index d4c6c9f..7ea6f18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ idna==2.8 jinja2==2.10.1 markupsafe==1.1.1 mondrian==0.8.0 -packaging==17.1 +packaging==19.0 pbr==5.2.0 psutil==5.6.2 pyparsing==2.4.0 @@ -19,7 +19,6 @@ pytz==2019.1 requests==2.21.0 six==1.12.0 stevedore==1.30.1 -typing==3.6.6 unidecode==1.0.23 urllib3==1.24.3 whistle==1.0.1 diff --git a/setup.py b/setup.py index a91d362..16fb16c 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# Generated by Medikit 0.7.1 on 2019-05-08. +# Generated by Medikit 0.7.1 on 2019-05-16. # All changes will be overriden. # Edit Projectfile and run “make update” (or “medikit update”) to regenerate. @@ -70,7 +70,7 @@ setup( "graphviz (>= 0.8, < 0.9)", "jinja2 (~= 2.9)", "mondrian (~= 0.8)", - "packaging (~= 17.0)", + "packaging (~= 19.0)", "psutil (~= 5.4)", "python-slugify (~= 1.2.0)", "requests (~= 2.0)",