diff --git a/.codacy.yml b/.codacy.yml index 390e58f..0dad881 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -1,4 +1,11 @@ --- exclude_paths: - - bonobo/examples/ - - bonobo/ext/ + - benchmarks/** + - bin/** + - bonobo/contrib/jupyter/**.js + - bonobo/examples/** + - bonobo/ext/** + - bonobo/util/testing.py + - docs/** + - setup.py + - tests/** diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..428700d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 + +[*.py] +indent = ' ' +indent_size = 4 +indent_style = space +line_length = 120 +multi_line_output = 5 + +[Makefile] +indent_style = tab + diff --git a/Makefile b/Makefile index 0b29ce5..5f078ed 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Generated by Medikit 0.6.3 on 2018-07-29. +# Generated by Medikit 0.6.3 on 2018-08-11. # All changes will be overriden. # Edit Projectfile and run “make update” (or “medikit update”) to regenerate. @@ -26,8 +26,6 @@ SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build SPHINX_OPTIONS ?= SPHINX_SOURCEDIR ?= docs SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build -YAPF ?= $(PYTHON) -m yapf -YAPF_OPTIONS ?= -rip SPHINX_AUTOBUILD ?= $(PYTHON_DIRNAME)/sphinx-autobuild MEDIKIT ?= $(PYTHON) -m medikit MEDIKIT_UPDATE_OPTIONS ?= @@ -44,7 +42,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_INLINE) -r $(PYTHON_REQUIREMENTS_FILE) @mkdir -p .medikit; touch $@ endif @@ -62,7 +60,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DEV_INLINE) -r $(PYTHON_REQUIREMENTS_DEV_FILE) @mkdir -p .medikit; touch $@ endif @@ -79,7 +77,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DOCKER_INLINE) -r $(PYTHON_REQUIREMENTS_DOCKER_FILE) @mkdir -p .medikit; touch $@ endif @@ -93,7 +91,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_JUPYTER_INLINE) -r $(PYTHON_REQUIREMENTS_JUPYTER_FILE) @mkdir -p .medikit; touch $@ endif @@ -107,7 +105,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_SQLALCHEMY_INLINE) -r $(PYTHON_REQUIREMENTS_SQLALCHEMY_FILE) @mkdir -p .medikit; touch $@ endif @@ -118,15 +116,15 @@ test: install-dev ## Runs the test suite. $(SPHINX_SOURCEDIR): install-dev ## $(SPHINX_BUILD) -b html -D latex_paper_size=a4 $(SPHINX_OPTIONS) $(SPHINX_SOURCEDIR) $(SPHINX_BUILDDIR)/html -format: install-dev ## Reformats the whole python codebase using yapf. - $(YAPF) $(YAPF_OPTIONS) . - $(YAPF) $(YAPF_OPTIONS) Projectfile - watch-$(SPHINX_SOURCEDIR): ## $(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d) +format: ## Reformats the whole codebase using our standards (requires black and isort). + black -l 120 --skip-string-normalization . + isort -rc -o mondrian -o whistle -y . + medikit: # Checks installed medikit version and updates it if it is outdated. - @$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=10.0" "medikit>=$(MEDIKIT_VERSION)" + @$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=18.0" "medikit>=$(MEDIKIT_VERSION)" update: medikit ## Update project artifacts using medikit. $(MEDIKIT) update $(MEDIKIT_UPDATE_OPTIONS) diff --git a/Projectfile b/Projectfile index 6174a89..0355430 100644 --- a/Projectfile +++ b/Projectfile @@ -6,7 +6,6 @@ make = require('make') pytest = require('pytest') python = require('python') sphinx = require('sphinx') -yapf = require('yapf') python.setup( name='bonobo', @@ -47,7 +46,7 @@ python.add_requirements( 'fs ~=2.0', 'graphviz >=0.8,<0.9', 'jinja2 ~=2.9', - 'mondrian ~=0.7', + 'mondrian ~=0.8', 'packaging ~=17.0', 'psutil ~=5.4', 'python-slugify ~=1.2.0', @@ -74,14 +73,29 @@ python.add_requirements( @listen(make.on_generate) def on_make_generate(event): - event.makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild' - event.makefile.add_target( + makefile = event.makefile + + # Sphinx + makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild' + makefile.add_target( 'watch-$(SPHINX_SOURCEDIR)', - ''' - $(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d) - ''', + '$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)', phony=True ) + # Formating + makefile.add_target( + 'format', + ''' + black -l 120 --skip-string-normalization . + isort -rc -o mondrian -o whistle -y . + ''', + phony=True, + doc='Reformats the whole codebase using our standards (requires black and isort).' + ) + + + + # vim: ft=python: diff --git a/benchmarks/parameters.py b/benchmarks/parameters.py index f51e389..f8e63a1 100644 --- a/benchmarks/parameters.py +++ b/benchmarks/parameters.py @@ -47,10 +47,9 @@ if __name__ == '__main__': for i in 1, 2, 3: print( - 'j{}'.format(i), - timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i)) + 'j{}'.format(i), timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i)) ) print( 'k{}'.format(i), - timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i)) + timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i)), ) diff --git a/bin/update_apidoc.py b/bin/update_apidoc.py index a03dc63..4ed763f 100644 --- a/bin/update_apidoc.py +++ b/bin/update_apidoc.py @@ -1,6 +1,6 @@ import os -from jinja2 import Environment, DictLoader +from jinja2 import DictLoader, Environment __path__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__), '..')) @@ -18,11 +18,7 @@ class Module: return '<{} ({})>'.format(self.title, self.name) def asdict(self): - return { - 'name': self.name, - 'title': self.title, - 'automodule_options': self.automodule_options, - } + return {'name': self.name, 'title': self.title, 'automodule_options': self.automodule_options} def get_path(self): return os.path.join(__path__, apidoc_root, *self.name.split('.')) + '.rst' @@ -45,9 +41,9 @@ def underlined_filter(txt, chr): env = Environment( - loader=DictLoader({ - 'module': - ''' + loader=DictLoader( + { + 'module': ''' {{ (':mod:`'~title~' <'~name~'>`') | underlined('=') }} .. currentmodule:: {{ name }} @@ -56,8 +52,12 @@ env = Environment( .. automodule:: {{ name }} {% for opt in automodule_options %} :{{ opt }}:{{ "\n" }}{% endfor %} - ''' [1:-1] + '\n' - }) + '''[ + 1:-1 + ] + + '\n' + } + ) ) env.filters['underlined'] = underlined_filter diff --git a/bonobo/__init__.py b/bonobo/__init__.py index d605183..f3cdfc0 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -7,7 +7,8 @@ import sys -assert sys.version_info >= (3, 5), "Python 3.5+ is required to use Bonobo." +if sys.version_info < (3, 5): + raise RuntimeError('Python 3.5+ is required to use Bonobo.') from bonobo._api import ( run, @@ -58,15 +59,14 @@ __version__ = __version__ def _repr_html_(): """This allows to easily display a version snippet in Jupyter.""" - from bonobo.util.pkgs import bonobo_packages from bonobo.commands.version import get_versions return ( '
' '
{}
' '
{}
' - "
" - ).format(__logo__, "
".join(get_versions(all=True))) + '' + ).format(__logo__, '
'.join(get_versions(all=True))) del sys diff --git a/bonobo/_api.py b/bonobo/_api.py index 78576aa..043aa77 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -187,7 +187,7 @@ def get_examples_path(*pathsegments): import os import pathlib - return str(pathlib.Path(os.path.dirname(__file__), "examples", *pathsegments)) + return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments)) @api.register diff --git a/bonobo/_version.py b/bonobo/_version.py index 22049ab..a68d2bd 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = "0.6.2" +__version__ = '0.6.3' diff --git a/bonobo/commands/__init__.py b/bonobo/commands/__init__.py index e1c0d2d..7568ae3 100644 --- a/bonobo/commands/__init__.py +++ b/bonobo/commands/__init__.py @@ -26,7 +26,9 @@ def entrypoint(args=None): commands = {} - def register_extension(ext, commands=commands): + def register_extension(ext): + nonlocal commands + try: parser = subparsers.add_parser(ext.name) if isinstance(ext.plugin, type) and issubclass(ext.plugin, BaseCommand): diff --git a/bonobo/commands/base.py b/bonobo/commands/base.py index 54ec3c6..b1210d3 100644 --- a/bonobo/commands/base.py +++ b/bonobo/commands/base.py @@ -7,6 +7,7 @@ from contextlib import contextmanager import bonobo.util.environ from bonobo.util import get_name from bonobo.util.environ import get_argument_parser, parse_args +from mondrian import humanizer class BaseCommand: @@ -60,7 +61,8 @@ class BaseGraphCommand(BaseCommand): return options def handle(self, file, mod, **options): - options = self.parse_options(**options) + with humanizer.humanize(): + options = self.parse_options(**options) with self.read(file, mod, **options) as (graph, graph_execution_options, options): return self.do_handle(graph, **graph_execution_options, **options) diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py index c94d37b..125c4b2 100644 --- a/bonobo/commands/convert.py +++ b/bonobo/commands/convert.py @@ -2,6 +2,7 @@ import bonobo from bonobo.commands import BaseCommand from bonobo.registry import READER, WRITER, default_registry from bonobo.util.resolvers import _resolve_options, _resolve_transformations +from mondrian import humanizer class ConvertCommand(BaseCommand): @@ -48,6 +49,7 @@ class ConvertCommand(BaseCommand): help="Add a named option to the writer factory.", ) + @humanizer.humanize() def handle( self, input_filename, @@ -60,6 +62,8 @@ class ConvertCommand(BaseCommand): limit=None, transformation=None, ): + graph = bonobo.Graph() + reader_factory = default_registry.get_reader_factory_for(input_filename, format=reader) reader_kwargs = _resolve_options((option or []) + (reader_option or [])) @@ -78,7 +82,6 @@ class ConvertCommand(BaseCommand): transformations += _resolve_transformations(transformation) - graph = bonobo.Graph() graph.add_chain( reader_factory(input_filename, **reader_kwargs), *transformations, diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index 7713912..4342e72 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -3,6 +3,7 @@ import os from jinja2 import Environment, FileSystemLoader from bonobo.commands import BaseCommand +from mondrian import humanizer class InitCommand(BaseCommand): @@ -30,12 +31,16 @@ class InitCommand(BaseCommand): with open(filename, "w+") as f: f.write(template.render(name=name)) - self.logger.info("Generated {} using template {!r}.".format(filename, template_name)) + print( + humanizer.Success( + "Generated {} using template {!r}.".format(filename, template_name) + ) + ) def create_package(self, *, filename): - name, ext = os.path.splitext(filename) - if ext != "": - raise ValueError("Package names should not have an extension.") + _, ext = os.path.splitext(filename) + if ext != '': + raise ValueError('Package names should not have an extension.') try: import medikit.commands @@ -52,18 +57,23 @@ class InitCommand(BaseCommand): self.logger.info('Generated "{}" package with medikit.'.format(package_name)) self.create_file_from_template(template="default", filename=os.path.join(filename, package_name, "__main__.py")) - print('Your "{}" package has been created.'.format(package_name)) - print() - print("Install it...") - print() - print(" pip install --editable {}".format(filename)) - print() - print("Then maybe run the example...") - print() - print(" python -m {}".format(package_name)) - print() - print("Enjoy!") + print( + humanizer.Success( + 'Package "{}" has been created.'.format(package_name), + '', + "Install it...", + '', + " $ `pip install --editable {}`".format(filename), + '', + "Then maybe run the example...", + '', + " $ `python -m {}`".format(package_name), + '', + "Enjoy!" + ) + ) + @humanizer.humanize() def handle(self, *, template, filename, package=False, force=False): if os.path.exists(filename) and not force: raise FileExistsError("Target filename already exists, use --force to override.") diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index 70468ef..0a8d9ab 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -1,4 +1,5 @@ from bonobo.commands import BaseCommand +from mondrian import humanizer def get_versions(*, all=False, quiet=None): @@ -21,6 +22,7 @@ def get_versions(*, all=False, quiet=None): class VersionCommand(BaseCommand): + @humanizer.humanize() def handle(self, *, all=False, quiet=False): for line in get_versions(all=all, quiet=quiet): print(line) diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py index ab21a28..6e9b466 100644 --- a/bonobo/config/processors.py +++ b/bonobo/config/processors.py @@ -133,7 +133,7 @@ class ContextCurrifier: try: # todo yield from ? how to ? processor.send(self._stack_values.pop()) - except StopIteration as exc: + except StopIteration: # This is normal, and wanted. pass else: diff --git a/bonobo/config/services.py b/bonobo/config/services.py index d09928e..1fb38cd 100644 --- a/bonobo/config/services.py +++ b/bonobo/config/services.py @@ -21,32 +21,32 @@ class Service(Option): identifier. For example, you can create a Configurable that has a "database" Service in its attribute, meaning that you'll define which database to use, by name, when creating the instance of this class, then provide an implementation when running the graph using a strategy. - + Example:: - + import bonobo - + class QueryExtractor(bonobo.Configurable): database = bonobo.Service(default='sqlalchemy.engine.default') - + graph = bonobo.Graph( QueryExtractor(database='sqlalchemy.engine.secondary'), *more_transformations, ) - + if __name__ == '__main__': engine = create_engine('... dsn ...') bonobo.run(graph, services={ 'sqlalchemy.engine.secondary': engine }) - + The main goal is not to tie transformations to actual dependencies, so the same can be run in different contexts (stages like preprod, prod, or tenants like client1, client2, or anything you want). .. attribute:: name Service name will be used to retrieve the implementation at runtime. - + """ def __init__(self, name, __doc__=None): @@ -66,8 +66,13 @@ class Service(Option): class Container(dict): def __new__(cls, *args, **kwargs): if len(args) == 1: - assert not len(kwargs), "only one usage at a time, my dear." - if not (args[0]): + if len(kwargs): + raise ValueError( + 'You can either use {} with one positional argument or with keyword arguments, not both.'.format( + cls.__name__ + ) + ) + if not args[0]: return super().__new__(cls) if isinstance(args[0], cls): return cls diff --git a/bonobo/contrib/django/commands.py b/bonobo/contrib/django/commands.py index a74bcc1..076ac23 100644 --- a/bonobo/contrib/django/commands.py +++ b/bonobo/contrib/django/commands.py @@ -4,11 +4,11 @@ from types import GeneratorType from colorama import Back, Fore, Style from django.core.management import BaseCommand from django.core.management.base import OutputWrapper +from mondrian import term import bonobo from bonobo.plugins.console import ConsoleOutputPlugin from bonobo.util.term import CLEAR_EOL -from mondrian import term from .utils import create_or_update @@ -59,8 +59,9 @@ class ETLCommand(BaseCommand): graph_coll = (graph_coll,) for i, graph in enumerate(graph_coll): - assert isinstance(graph, bonobo.Graph), "Invalid graph provided." - print(term.lightwhite("{}. {}".format(i + 1, graph.name))) + if not isinstance(graph, bonobo.Graph): + raise ValueError('Expected a Graph instance, got {!r}.'.format(graph)) + print(term.lightwhite('{}. {}'.format(i + 1, graph.name))) result = bonobo.run(graph, services=services, strategy=strategy) results.append(result) print(term.lightblack(" ... return value: " + str(result))) diff --git a/bonobo/examples/datasets/coffeeshops.py b/bonobo/examples/datasets/coffeeshops.py index cd52d83..ac6552b 100644 --- a/bonobo/examples/datasets/coffeeshops.py +++ b/bonobo/examples/datasets/coffeeshops.py @@ -8,6 +8,11 @@ from bonobo.structs.graphs import PartialGraph def get_graph(graph=None, *, _limit=(), _print=()): + """ + Extracts a list of cafes with on euro in Paris, renames the name, address and zipcode fields, + reorders the fields and formats to json and csv files. + + """ graph = graph or bonobo.Graph() producer = ( diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index 5e665d5..49f242e 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -1,5 +1,5 @@ """ -Extracts a list of fablabs in the world, restrict to the ones in france, then format it both for a nice console output +Extracts a list of fablabs in the world, restricted to the ones in france, then format its both for a nice console output and a flat txt file. .. graphviz:: diff --git a/bonobo/execution/contexts/base.py b/bonobo/execution/contexts/base.py index c6117d1..9399d3f 100644 --- a/bonobo/execution/contexts/base.py +++ b/bonobo/execution/contexts/base.py @@ -2,16 +2,17 @@ import logging import sys from contextlib import contextmanager +from mondrian import term + from bonobo.util import deprecated from bonobo.util.objects import Wrapper, get_name -from mondrian import term @contextmanager def recoverable(error_handler): try: yield - except Exception as exc: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except error_handler(*sys.exc_info(), level=logging.ERROR) @@ -19,9 +20,9 @@ def recoverable(error_handler): def unrecoverable(error_handler): try: yield - except Exception as exc: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except error_handler(*sys.exc_info(), level=logging.ERROR) - raise # raise unrecoverableerror from exc ? + raise # raise unrecoverableerror from x ? class Lifecycle: diff --git a/bonobo/execution/contexts/node.py b/bonobo/execution/contexts/node.py index 9248356..30b0d5b 100644 --- a/bonobo/execution/contexts/node.py +++ b/bonobo/execution/contexts/node.py @@ -190,7 +190,7 @@ class NodeExecutionContext(BaseContext, WithStatistics): if self._stack: try: self._stack.teardown() - except Exception as exc: + except Exception: self.fatal(sys.exc_info()) super().stop() @@ -207,7 +207,7 @@ class NodeExecutionContext(BaseContext, WithStatistics): if self._input_type is not None: raise RuntimeError("Cannot override input type, already have %r.", self._input_type) - if type(input_type) is not type: + if not isinstance(input_type, type): raise UnrecoverableTypeError("Input types must be regular python types.") if not issubclass(input_type, tuple): diff --git a/bonobo/execution/strategies/__init__.py b/bonobo/execution/strategies/__init__.py index cb9944a..1d66554 100644 --- a/bonobo/execution/strategies/__init__.py +++ b/bonobo/execution/strategies/__init__.py @@ -28,7 +28,7 @@ DEFAULT_STRATEGY = "threadpool" def create_strategy(name=None): """ Create a strategy, or just returns it if it's already one. - + :param name: :return: Strategy """ diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index 9104a7c..e55f393 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -3,6 +3,8 @@ import html import itertools import pprint +from mondrian import term + from bonobo import settings from bonobo.config import Configurable, Method, Option, use_context, use_no_input, use_raw_input from bonobo.config.functools import transformation_factory @@ -10,7 +12,6 @@ from bonobo.config.processors import ContextProcessor, use_context_processor from bonobo.constants import NOT_MODIFIED from bonobo.util.objects import ValueHolder from bonobo.util.term import CLEAR_EOL -from mondrian import term __all__ = [ "FixedWindow", @@ -57,8 +58,6 @@ class Limit(Configurable): def Tee(f): - from bonobo.constants import NOT_MODIFIED - @functools.wraps(f) def wrapped(*args, **kwargs): nonlocal f diff --git a/bonobo/nodes/filter.py b/bonobo/nodes/filter.py index 88e3101..879e322 100644 --- a/bonobo/nodes/filter.py +++ b/bonobo/nodes/filter.py @@ -11,9 +11,9 @@ class Filter(Configurable): .. attribute:: filter A callable used to filter lines. - + If the callable returns a true-ish value, the input will be passed unmodified to the next items. - + Otherwise, it'll be burnt. """ diff --git a/bonobo/plugins/__init__.py b/bonobo/plugins/__init__.py index 68f91b2..c84b303 100644 --- a/bonobo/plugins/__init__.py +++ b/bonobo/plugins/__init__.py @@ -2,13 +2,13 @@ class Plugin: """ A plugin is an extension to the core behavior of bonobo. If you're writing transformations, you should not need to use this interface. - + For examples, you can read bonobo.plugins.console.ConsoleOutputPlugin, or bonobo.plugins.jupyter.JupyterOutputPlugin that respectively permits an interactive output on an ANSI console and a rich output in a jupyter notebook. Note that you most probably won't instanciate them by yourself at runtime, as it's the default behaviour of bonobo to use them if your in a compatible context (aka an interactive terminal for the console plugin, or a jupyter notebook for the notebook plugin.) - + Warning: THE PLUGIN API IS PRE-ALPHA AND WILL EVOLVE BEFORE 1.0, DO NOT RELY ON IT BEING STABLE! """ diff --git a/bonobo/settings.py b/bonobo/settings.py index 16f10a9..44928c7 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -7,7 +7,7 @@ from bonobo.errors import ValidationError def to_bool(s): if s is None: return False - if type(s) is bool: + if isinstance(s, bool): return s if len(s): if s.lower() in ("f", "false", "n", "no", "0"): @@ -31,7 +31,7 @@ class Setting: def __init__(self, name, default=None, validator=None, formatter=None): self.name = name - if default: + if default is not None: self.default = default if callable(default) else lambda: default else: self.default = lambda: None diff --git a/bonobo/structs/graphs.py b/bonobo/structs/graphs.py index 222ef69..10f88f4 100644 --- a/bonobo/structs/graphs.py +++ b/bonobo/structs/graphs.py @@ -36,7 +36,6 @@ class GraphCursor: "Expected something looking like a node, but got an Ellipsis (...). Did you forget to complete the graph?" ) - if len(nodes): chain = self.graph.add_chain(*nodes, _input=self.last) return GraphCursor(chain.graph, first=self.first, last=chain.output) diff --git a/bonobo/util/bags.py b/bonobo/util/bags.py index c0d221e..e8cfeff 100644 --- a/bonobo/util/bags.py +++ b/bonobo/util/bags.py @@ -125,7 +125,7 @@ def BagType(typename, fields, *, verbose=False, module=None): # message or automatically replace the field name with a valid name. attrs = tuple(map(_uniquify(_make_valid_attr_name), fields)) - if type(fields) is str: + if isinstance(fields, str): raise TypeError("BagType does not support providing fields as a string.") fields = list(map(str, fields)) typename = str(typename) @@ -133,6 +133,8 @@ def BagType(typename, fields, *, verbose=False, module=None): for i, name in enumerate([typename] + fields): if type(name) is not str: raise TypeError("Type names and field names must be strings, got {name!r}".format(name=name)) + if not isinstance(name, str): + raise TypeError('Type names and field names must be strings, got {name!r}'.format(name=name)) if not i: if not name.isidentifier(): raise ValueError("Type names must be valid identifiers: {name!r}".format(name=name)) diff --git a/bonobo/util/objects.py b/bonobo/util/objects.py index 847b10d..bfc39a7 100644 --- a/bonobo/util/objects.py +++ b/bonobo/util/objects.py @@ -24,7 +24,7 @@ class ValueHolder: For the sake of concistency, all operator methods have been implemented (see https://docs.python.org/3/reference/datamodel.html) or at least all in a certain category, but it feels like a more correct method should exist, like with a getattr-something on the value. Let's see later. - + """ def __init__(self, value): diff --git a/bonobo/util/term.py b/bonobo/util/term.py index 389f02f..ca7375d 100644 --- a/bonobo/util/term.py +++ b/bonobo/util/term.py @@ -1,2 +1,2 @@ CLEAR_EOL = "\033[0K" -MOVE_CURSOR_UP = lambda n: "\033[{}A".format(n) +MOVE_CURSOR_UP = '\033[{}A'.format diff --git a/docs/_templates/alabaster/support.py b/docs/_templates/alabaster/support.py index 9147b52..6788588 100644 --- a/docs/_templates/alabaster/support.py +++ b/docs/_templates/alabaster/support.py @@ -1,8 +1,9 @@ # flake8: noqa from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace, Punctuation, Other, Literal +from pygments.token import ( + Comment, Error, Generic, Keyword, Literal, Name, Number, Operator, Other, Punctuation, String, Whitespace +) # Originally based on FlaskyStyle which was based on 'tango'. @@ -12,7 +13,7 @@ class Alabaster(Style): styles = { # No corresponding class for the following: - #Text: "", # class: '' + # Text: "", # class: '' Whitespace: "underline #f8f8f8", # class: 'w' Error: "#a40000 border:#ef2929", # class: 'err' Other: "#000000", # class 'x' @@ -28,7 +29,6 @@ class Alabaster(Style): Operator: "#582800", # class: 'o' Operator.Word: "bold #004461", # class: 'ow' - like keywords Punctuation: "bold #000000", # class: 'p' - # because special names such as Name.Class, Name.Function, etc. # are not recognized as such later in the parsing, we choose them # to look the same as ordinary variables. diff --git a/docs/conf.py b/docs/conf.py index 877ebd7..73e6ec3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,10 +5,11 @@ import datetime import os import sys +import bonobo + sys.path.insert(0, os.path.abspath('..')) sys.path.insert(0, os.path.abspath('_themes')) -import bonobo # -- General configuration ------------------------------------------------ @@ -63,11 +64,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] autoclass_content = 'both' autodoc_member_order = 'groupwise' -autodoc_default_flags = [ - 'members', - 'undoc-members', - 'show-inheritance', -] +autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance'] add_module_names = False pygments_style = 'sphinx' @@ -112,7 +109,7 @@ html_sidebars = { 'sourcelink.html', 'searchbox.html', 'sidebarinfos.html', - ] + ], } html_theme_path = ['_themes'] @@ -137,15 +134,12 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -154,9 +148,7 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual'), -] +latex_documents = [(master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual')] # -- Options for manual page output --------------------------------------- @@ -171,9 +163,14 @@ man_pages = [(master_doc, 'bonobo', 'Bonobo Documentation', [author], 1)] # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, 'Bonobo', 'Bonobo Documentation', author, 'Bonobo', 'One line description of project.', - 'Miscellaneous' - ), + master_doc, + 'Bonobo', + 'Bonobo Documentation', + author, + 'Bonobo', + 'One line description of project.', + 'Miscellaneous', + ) ] # -- Options for Epub output ---------------------------------------------- @@ -209,4 +206,6 @@ rst_epilog = """ .. |longversion| replace:: v.{version} -""".format(version=version, ) +""".format( + version=version +) diff --git a/docs/reference/examples.rst b/docs/reference/examples.rst index b36c414..4edc910 100644 --- a/docs/reference/examples.rst +++ b/docs/reference/examples.rst @@ -5,7 +5,19 @@ There are a few examples bundled with **bonobo**. You'll find them under the :mod:`bonobo.examples` package, and you can run them directly as modules: - $ bonobo run -m bonobo.examples...module +.. code-block:: shell-session + + $ bonobo run -m bonobo.examples.module + + +or + +.. code-block:: shell-session + + $ python -m bonobo.examples.module + + + .. toctree:: :maxdepth: 4 diff --git a/docs/reference/examples/tutorials.rst b/docs/reference/examples/tutorials.rst deleted file mode 100644 index dbee6fd..0000000 --- a/docs/reference/examples/tutorials.rst +++ /dev/null @@ -1,50 +0,0 @@ -Examples from the tutorial -========================== - -Examples from :doc:`/tutorial/tut01` -:::::::::::::::::::::::::::::::::::: - -Example 1 ---------- - -.. automodule:: bonobo.examples.tutorials.tut01e01 - :members: - :undoc-members: - :show-inheritance: - -Example 2 ---------- - -.. automodule:: bonobo.examples.tutorials.tut01e02 - :members: - :undoc-members: - :show-inheritance: - -Examples from :doc:`/tutorial/tut02` -:::::::::::::::::::::::::::::::::::: - -Example 1: Read ---------------- - -.. automodule:: bonobo.examples.tutorials.tut02e01_read - :members: - :undoc-members: - :show-inheritance: - -Example 2: Write ----------------- - -.. automodule:: bonobo.examples.tutorials.tut02e02_write - :members: - :undoc-members: - :show-inheritance: - -Example 3: Write as map ------------------------ - -.. automodule:: bonobo.examples.tutorials.tut02e03_writeasmap - :members: - :undoc-members: - :show-inheritance: - - diff --git a/requirements-dev.txt b/requirements-dev.txt index 601aefd..366ecf1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,8 +18,9 @@ imagesize==1.0.0 jinja2-time==0.2.0 jinja2==2.10 markupsafe==1.0 -more-itertools==4.2.0 +more-itertools==4.3.0 packaging==17.1 +pathlib2==2.3.2 pluggy==0.7.1 poyo==0.4.1 py==1.5.4 @@ -27,7 +28,7 @@ pygments==2.2.0 pyparsing==2.2.0 pytest-cov==2.5.1 pytest-timeout==1.3.1 -pytest==3.6.4 +pytest==3.7.1 python-dateutil==2.7.3 pytz==2018.5 requests==2.19.1 @@ -38,4 +39,3 @@ sphinx==1.7.6 sphinxcontrib-websupport==1.1.0 urllib3==1.23 whichcraft==0.4.1 -yapf==0.22.0 diff --git a/requirements-docker.txt b/requirements-docker.txt index bce3bba..389588d 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -7,12 +7,12 @@ chardet==3.0.4 colorama==0.3.9 docker-pycreds==0.3.0 docker==2.7.0 -fs==2.0.26 +fs==2.0.27 graphviz==0.8.4 idna==2.7 jinja2==2.10 markupsafe==1.0 -mondrian==0.7.0 +mondrian==0.8.0 packaging==17.1 pbr==4.2.0 psutil==5.4.6 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 7f6183f..f96b3b6 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -8,7 +8,7 @@ entrypoints==0.2.3 html5lib==1.0.1 ipykernel==4.8.2 ipython-genutils==0.2.0 -ipython==6.4.0 +ipython==6.5.0 ipywidgets==6.0.1 jedi==0.12.1 jinja2==2.10 @@ -26,12 +26,12 @@ pandocfilters==1.4.2 parso==0.3.1 pexpect==4.6.0 pickleshare==0.7.4 -prometheus-client==0.3.0 +prometheus-client==0.3.1 prompt-toolkit==1.0.15 ptyprocess==0.6.0 pygments==2.2.0 python-dateutil==2.7.3 -pyzmq==17.1.0 +pyzmq==17.1.2 qtconsole==4.3.1 send2trash==1.5.0 simplegeneric==0.8.1 diff --git a/requirements-sqlalchemy.txt b/requirements-sqlalchemy.txt index 724a70d..d6c7845 100644 --- a/requirements-sqlalchemy.txt +++ b/requirements-sqlalchemy.txt @@ -5,12 +5,12 @@ bonobo-sqlalchemy==0.6.0 certifi==2018.4.16 chardet==3.0.4 colorama==0.3.9 -fs==2.0.26 +fs==2.0.27 graphviz==0.8.4 idna==2.7 jinja2==2.10 markupsafe==1.0 -mondrian==0.7.0 +mondrian==0.8.0 packaging==17.1 pbr==4.2.0 psutil==5.4.6 diff --git a/requirements.txt b/requirements.txt index 9018ecd..a3dc418 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,12 +4,12 @@ cached-property==1.4.3 certifi==2018.4.16 chardet==3.0.4 colorama==0.3.9 -fs==2.0.26 +fs==2.0.27 graphviz==0.8.4 idna==2.7 jinja2==2.10 markupsafe==1.0 -mondrian==0.7.0 +mondrian==0.8.0 packaging==17.1 pbr==4.2.0 psutil==5.4.6 diff --git a/setup.py b/setup.py index 5e0be32..daeab3e 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# Generated by Medikit 0.6.3 on 2018-07-29. +# Generated by Medikit 0.6.3 on 2018-08-11. # All changes will be overriden. # Edit Projectfile and run “make update” (or “medikit update”) to regenerate. @@ -44,14 +44,14 @@ else: setup( author='Romain Dorgueil', author_email='romain@dorgueil.net', - data_files=[( - 'share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/contrib/jupyter/static/extension.js', 'bonobo/contrib/jupyter/static/index.js', - 'bonobo/contrib/jupyter/static/index.js.map' - ] - )], - description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' - 'python 3.5+.'), + data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/contrib/jupyter/static/extension.js', + 'bonobo/contrib/jupyter/static/index.js', + 'bonobo/contrib/jupyter/static/index.js.map' + ])], + description=( + 'Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' + 'python 3.5+.'), license='Apache License, Version 2.0', name='bonobo', python_requires='>=3.5', @@ -61,14 +61,17 @@ setup( packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, install_requires=[ - 'cached-property (~= 1.4)', 'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (~= 2.9)', 'mondrian (~= 0.7)', - 'packaging (~= 17.0)', 'psutil (~= 5.4)', 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)', + 'cached-property (~= 1.4)', 'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', + 'jinja2 (~= 2.9)', 'mondrian (~= 0.8)', 'packaging (~= 17.0)', + 'psutil (~= 5.4)', 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)', 'stevedore (~= 1.27)', 'whistle (~= 1.0)' ], extras_require={ 'dev': [ - 'cookiecutter (>= 1.5, < 1.6)', 'coverage (~= 4.4)', 'pytest (~= 3.4)', 'pytest-cov (~= 2.5)', - 'pytest-timeout (>= 1, < 2)', 'sphinx (~= 1.7)', 'sphinx-sitemap (>= 0.2, < 0.3)', 'yapf' + 'cookiecutter (>= 1.5, < 1.6)', 'coverage (~= 4.4)', + 'pytest (~= 3.4)', 'pytest-cov (~= 2.5)', + 'pytest-timeout (>= 1, < 2)', 'sphinx (~= 1.7)', + 'sphinx-sitemap (>= 0.2, < 0.3)' ], 'docker': ['bonobo-docker (~= 0.6.0a1)'], 'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'], @@ -76,13 +79,17 @@ setup( }, entry_points={ 'bonobo.commands': [ - 'convert = bonobo.commands.convert:ConvertCommand', 'download = bonobo.commands.download:DownloadCommand', - 'examples = bonobo.commands.examples:ExamplesCommand', 'init = bonobo.commands.init:InitCommand', - 'inspect = bonobo.commands.inspect:InspectCommand', 'run = bonobo.commands.run:RunCommand', + 'convert = bonobo.commands.convert:ConvertCommand', + 'download = bonobo.commands.download:DownloadCommand', + 'examples = bonobo.commands.examples:ExamplesCommand', + 'init = bonobo.commands.init:InitCommand', + 'inspect = bonobo.commands.inspect:InspectCommand', + 'run = bonobo.commands.run:RunCommand', 'version = bonobo.commands.version:VersionCommand' ], 'console_scripts': ['bonobo = bonobo.commands:entrypoint'] }, url='https://www.bonobo-project.org/', - download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), + download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'. + format(version=version), ) diff --git a/tests/config/test_services.py b/tests/config/test_services.py index ffb815b..20b0ee2 100644 --- a/tests/config/test_services.py +++ b/tests/config/test_services.py @@ -130,7 +130,21 @@ def test_requires(): assert svcargs["output"] == vcr.append -@pytest.mark.parametrize("services", [None, {}]) +def test_constructor(): + c1 = Container(foo='foo', bar='bar') + assert 2 == len(c1) + + c2 = Container({'foo': 'foo', 'bar': 'bar'}) + assert 2 == len(c2) + + assert c1['foo'] == c2['foo'] + assert c1['bar'] == c2['bar'] + + with pytest.raises(ValueError): + Container({'bar': 'bar'}, foo='foo') + + +@pytest.mark.parametrize('services', [None, {}]) def test_create_container_empty_values(services): c = create_container(services) assert len(c) == 2 diff --git a/tests/nodes/io/test_csv.py b/tests/nodes/io/test_csv.py index 3173768..0f853a3 100644 --- a/tests/nodes/io/test_csv.py +++ b/tests/nodes/io/test_csv.py @@ -63,9 +63,9 @@ class CsvReaderTest(Csv, ReaderTest, TestCase): def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - self.check_output(context, prepend=[("id", "name")]) + self.check_output(context, prepend=[('id', 'name')]) - @incontext(output_fields=("x", "y"), skip=1) + @incontext(output_fields=('x', 'y'), skip=1) def test_output_fields(self, context): context.write_sync(EMPTY) context.stop() @@ -103,7 +103,7 @@ class CsvWriterTest(Csv, WriterTest, TestCase): context.write_sync((L1, L2), (L3, L4)) context.stop() - assert self.readlines() == ("a,hey", "b,bee", "c,see", "d,dee") + assert self.readlines() == ('a,hey', 'b,bee', 'c,see', 'd,dee') @incontext() def test_nofields_multiple_args_length_mismatch(self, context): diff --git a/tests/nodes/io/test_json.py b/tests/nodes/io/test_json.py index fac797f..d432f8f 100644 --- a/tests/nodes/io/test_json.py +++ b/tests/nodes/io/test_json.py @@ -14,6 +14,7 @@ FOOBAZ = {"foo": "baz"} incontext = ConfigurableNodeTest.incontext + ### # Standard JSON Readers / Writers ### diff --git a/tests/test_settings.py b/tests/test_settings.py index 4e49e57..d34bd24 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest from bonobo import settings +from bonobo.errors import ValidationError TEST_SETTING = "TEST_SETTING" @@ -38,6 +39,15 @@ def test_setting(): s.clear() assert s.get() == "hello" + s = settings.Setting(TEST_SETTING, default=0, validator=lambda x: x == 42) + with pytest.raises(ValidationError): + assert s.get() is 0 + + s.set(42) + + with pytest.raises(ValidationError): + s.set(21) + def test_default_settings(): settings.clear_all()