style: switching from yapf to isort/black

This commit is contained in:
Romain Dorgueil
2018-08-11 06:34:37 +02:00
parent ebba06822b
commit d1c9beae97
93 changed files with 805 additions and 816 deletions

17
.editorconfig Normal file
View File

@ -0,0 +1,17 @@
root = true
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
[*.py]
indent = ' '
indent_size = 4
indent_style = space
line_length = 120
multi_line_output = 5
[Makefile]
indent_style = tab

View File

@ -1,4 +1,4 @@
# Generated by Medikit 0.6.1 on 2018-05-21. # Generated by Medikit 0.6.3 on 2018-08-11.
# All changes will be overriden. # All changes will be overriden.
# Edit Projectfile and run “make update” (or “medikit update”) to regenerate. # Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
@ -26,12 +26,10 @@ SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build
SPHINX_OPTIONS ?= SPHINX_OPTIONS ?=
SPHINX_SOURCEDIR ?= docs SPHINX_SOURCEDIR ?= docs
SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
YAPF ?= $(PYTHON) -m yapf
YAPF_OPTIONS ?= -rip
SPHINX_AUTOBUILD ?= $(PYTHON_DIRNAME)/sphinx-autobuild SPHINX_AUTOBUILD ?= $(PYTHON_DIRNAME)/sphinx-autobuild
MEDIKIT ?= $(PYTHON) -m medikit MEDIKIT ?= $(PYTHON) -m medikit
MEDIKIT_UPDATE_OPTIONS ?= MEDIKIT_UPDATE_OPTIONS ?=
MEDIKIT_VERSION ?= 0.6.1 MEDIKIT_VERSION ?= 0.6.3
.PHONY: $(SPHINX_SOURCEDIR) clean format help install install-dev install-docker install-jupyter install-sqlalchemy medikit quick test update update-requirements watch-$(SPHINX_SOURCEDIR) .PHONY: $(SPHINX_SOURCEDIR) clean format help install install-dev install-docker install-jupyter install-sqlalchemy medikit quick test update update-requirements watch-$(SPHINX_SOURCEDIR)
@ -44,7 +42,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else else
@printf "Applying \033[36m%s\033[0m target...\n" $(target) @printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_INLINE) -r $(PYTHON_REQUIREMENTS_FILE) $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_INLINE) -r $(PYTHON_REQUIREMENTS_FILE)
@mkdir -p .medikit; touch $@ @mkdir -p .medikit; touch $@
endif endif
@ -62,7 +60,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else else
@printf "Applying \033[36m%s\033[0m target...\n" $(target) @printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DEV_INLINE) -r $(PYTHON_REQUIREMENTS_DEV_FILE) $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DEV_INLINE) -r $(PYTHON_REQUIREMENTS_DEV_FILE)
@mkdir -p .medikit; touch $@ @mkdir -p .medikit; touch $@
endif endif
@ -79,7 +77,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else else
@printf "Applying \033[36m%s\033[0m target...\n" $(target) @printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DOCKER_INLINE) -r $(PYTHON_REQUIREMENTS_DOCKER_FILE) $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DOCKER_INLINE) -r $(PYTHON_REQUIREMENTS_DOCKER_FILE)
@mkdir -p .medikit; touch $@ @mkdir -p .medikit; touch $@
endif endif
@ -93,7 +91,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else else
@printf "Applying \033[36m%s\033[0m target...\n" $(target) @printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_JUPYTER_INLINE) -r $(PYTHON_REQUIREMENTS_JUPYTER_FILE) $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_JUPYTER_INLINE) -r $(PYTHON_REQUIREMENTS_JUPYTER_FILE)
@mkdir -p .medikit; touch $@ @mkdir -p .medikit; touch $@
endif endif
@ -107,7 +105,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else else
@printf "Applying \033[36m%s\033[0m target...\n" $(target) @printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_SQLALCHEMY_INLINE) -r $(PYTHON_REQUIREMENTS_SQLALCHEMY_FILE) $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_SQLALCHEMY_INLINE) -r $(PYTHON_REQUIREMENTS_SQLALCHEMY_FILE)
@mkdir -p .medikit; touch $@ @mkdir -p .medikit; touch $@
endif endif
@ -118,15 +116,15 @@ test: install-dev ## Runs the test suite.
$(SPHINX_SOURCEDIR): install-dev ## $(SPHINX_SOURCEDIR): install-dev ##
$(SPHINX_BUILD) -b html -D latex_paper_size=a4 $(SPHINX_OPTIONS) $(SPHINX_SOURCEDIR) $(SPHINX_BUILDDIR)/html $(SPHINX_BUILD) -b html -D latex_paper_size=a4 $(SPHINX_OPTIONS) $(SPHINX_SOURCEDIR) $(SPHINX_BUILDDIR)/html
format: install-dev ## Reformats the whole python codebase using yapf.
$(YAPF) $(YAPF_OPTIONS) .
$(YAPF) $(YAPF_OPTIONS) Projectfile
watch-$(SPHINX_SOURCEDIR): ## watch-$(SPHINX_SOURCEDIR): ##
$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d) $(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)
format: ## Reformats the whole codebase using our standards (requires black and isort).
black -l 120 --skip-string-normalization .
isort -rc -o mondrian -o whistle -y .
medikit: # Checks installed medikit version and updates it if it is outdated. medikit: # Checks installed medikit version and updates it if it is outdated.
@$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=10.0" "medikit>=$(MEDIKIT_VERSION)" @$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=18.0" "medikit>=$(MEDIKIT_VERSION)"
update: medikit ## Update project artifacts using medikit. update: medikit ## Update project artifacts using medikit.
$(MEDIKIT) update $(MEDIKIT_UPDATE_OPTIONS) $(MEDIKIT) update $(MEDIKIT_UPDATE_OPTIONS)

View File

@ -6,7 +6,6 @@ make = require('make')
pytest = require('pytest') pytest = require('pytest')
python = require('python') python = require('python')
sphinx = require('sphinx') sphinx = require('sphinx')
yapf = require('yapf')
python.setup( python.setup(
name='bonobo', name='bonobo',
@ -72,14 +71,29 @@ python.add_requirements(
@listen(make.on_generate) @listen(make.on_generate)
def on_make_generate(event): def on_make_generate(event):
event.makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild' makefile = event.makefile
event.makefile.add_target(
# Sphinx
makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild'
makefile.add_target(
'watch-$(SPHINX_SOURCEDIR)', 'watch-$(SPHINX_SOURCEDIR)',
''' '$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)',
$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)
''',
phony=True phony=True
) )
# Formating
makefile.add_target(
'format',
'''
black -l 120 --skip-string-normalization .
isort -rc -o mondrian -o whistle -y .
''',
phony=True,
doc='Reformats the whole codebase using our standards (requires black and isort).'
)
# vim: ft=python: # vim: ft=python:

View File

@ -47,10 +47,9 @@ if __name__ == '__main__':
for i in 1, 2, 3: for i in 1, 2, 3:
print( print(
'j{}'.format(i), 'j{}'.format(i), timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i))
timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i))
) )
print( print(
'k{}'.format(i), 'k{}'.format(i),
timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i)) timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i)),
) )

View File

@ -1,6 +1,6 @@
import os import os
from jinja2 import Environment, DictLoader from jinja2 import DictLoader, Environment
__path__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__), '..')) __path__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__), '..'))
@ -18,11 +18,7 @@ class Module:
return '<{} ({})>'.format(self.title, self.name) return '<{} ({})>'.format(self.title, self.name)
def asdict(self): def asdict(self):
return { return {'name': self.name, 'title': self.title, 'automodule_options': self.automodule_options}
'name': self.name,
'title': self.title,
'automodule_options': self.automodule_options,
}
def get_path(self): def get_path(self):
return os.path.join(__path__, apidoc_root, *self.name.split('.')) + '.rst' return os.path.join(__path__, apidoc_root, *self.name.split('.')) + '.rst'
@ -45,9 +41,9 @@ def underlined_filter(txt, chr):
env = Environment( env = Environment(
loader=DictLoader({ loader=DictLoader(
'module': {
''' 'module': '''
{{ (':mod:`'~title~' <'~name~'>`') | underlined('=') }} {{ (':mod:`'~title~' <'~name~'>`') | underlined('=') }}
.. currentmodule:: {{ name }} .. currentmodule:: {{ name }}
@ -56,8 +52,12 @@ env = Environment(
.. automodule:: {{ name }} .. automodule:: {{ name }}
{% for opt in automodule_options %} :{{ opt }}:{{ "\n" }}{% endfor %} {% for opt in automodule_options %} :{{ opt }}:{{ "\n" }}{% endfor %}
''' [1:-1] + '\n' '''[
}) 1:-1
]
+ '\n'
}
)
) )
env.filters['underlined'] = underlined_filter env.filters['underlined'] = underlined_filter

View File

@ -31,9 +31,7 @@ def _repr_html_():
' <div style="float: left; width: 20px; height: 20px;">{}</div>' ' <div style="float: left; width: 20px; height: 20px;">{}</div>'
' <pre style="white-space: nowrap; padding-left: 8px">{}</pre>' ' <pre style="white-space: nowrap; padding-left: 8px">{}</pre>'
'</div>' '</div>'
).format( ).format(__logo__, '<br/>'.join(get_versions(all=True)))
__logo__, '<br/>'.join(get_versions(all=True))
)
del sys del sys

View File

@ -8,11 +8,11 @@ to another is maximal.
""" """
from bonobo.execution.strategies import create_strategy from bonobo.execution.strategies import create_strategy
from bonobo.nodes import __all__ as _all_nodes
from bonobo.nodes import * from bonobo.nodes import *
from bonobo.nodes import __all__ as _all_nodes
from bonobo.structs import Graph from bonobo.structs import Graph
from bonobo.util.api import ApiHelper from bonobo.util.api import ApiHelper
from bonobo.util.environ import parse_args, get_argument_parser from bonobo.util.environ import get_argument_parser, parse_args
__all__ = [] __all__ = []
@ -44,14 +44,17 @@ def run(graph, *, plugins=None, services=None, strategy=None):
plugins = plugins or [] plugins = plugins or []
from bonobo import settings from bonobo import settings
settings.check() settings.check()
if not settings.QUIET.get(): # pragma: no cover if not settings.QUIET.get(): # pragma: no cover
if _is_interactive_console(): if _is_interactive_console():
import mondrian import mondrian
mondrian.setup(excepthook=True) mondrian.setup(excepthook=True)
from bonobo.plugins.console import ConsoleOutputPlugin from bonobo.plugins.console import ConsoleOutputPlugin
if ConsoleOutputPlugin not in plugins: if ConsoleOutputPlugin not in plugins:
plugins.append(ConsoleOutputPlugin) plugins.append(ConsoleOutputPlugin)
@ -60,6 +63,7 @@ def run(graph, *, plugins=None, services=None, strategy=None):
from bonobo.contrib.jupyter import JupyterOutputPlugin from bonobo.contrib.jupyter import JupyterOutputPlugin
except ImportError: except ImportError:
import logging import logging
logging.warning( logging.warning(
'Failed to load jupyter widget. Easiest way is to install the optional "jupyter" ' 'Failed to load jupyter widget. Easiest way is to install the optional "jupyter" '
'dependencies with «pip install bonobo[jupyter]», but you can also install a specific ' 'dependencies with «pip install bonobo[jupyter]», but you can also install a specific '
@ -70,6 +74,7 @@ def run(graph, *, plugins=None, services=None, strategy=None):
plugins.append(JupyterOutputPlugin) plugins.append(JupyterOutputPlugin)
import logging import logging
logging.getLogger().setLevel(settings.LOGGING_LEVEL.get()) logging.getLogger().setLevel(settings.LOGGING_LEVEL.get())
strategy = create_strategy(strategy) strategy = create_strategy(strategy)
return strategy.execute(graph, plugins=plugins, services=services) return strategy.execute(graph, plugins=plugins, services=services)
@ -158,6 +163,7 @@ api.register_group(
def _is_interactive_console(): def _is_interactive_console():
import sys import sys
return sys.stdout.isatty() return sys.stdout.isatty()
@ -172,6 +178,7 @@ def _is_jupyter_notebook():
def get_examples_path(*pathsegments): def get_examples_path(*pathsegments):
import os import os
import pathlib import pathlib
return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments)) return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments))

View File

@ -42,6 +42,7 @@ def entrypoint(args=None):
logger.exception('Error while loading command {}.'.format(ext.name)) logger.exception('Error while loading command {}.'.format(ext.name))
from stevedore import ExtensionManager from stevedore import ExtensionManager
mgr = ExtensionManager(namespace='bonobo.commands') mgr = ExtensionManager(namespace='bonobo.commands')
mgr.map(register_extension) mgr.map(register_extension)

View File

@ -41,6 +41,7 @@ class BaseGraphCommand(BaseCommand):
Base class for CLI commands that depends on a graph definition, either from a file or from a module. Base class for CLI commands that depends on a graph definition, either from a file or from a module.
""" """
required = True required = True
handler = None handler = None

View File

@ -1,7 +1,7 @@
import bonobo import bonobo
from bonobo.commands import BaseCommand from bonobo.commands import BaseCommand
from bonobo.registry import READER, WRITER, default_registry from bonobo.registry import READER, WRITER, default_registry
from bonobo.util.resolvers import _resolve_transformations, _resolve_options from bonobo.util.resolvers import _resolve_options, _resolve_transformations
class ConvertCommand(BaseCommand): class ConvertCommand(BaseCommand):
@ -11,21 +11,14 @@ class ConvertCommand(BaseCommand):
parser.add_argument( parser.add_argument(
'--' + READER, '--' + READER,
'-r', '-r',
help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.' help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.',
) )
parser.add_argument( parser.add_argument(
'--' + WRITER, '--' + WRITER,
'-w', '-w',
help= help='Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).',
'Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).'
)
parser.add_argument(
'--limit',
'-l',
type=int,
help='Adds a Limit() after the reader instance.',
default=None,
) )
parser.add_argument('--limit', '-l', type=int, help='Adds a Limit() after the reader instance.', default=None)
parser.add_argument( parser.add_argument(
'--transformation', '--transformation',
'-t', '-t',
@ -56,16 +49,16 @@ class ConvertCommand(BaseCommand):
) )
def handle( def handle(
self, self,
input_filename, input_filename,
output_filename, output_filename,
reader=None, reader=None,
reader_option=None, reader_option=None,
writer=None, writer=None,
writer_option=None, writer_option=None,
option=None, option=None,
limit=None, limit=None,
transformation=None, transformation=None,
): ):
reader_factory = default_registry.get_reader_factory_for(input_filename, format=reader) reader_factory = default_registry.get_reader_factory_for(input_filename, format=reader)
reader_kwargs = _resolve_options((option or []) + (reader_option or [])) reader_kwargs = _resolve_options((option or []) + (reader_option or []))
@ -75,13 +68,13 @@ class ConvertCommand(BaseCommand):
writer_args = () writer_args = ()
else: else:
writer_factory = default_registry.get_writer_factory_for(output_filename, format=writer) writer_factory = default_registry.get_writer_factory_for(output_filename, format=writer)
writer_args = (output_filename, ) writer_args = (output_filename,)
writer_kwargs = _resolve_options((option or []) + (writer_option or [])) writer_kwargs = _resolve_options((option or []) + (writer_option or []))
transformations = () transformations = ()
if limit: if limit:
transformations += (bonobo.Limit(limit), ) transformations += (bonobo.Limit(limit),)
transformations += _resolve_transformations(transformation) transformations += _resolve_transformations(transformation)
@ -92,8 +85,4 @@ class ConvertCommand(BaseCommand):
writer_factory(*writer_args, **writer_kwargs), writer_factory(*writer_args, **writer_kwargs),
) )
return bonobo.run( return bonobo.run(graph, services={'fs': bonobo.open_fs()})
graph, services={
'fs': bonobo.open_fs(),
}
)

View File

@ -19,6 +19,7 @@ class RunCommand(BaseGraphCommand):
def parse_options(self, *, quiet=False, verbose=False, install=False, **options): def parse_options(self, *, quiet=False, verbose=False, install=False, **options):
from bonobo import settings from bonobo import settings
settings.QUIET.set_if_true(quiet) settings.QUIET.set_if_true(quiet)
settings.DEBUG.set_if_true(verbose) settings.DEBUG.set_if_true(verbose)
self.install = install self.install = install
@ -65,4 +66,5 @@ def _install_requirements(requirements):
# python interpreter. # python interpreter.
pip.utils.pkg_resources = importlib.reload(pip.utils.pkg_resources) pip.utils.pkg_resources = importlib.reload(pip.utils.pkg_resources)
import site import site
importlib.reload(site) importlib.reload(site)

View File

@ -32,10 +32,11 @@ class VersionCommand(BaseCommand):
def _format_version(mod, *, name=None, quiet=False): def _format_version(mod, *, name=None, quiet=False):
from bonobo.util.pkgs import bonobo_packages from bonobo.util.pkgs import bonobo_packages
args = { args = {
'name': name or mod.__name__, 'name': name or mod.__name__,
'version': mod.__version__, 'version': mod.__version__,
'location': bonobo_packages[name or mod.__name__].location 'location': bonobo_packages[name or mod.__name__].location,
} }
if not quiet: if not quiet:

View File

@ -1,9 +1,7 @@
from bonobo.errors import AbstractError from bonobo.errors import AbstractError
from bonobo.util import isoption, iscontextprocessor, sortedlist, get_name from bonobo.util import get_name, iscontextprocessor, isoption, sortedlist
__all__ = [ __all__ = ['Configurable']
'Configurable',
]
get_creation_counter = lambda v: v._creation_counter get_creation_counter = lambda v: v._creation_counter
@ -64,10 +62,7 @@ class ConfigurableMeta(type):
return cls.__processors_cache return cls.__processors_cache
def __repr__(self): def __repr__(self):
return ' '.join(( return ' '.join(('<Configurable', super(ConfigurableMeta, self).__repr__().split(' ', 1)[1]))
'<Configurable',
super(ConfigurableMeta, self).__repr__().split(' ', 1)[1],
))
try: try:
@ -156,8 +151,10 @@ class Configurable(metaclass=ConfigurableMeta):
if len(extraneous): if len(extraneous):
raise TypeError( raise TypeError(
'{}() got {} unexpected option{}: {}.'.format( '{}() got {} unexpected option{}: {}.'.format(
cls.__name__, len(extraneous), 's' cls.__name__,
if len(extraneous) > 1 else '', ', '.join(map(repr, sorted(extraneous))) len(extraneous),
's' if len(extraneous) > 1 else '',
', '.join(map(repr, sorted(extraneous))),
) )
) )
@ -167,8 +164,10 @@ class Configurable(metaclass=ConfigurableMeta):
if _final: if _final:
raise TypeError( raise TypeError(
'{}() missing {} required option{}: {}.'.format( '{}() missing {} required option{}: {}.'.format(
cls.__name__, len(missing), 's' cls.__name__,
if len(missing) > 1 else '', ', '.join(map(repr, sorted(missing))) len(missing),
's' if len(missing) > 1 else '',
', '.join(map(repr, sorted(missing))),
) )
) )
return PartiallyConfigured(cls, *args, **kwargs) return PartiallyConfigured(cls, *args, **kwargs)

View File

@ -1,5 +1,4 @@
import functools import functools
import itertools import itertools

View File

@ -113,8 +113,9 @@ class RemovedOption(Option):
def clean(self, value): def clean(self, value):
if value != self.value: if value != self.value:
raise ValueError( raise ValueError(
'Removed options cannot change value, {!r} must now be {!r} (and you should remove setting the value explicitely, as it is deprecated and will be removed quite soon.'. 'Removed options cannot change value, {!r} must now be {!r} (and you should remove setting the value explicitely, as it is deprecated and will be removed quite soon.'.format(
format(self.name, self.value) self.name, self.value
)
) )
return self.value return self.value
@ -195,9 +196,7 @@ class Method(Option):
if not callable(value): if not callable(value):
raise TypeError( raise TypeError(
'Option {!r} ({}) is expecting a callable value, got {!r} object: {!r}.'.format( 'Option {!r} ({}) is expecting a callable value, got {!r} object: {!r}.'.format(
self.name, self.name, type(self).__name__, type(value).__name__, value
type(self).__name__,
type(value).__name__, value
) )
) )
inst._options_values[self.name] = self.type(value) if self.type else value inst._options_values[self.name] = self.type(value) if self.type else value

View File

@ -101,15 +101,17 @@ class ContextCurrifier:
try: try:
bound = self._bind(_input) bound = self._bind(_input)
except TypeError as exc: except TypeError as exc:
raise UnrecoverableTypeError(( raise UnrecoverableTypeError(
'Input of {wrapped!r} does not bind to the node signature.\n' (
'Args: {args}\n' 'Input of {wrapped!r} does not bind to the node signature.\n'
'Input: {input}\n' 'Args: {args}\n'
'Kwargs: {kwargs}\n' 'Input: {input}\n'
'Signature: {sig}' 'Kwargs: {kwargs}\n'
).format( 'Signature: {sig}'
wrapped=self.wrapped, args=self.args, input=_input, kwargs=self.kwargs, sig=signature(self.wrapped) ).format(
)) from exc wrapped=self.wrapped, args=self.args, input=_input, kwargs=self.kwargs, sig=signature(self.wrapped)
)
) from exc
return self.wrapped(*bound.args, **bound.kwargs) return self.wrapped(*bound.args, **bound.kwargs)
def setup(self, *context): def setup(self, *context):

View File

@ -112,10 +112,12 @@ def create_container(services=None, factory=Container):
if not 'fs' in container: if not 'fs' in container:
import bonobo import bonobo
container.setdefault('fs', bonobo.open_fs()) container.setdefault('fs', bonobo.open_fs())
if not 'http' in container: if not 'http' in container:
import requests import requests
container.setdefault('http', requests) container.setdefault('http', requests)
return container return container
@ -139,6 +141,7 @@ class Exclusive(ContextDecorator):
ensure that. ensure that.
""" """
_locks = {} _locks = {}
def __init__(self, wrapped): def __init__(self, wrapped):

View File

@ -9,7 +9,4 @@ This module contains all tools for Bonobo and Django to interract nicely.
from .utils import create_or_update from .utils import create_or_update
from .commands import ETLCommand from .commands import ETLCommand
__all__ = [ __all__ = ['ETLCommand', 'create_or_update']
'ETLCommand',
'create_or_update',
]

View File

@ -1,13 +1,14 @@
from logging import getLogger from logging import getLogger
from types import GeneratorType from types import GeneratorType
from colorama import Back, Fore, Style
from django.core.management import BaseCommand
from django.core.management.base import OutputWrapper
from mondrian import term
import bonobo import bonobo
from bonobo.plugins.console import ConsoleOutputPlugin from bonobo.plugins.console import ConsoleOutputPlugin
from bonobo.util.term import CLEAR_EOL from bonobo.util.term import CLEAR_EOL
from colorama import Fore, Back, Style
from django.core.management import BaseCommand
from django.core.management.base import OutputWrapper
from mondrian import term
from .utils import create_or_update from .utils import create_or_update
@ -55,7 +56,7 @@ class ETLCommand(BaseCommand):
graph_coll = self.get_graph(*args, **options) graph_coll = self.get_graph(*args, **options)
if not isinstance(graph_coll, GeneratorType): if not isinstance(graph_coll, GeneratorType):
graph_coll = (graph_coll, ) graph_coll = (graph_coll,)
for i, graph in enumerate(graph_coll): for i, graph in enumerate(graph_coll):
assert isinstance(graph, bonobo.Graph), 'Invalid graph provided.' assert isinstance(graph, bonobo.Graph), 'Invalid graph provided.'

View File

@ -41,14 +41,14 @@ def get_credentials(*, scopes):
return credentials return credentials
def get_google_spreadsheets_api_client(scopes=('https://www.googleapis.com/auth/spreadsheets', )): def get_google_spreadsheets_api_client(scopes=('https://www.googleapis.com/auth/spreadsheets',)):
credentials = get_credentials(scopes=scopes) credentials = get_credentials(scopes=scopes)
http = credentials.authorize(httplib2.Http()) http = credentials.authorize(httplib2.Http())
discoveryUrl = 'https://sheets.googleapis.com/$discovery/rest?version=v4' discoveryUrl = 'https://sheets.googleapis.com/$discovery/rest?version=v4'
return discovery.build('sheets', 'v4', http=http, discoveryServiceUrl=discoveryUrl, cache_discovery=False) return discovery.build('sheets', 'v4', http=http, discoveryServiceUrl=discoveryUrl, cache_discovery=False)
def get_google_people_api_client(scopes=('https://www.googleapis.com/auth/contacts', )): def get_google_people_api_client(scopes=('https://www.googleapis.com/auth/contacts',)):
credentials = get_credentials(scopes=scopes) credentials = get_credentials(scopes=scopes)
http = credentials.authorize(httplib2.Http()) http = credentials.authorize(httplib2.Http())
discoveryUrl = 'https://people.googleapis.com/$discovery/rest?version=v1' discoveryUrl = 'https://people.googleapis.com/$discovery/rest?version=v1'

View File

@ -5,6 +5,4 @@ def _jupyter_nbextension_paths():
return [{'section': 'notebook', 'src': 'static', 'dest': 'bonobo-jupyter', 'require': 'bonobo-jupyter/extension'}] return [{'section': 'notebook', 'src': 'static', 'dest': 'bonobo-jupyter', 'require': 'bonobo-jupyter/extension'}]
__all__ = [ __all__ = ['JupyterOutputPlugin']
'JupyterOutputPlugin',
]

View File

@ -44,15 +44,9 @@ class OpenDataSoftAPI(Configurable):
break break
for row in records: for row in records:
yield { yield {**row.get('fields', {}), 'geometry': row.get('geometry', {}), 'recordid': row.get('recordid')}
**row.get('fields', {}),
'geometry': row.get('geometry', {}),
'recordid': row.get('recordid'),
}
start += self.rows start += self.rows
__all__ = [ __all__ = ['OpenDataSoftAPI']
'OpenDataSoftAPI',
]

View File

@ -16,10 +16,7 @@ class InactiveWritableError(InactiveIOError):
class ValidationError(RuntimeError): class ValidationError(RuntimeError):
def __init__(self, inst, message): def __init__(self, inst, message):
super(ValidationError, self).__init__( super(ValidationError, self).__init__(
'Validation error in {class_name}: {message}'.format( 'Validation error in {class_name}: {message}'.format(class_name=type(inst).__name__, message=message)
class_name=type(inst).__name__,
message=message,
)
) )
@ -42,8 +39,7 @@ class AbstractError(UnrecoverableError, NotImplementedError):
def __init__(self, method): def __init__(self, method):
super().__init__( super().__init__(
'Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format( 'Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format(
class_name=get_name(method.__self__), class_name=get_name(method.__self__), method_name=get_name(method)
method_name=get_name(method),
) )
) )

View File

@ -4,19 +4,9 @@ import bonobo
def get_argument_parser(parser=None): def get_argument_parser(parser=None):
parser = bonobo.get_argument_parser(parser=parser) parser = bonobo.get_argument_parser(parser=parser)
parser.add_argument('--limit', '-l', type=int, default=None, help='If set, limits the number of processed lines.')
parser.add_argument( parser.add_argument(
'--limit', '--print', '-p', action='store_true', default=False, help='If set, pretty prints before writing to output file.'
'-l',
type=int,
default=None,
help='If set, limits the number of processed lines.'
)
parser.add_argument(
'--print',
'-p',
action='store_true',
default=False,
help='If set, pretty prints before writing to output file.'
) )
return parser return parser
@ -26,7 +16,4 @@ def get_graph_options(options):
_limit = options.pop('limit', None) _limit = options.pop('limit', None)
_print = options.pop('print', False) _print = options.pop('print', False)
return { return {'_limit': (bonobo.Limit(_limit),) if _limit else (), '_print': (bonobo.PrettyPrinter(),) if _print else ()}
'_limit': (bonobo.Limit(_limit), ) if _limit else (),
'_print': (bonobo.PrettyPrinter(), ) if _print else (),
}

View File

@ -1,7 +1,8 @@
import bonobo
import datetime import datetime
import time import time
import bonobo
def extract(): def extract():
"""Placeholder, change, rename, remove... """ """Placeholder, change, rename, remove... """
@ -13,10 +14,7 @@ def extract():
def get_graph(): def get_graph():
graph = bonobo.Graph() graph = bonobo.Graph()
graph.add_chain( graph.add_chain(extract, print)
extract,
print,
)
return graph return graph

View File

@ -4,26 +4,18 @@ import bonobo
from bonobo import examples from bonobo import examples
from bonobo.examples.datasets.coffeeshops import get_graph as get_coffeeshops_graph from bonobo.examples.datasets.coffeeshops import get_graph as get_coffeeshops_graph
from bonobo.examples.datasets.fablabs import get_graph as get_fablabs_graph from bonobo.examples.datasets.fablabs import get_graph as get_fablabs_graph
from bonobo.examples.datasets.services import get_services, get_datasets_dir, get_minor_version from bonobo.examples.datasets.services import get_datasets_dir, get_minor_version, get_services
graph_factories = { graph_factories = {'coffeeshops': get_coffeeshops_graph, 'fablabs': get_fablabs_graph}
'coffeeshops': get_coffeeshops_graph,
'fablabs': get_fablabs_graph,
}
if __name__ == '__main__': if __name__ == '__main__':
parser = examples.get_argument_parser() parser = examples.get_argument_parser()
parser.add_argument( parser.add_argument('--target', '-t', choices=graph_factories.keys(), nargs='+')
'--target', '-t', choices=graph_factories.keys(), nargs='+'
)
parser.add_argument('--sync', action='store_true', default=False) parser.add_argument('--sync', action='store_true', default=False)
with bonobo.parse_args(parser) as options: with bonobo.parse_args(parser) as options:
graph_options = examples.get_graph_options(options) graph_options = examples.get_graph_options(options)
graph_names = list( graph_names = list(options['target'] if options['target'] else sorted(graph_factories.keys()))
options['target']
if options['target'] else sorted(graph_factories.keys())
)
# Create a graph with all requested subgraphs # Create a graph with all requested subgraphs
graph = bonobo.Graph() graph = bonobo.Graph()
@ -43,18 +35,9 @@ if __name__ == '__main__':
for filename in files: for filename in files:
local_path = os.path.join(root, filename) local_path = os.path.join(root, filename)
relative_path = os.path.relpath(local_path, local_dir) relative_path = os.path.relpath(local_path, local_dir)
s3_path = os.path.join( s3_path = os.path.join(get_minor_version(), relative_path)
get_minor_version(), relative_path
)
try: try:
s3.head_object( s3.head_object(Bucket='bonobo-examples', Key=s3_path)
Bucket='bonobo-examples', Key=s3_path
)
except: except:
s3.upload_file( s3.upload_file(local_path, 'bonobo-examples', s3_path, ExtraArgs={'ACL': 'public-read'})
local_path,
'bonobo-examples',
s3_path,
ExtraArgs={'ACL': 'public-read'}
)

View File

@ -13,48 +13,26 @@ def get_graph(graph=None, *, _limit=(), _print=()):
graph = graph or bonobo.Graph() graph = graph or bonobo.Graph()
producer = graph.add_chain( producer = graph.add_chain(
ODSReader( ODSReader(dataset='liste-des-cafes-a-un-euro', netloc='opendata.paris.fr'),
dataset='liste-des-cafes-a-un-euro',
netloc='opendata.paris.fr'
),
*_limit, *_limit,
bonobo.UnpackItems(0), bonobo.UnpackItems(0),
bonobo.Rename( bonobo.Rename(name='nom_du_cafe', address='adresse', zipcode='arrondissement'),
name='nom_du_cafe',
address='adresse',
zipcode='arrondissement'
),
bonobo.Format(city='Paris', country='France'), bonobo.Format(city='Paris', country='France'),
bonobo.OrderFields( bonobo.OrderFields(['name', 'address', 'zipcode', 'city', 'country', 'geometry', 'geoloc']),
[
'name', 'address', 'zipcode', 'city', 'country',
'geometry', 'geoloc'
]
),
*_print, *_print,
) )
# Comma separated values. # Comma separated values.
graph.add_chain( graph.add_chain(
bonobo.CsvWriter( bonobo.CsvWriter('coffeeshops.csv', fields=['name', 'address', 'zipcode', 'city'], delimiter=','),
'coffeeshops.csv',
fields=['name', 'address', 'zipcode', 'city'],
delimiter=','
),
_input=producer.output, _input=producer.output,
) )
# Standard JSON # Standard JSON
graph.add_chain( graph.add_chain(bonobo.JsonWriter(path='coffeeshops.json'), _input=producer.output)
bonobo.JsonWriter(path='coffeeshops.json'),
_input=producer.output,
)
# Line-delimited JSON # Line-delimited JSON
graph.add_chain( graph.add_chain(bonobo.LdjsonWriter(path='coffeeshops.ldjson'), _input=producer.output)
bonobo.LdjsonWriter(path='coffeeshops.ldjson'),
_input=producer.output,
)
return graph return graph
@ -63,7 +41,4 @@ if __name__ == '__main__':
parser = examples.get_argument_parser() parser = examples.get_argument_parser()
with bonobo.parse_args(parser) as options: with bonobo.parse_args(parser) as options:
bonobo.run( bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())
get_graph(**examples.get_graph_options(options)),
services=get_services()
)

View File

@ -24,9 +24,7 @@ from bonobo.examples.datasets.services import get_services
try: try:
import pycountry import pycountry
except ImportError as exc: except ImportError as exc:
raise ImportError( raise ImportError('You must install package "pycountry" to run this example.') from exc
'You must install package "pycountry" to run this example.'
) from exc
API_DATASET = 'fablabs@public-us' API_DATASET = 'fablabs@public-us'
ROWS = 100 ROWS = 100
@ -39,12 +37,8 @@ def _getlink(x):
def normalize(row): def normalize(row):
result = { result = {
**row, **row,
'links': 'links': list(filter(None, map(_getlink, json.loads(row.get('links'))))),
list(filter(None, map(_getlink, json.loads(row.get('links'))))), 'country': pycountry.countries.get(alpha_2=row.get('country_code', '').upper()).name,
'country':
pycountry.countries.get(
alpha_2=row.get('country_code', '').upper()
).name,
} }
return result return result
@ -66,7 +60,4 @@ if __name__ == '__main__':
parser = examples.get_argument_parser() parser = examples.get_argument_parser()
with bonobo.parse_args(parser) as options: with bonobo.parse_args(parser) as options:
bonobo.run( bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())
get_graph(**examples.get_graph_options(options)),
services=get_services()
)

View File

@ -9,9 +9,7 @@ def get_minor_version():
def get_datasets_dir(*dirs): def get_datasets_dir(*dirs):
home_dir = os.path.expanduser('~') home_dir = os.path.expanduser('~')
target_dir = os.path.join( target_dir = os.path.join(home_dir, '.cache/bonobo', get_minor_version(), *dirs)
home_dir, '.cache/bonobo', get_minor_version(), *dirs
)
os.makedirs(target_dir, exist_ok=True) os.makedirs(target_dir, exist_ok=True)
return target_dir return target_dir

View File

@ -2,7 +2,4 @@ from bonobo import get_examples_path, open_fs
def get_services(): def get_services():
return { return {'fs': open_fs(get_examples_path()), 'fs.output': open_fs()}
'fs': open_fs(get_examples_path()),
'fs.output': open_fs(),
}

View File

@ -5,8 +5,8 @@ from bonobo.examples.files._services import get_services
def get_graph(*, _limit=None, _print=False): def get_graph(*, _limit=None, _print=False):
return bonobo.Graph( return bonobo.Graph(
bonobo.CsvReader('datasets/coffeeshops.txt'), bonobo.CsvReader('datasets/coffeeshops.txt'),
*((bonobo.Limit(_limit), ) if _limit else ()), *((bonobo.Limit(_limit),) if _limit else ()),
*((bonobo.PrettyPrinter(), ) if _print else ()), *((bonobo.PrettyPrinter(),) if _print else ()),
bonobo.CsvWriter('coffeeshops.csv', fs='fs.output') bonobo.CsvWriter('coffeeshops.csv', fs='fs.output')
) )
@ -14,23 +14,10 @@ def get_graph(*, _limit=None, _print=False):
if __name__ == '__main__': if __name__ == '__main__':
parser = bonobo.get_argument_parser() parser = bonobo.get_argument_parser()
parser.add_argument('--limit', '-l', type=int, default=None, help='If set, limits the number of processed lines.')
parser.add_argument( parser.add_argument(
'--limit', '--print', '-p', action='store_true', default=False, help='If set, pretty prints before writing to output file.'
'-l',
type=int,
default=None,
help='If set, limits the number of processed lines.'
)
parser.add_argument(
'--print',
'-p',
action='store_true',
default=False,
help='If set, pretty prints before writing to output file.'
) )
with bonobo.parse_args(parser) as options: with bonobo.parse_args(parser) as options:
bonobo.run( bonobo.run(get_graph(_limit=options['limit'], _print=options['print']), services=get_services())
get_graph(_limit=options['limit'], _print=options['print']),
services=get_services()
)

View File

@ -5,22 +5,13 @@ from bonobo.examples.files._services import get_services
def get_graph(*, _limit=None, _print=False): def get_graph(*, _limit=None, _print=False):
graph = bonobo.Graph() graph = bonobo.Graph()
trunk = graph.add_chain( trunk = graph.add_chain(bonobo.JsonReader('datasets/theaters.json'), *((bonobo.Limit(_limit),) if _limit else ()))
bonobo.JsonReader('datasets/theaters.json'),
*((bonobo.Limit(_limit), ) if _limit else ()),
)
if _print: if _print:
graph.add_chain(bonobo.PrettyPrinter(), _input=trunk.output) graph.add_chain(bonobo.PrettyPrinter(), _input=trunk.output)
graph.add_chain( graph.add_chain(bonobo.JsonWriter('theaters.json', fs='fs.output'), _input=trunk.output)
bonobo.JsonWriter('theaters.json', fs='fs.output'), graph.add_chain(bonobo.LdjsonWriter('theaters.ldjson', fs='fs.output'), _input=trunk.output)
_input=trunk.output
)
graph.add_chain(
bonobo.LdjsonWriter('theaters.ldjson', fs='fs.output'),
_input=trunk.output
)
return graph return graph
@ -28,23 +19,10 @@ def get_graph(*, _limit=None, _print=False):
if __name__ == '__main__': if __name__ == '__main__':
parser = bonobo.get_argument_parser() parser = bonobo.get_argument_parser()
parser.add_argument('--limit', '-l', type=int, default=None, help='If set, limits the number of processed lines.')
parser.add_argument( parser.add_argument(
'--limit', '--print', '-p', action='store_true', default=False, help='If set, pretty prints before writing to output file.'
'-l',
type=int,
default=None,
help='If set, limits the number of processed lines.'
)
parser.add_argument(
'--print',
'-p',
action='store_true',
default=False,
help='If set, pretty prints before writing to output file.'
) )
with bonobo.parse_args(parser) as options: with bonobo.parse_args(parser) as options:
bonobo.run( bonobo.run(get_graph(_limit=options['limit'], _print=options['print']), services=get_services())
get_graph(_limit=options['limit'], _print=options['print']),
services=get_services()
)

View File

@ -35,9 +35,7 @@ from bonobo import examples
def cleanse_sms(category, sms): def cleanse_sms(category, sms):
if category == 'spam': if category == 'spam':
sms_clean = '**MARKED AS SPAM** ' + sms[0:50] + ( sms_clean = '**MARKED AS SPAM** ' + sms[0:50] + ('...' if len(sms) > 50 else '')
'...' if len(sms) > 50 else ''
)
elif category == 'ham': elif category == 'ham':
sms_clean = sms sms_clean = sms
else: else:
@ -62,16 +60,11 @@ def get_graph(*, _limit=(), _print=()):
def get_services(): def get_services():
from ._services import get_services from ._services import get_services
return {
**get_services(), 'fs': return {**get_services(), 'fs': TarFS(bonobo.get_examples_path('datasets/spam.tgz'))}
TarFS(bonobo.get_examples_path('datasets/spam.tgz'))
}
if __name__ == '__main__': if __name__ == '__main__':
parser = examples.get_argument_parser() parser = examples.get_argument_parser()
with bonobo.parse_args(parser) as options: with bonobo.parse_args(parser) as options:
bonobo.run( bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())
get_graph(**examples.get_graph_options(options)),
services=get_services()
)

View File

@ -23,7 +23,4 @@ def get_graph(*, _limit=(), _print=()):
if __name__ == '__main__': if __name__ == '__main__':
parser = examples.get_argument_parser() parser = examples.get_argument_parser()
with bonobo.parse_args(parser) as options: with bonobo.parse_args(parser) as options:
bonobo.run( bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())
get_graph(**examples.get_graph_options(options)),
services=get_services()
)

View File

@ -9,8 +9,4 @@ from bonobo.execution.contexts.graph import GraphExecutionContext
from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.execution.contexts.node import NodeExecutionContext
from bonobo.execution.contexts.plugin import PluginExecutionContext from bonobo.execution.contexts.plugin import PluginExecutionContext
__all__ = [ __all__ = ['GraphExecutionContext', 'NodeExecutionContext', 'PluginExecutionContext']
'GraphExecutionContext',
'NodeExecutionContext',
'PluginExecutionContext',
]

View File

@ -1,12 +1,13 @@
from functools import partial from functools import partial
from time import sleep from time import sleep
from whistle import EventDispatcher
from bonobo.config import create_container from bonobo.config import create_container
from bonobo.constants import BEGIN, END from bonobo.constants import BEGIN, END
from bonobo.execution import events from bonobo.execution import events
from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.execution.contexts.node import NodeExecutionContext
from bonobo.execution.contexts.plugin import PluginExecutionContext from bonobo.execution.contexts.plugin import PluginExecutionContext
from whistle import EventDispatcher
class GraphExecutionContext: class GraphExecutionContext:

View File

@ -7,11 +7,11 @@ from types import GeneratorType
from bonobo.config import create_container from bonobo.config import create_container
from bonobo.config.processors import ContextCurrifier from bonobo.config.processors import ContextCurrifier
from bonobo.constants import NOT_MODIFIED, BEGIN, END, TICK_PERIOD, Token, Flag, INHERIT from bonobo.constants import BEGIN, END, INHERIT, NOT_MODIFIED, TICK_PERIOD, Flag, Token
from bonobo.errors import InactiveReadableError, UnrecoverableError, UnrecoverableTypeError from bonobo.errors import InactiveReadableError, UnrecoverableError, UnrecoverableTypeError
from bonobo.execution.contexts.base import BaseContext from bonobo.execution.contexts.base import BaseContext
from bonobo.structs.inputs import Input from bonobo.structs.inputs import Input
from bonobo.util import get_name, isconfigurabletype, ensure_tuple from bonobo.util import ensure_tuple, get_name, isconfigurabletype
from bonobo.util.bags import BagType from bonobo.util.bags import BagType
from bonobo.util.statistics import WithStatistics from bonobo.util.statistics import WithStatistics
@ -105,10 +105,7 @@ class NodeExecutionContext(BaseContext, WithStatistics):
except Empty: except Empty:
sleep(TICK_PERIOD) # XXX: How do we determine this constant? sleep(TICK_PERIOD) # XXX: How do we determine this constant?
continue continue
except ( except (NotImplementedError, UnrecoverableError):
NotImplementedError,
UnrecoverableError,
):
self.fatal(sys.exc_info()) # exit loop self.fatal(sys.exc_info()) # exit loop
except Exception: # pylint: disable=broad-except except Exception: # pylint: disable=broad-except
self.error(sys.exc_info()) # does not exit loop self.error(sys.exc_info()) # does not exit loop

View File

@ -9,9 +9,7 @@ at home if you want to give it a shot.
from bonobo.execution.strategies.executor import ProcessPoolExecutorStrategy, ThreadPoolExecutorStrategy from bonobo.execution.strategies.executor import ProcessPoolExecutorStrategy, ThreadPoolExecutorStrategy
from bonobo.execution.strategies.naive import NaiveStrategy from bonobo.execution.strategies.naive import NaiveStrategy
__all__ = [ __all__ = ['create_strategy']
'create_strategy',
]
STRATEGIES = { STRATEGIES = {
'naive': NaiveStrategy, 'naive': NaiveStrategy,

View File

@ -6,6 +6,7 @@ class Strategy:
Base class for execution strategies. Base class for execution strategies.
""" """
GraphExecutionContextType = GraphExecutionContext GraphExecutionContextType = GraphExecutionContext
def __init__(self, GraphExecutionContextType=None): def __init__(self, GraphExecutionContextType=None):

View File

@ -3,14 +3,15 @@ import html
import itertools import itertools
import pprint import pprint
from mondrian import term
from bonobo import settings from bonobo import settings
from bonobo.config import Configurable, Option, Method, use_raw_input, use_context, use_no_input from bonobo.config import Configurable, Method, Option, use_context, use_no_input, use_raw_input
from bonobo.config.functools import transformation_factory from bonobo.config.functools import transformation_factory
from bonobo.config.processors import ContextProcessor, use_context_processor from bonobo.config.processors import ContextProcessor, use_context_processor
from bonobo.constants import NOT_MODIFIED from bonobo.constants import NOT_MODIFIED
from bonobo.util.objects import ValueHolder from bonobo.util.objects import ValueHolder
from bonobo.util.term import CLEAR_EOL from bonobo.util.term import CLEAR_EOL
from mondrian import term
__all__ = [ __all__ = [
'FixedWindow', 'FixedWindow',
@ -43,6 +44,7 @@ class Limit(Configurable):
TODO: simplify into a closure building factory? TODO: simplify into a closure building factory?
""" """
limit = Option(positional=True, default=10) limit = Option(positional=True, default=10)
@ContextProcessor @ContextProcessor
@ -69,7 +71,7 @@ def Tee(f):
def _shorten(s, w): def _shorten(s, w):
if w and len(s) > w: if w and len(s) > w:
s = s[0:w - 3] + '...' s = s[0 : w - 3] + '...'
return s return s
@ -80,17 +82,19 @@ class PrettyPrinter(Configurable):
required=False, required=False,
__doc__=''' __doc__='''
If set, truncates the output values longer than this to this width. If set, truncates the output values longer than this to this width.
''' ''',
) )
filter = Method( filter = Method(
default= default=(
(lambda self, index, key, value: (value is not None) and (not isinstance(key, str) or not key.startswith('_'))), lambda self, index, key, value: (value is not None)
and (not isinstance(key, str) or not key.startswith('_'))
),
__doc__=''' __doc__='''
A filter that determine what to print. A filter that determine what to print.
Default is to ignore any key starting with an underscore and none values. Default is to ignore any key starting with an underscore and none values.
''' ''',
) )
@ContextProcessor @ContextProcessor
@ -99,6 +103,7 @@ class PrettyPrinter(Configurable):
yield context yield context
if context._jupyter_html is not None: if context._jupyter_html is not None:
from IPython.display import display, HTML from IPython.display import display, HTML
display(HTML('\n'.join(['<table>'] + context._jupyter_html + ['</table>']))) display(HTML('\n'.join(['<table>'] + context._jupyter_html + ['</table>'])))
def __call__(self, context, *args, **kwargs): def __call__(self, context, *args, **kwargs):
@ -153,16 +158,11 @@ class PrettyPrinter(Configurable):
if not context._jupyter_html: if not context._jupyter_html:
context._jupyter_html = [ context._jupyter_html = [
'<thead><tr>', '<thead><tr>',
*map('<th>{}</th>'.format, map(html.escape, map(str, *map('<th>{}</th>'.format, map(html.escape, map(str, context.get_input_fields() or range(len(args))))),
context.get_input_fields() or range(len(args))))),
'</tr></thead>', '</tr></thead>',
] ]
context._jupyter_html += [ context._jupyter_html += ['<tr>', *map('<td>{}</td>'.format, map(html.escape, map(repr, args))), '</tr>']
'<tr>',
*map('<td>{}</td>'.format, map(html.escape, map(repr, args))),
'</tr>',
]
@use_no_input @use_no_input

View File

@ -1,6 +1,5 @@
from bonobo.constants import NOT_MODIFIED
from bonobo.config import Configurable, Method from bonobo.config import Configurable, Method
from bonobo.constants import NOT_MODIFIED
class Filter(Configurable): class Filter(Configurable):

View File

@ -13,22 +13,39 @@ class FileHandler(Configurable):
""" """
path = Option( path = Option(
str, required=True, positional=True, __doc__=''' str,
required=True,
positional=True,
__doc__='''
Path to use within the provided filesystem. Path to use within the provided filesystem.
''' ''',
) # type: str ) # type: str
eol = Option(str, default='\n', __doc__=''' eol = Option(
str,
default='\n',
__doc__='''
Character to use as line separator. Character to use as line separator.
''') # type: str ''',
mode = Option(str, __doc__=''' ) # type: str
mode = Option(
str,
__doc__='''
What mode to use for open() call. What mode to use for open() call.
''') # type: str ''',
encoding = Option(str, default='utf-8', __doc__=''' ) # type: str
encoding = Option(
str,
default='utf-8',
__doc__='''
Encoding. Encoding.
''') # type: str ''',
fs = Service('fs', __doc__=''' ) # type: str
fs = Service(
'fs',
__doc__='''
The filesystem instance to use. The filesystem instance to use.
''') # type: str ''',
) # type: str
@ContextProcessor @ContextProcessor
def file(self, context, *, fs): def file(self, context, *, fs):

View File

@ -1,6 +1,6 @@
import csv import csv
from bonobo.config import Option, use_raw_input, use_context from bonobo.config import Option, use_context, use_raw_input
from bonobo.config.options import Method, RenamedOption from bonobo.config.options import Method, RenamedOption
from bonobo.constants import NOT_MODIFIED from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.base import FileHandler from bonobo.nodes.io.base import FileHandler
@ -62,7 +62,7 @@ class CsvReader(FileReader, CsvHandler):
default=0, default=0,
__doc__=''' __doc__='''
If set and greater than zero, the reader will skip this amount of lines. If set and greater than zero, the reader will skip this amount of lines.
''' ''',
) )
@Method( @Method(
@ -72,7 +72,7 @@ class CsvReader(FileReader, CsvHandler):
iterable. iterable.
Defaults to builtin csv.reader(...), but can be overriden to fit your special needs. Defaults to builtin csv.reader(...), but can be overriden to fit your special needs.
''' ''',
) )
def reader_factory(self, file): def reader_factory(self, file):
return csv.reader(file, **self.get_dialect_kwargs()) return csv.reader(file, **self.get_dialect_kwargs())

View File

@ -1,4 +1,4 @@
from bonobo.config import Option, ContextProcessor, use_context from bonobo.config import ContextProcessor, Option, use_context
from bonobo.constants import NOT_MODIFIED from bonobo.constants import NOT_MODIFIED
from bonobo.errors import UnrecoverableError from bonobo.errors import UnrecoverableError
from bonobo.nodes.io.base import FileHandler, Reader, Writer from bonobo.nodes.io.base import FileHandler, Reader, Writer
@ -12,9 +12,13 @@ class FileReader(Reader, FileHandler):
present. Extending it is usually the right way to create more specific file readers (like json, csv, etc.) present. Extending it is usually the right way to create more specific file readers (like json, csv, etc.)
""" """
mode = Option(str, default='r', __doc__=''' mode = Option(
str,
default='r',
__doc__='''
What mode to use for open() call. What mode to use for open() call.
''') # type: str ''',
) # type: str
output_fields = Option( output_fields = Option(
ensure_tuple, ensure_tuple,
@ -22,14 +26,14 @@ class FileReader(Reader, FileHandler):
__doc__=''' __doc__='''
Specify the field names of output lines. Specify the field names of output lines.
Mutually exclusive with "output_type". Mutually exclusive with "output_type".
''' ''',
) )
output_type = Option( output_type = Option(
required=False, required=False,
__doc__=''' __doc__='''
Specify the type of output lines. Specify the type of output lines.
Mutually exclusive with "output_fields". Mutually exclusive with "output_fields".
''' ''',
) )
@ContextProcessor @ContextProcessor
@ -72,9 +76,13 @@ class FileWriter(Writer, FileHandler):
usually the right way to create more specific file writers (like json, csv, etc.) usually the right way to create more specific file writers (like json, csv, etc.)
""" """
mode = Option(str, default='w+', __doc__=''' mode = Option(
str,
default='w+',
__doc__='''
What mode to use for open() call. What mode to use for open() call.
''') # type: str ''',
) # type: str
def write(self, file, context, line, *, fs): def write(self, file, context, line, *, fs):
""" """

View File

@ -1,8 +1,9 @@
import io import io
import sys import sys
from contextlib import redirect_stdout, redirect_stderr from contextlib import redirect_stderr, redirect_stdout
from colorama import Style, Fore, init as initialize_colorama_output_wrappers from colorama import Fore, Style
from colorama import init as initialize_colorama_output_wrappers
from bonobo import settings from bonobo import settings
from bonobo.execution import events from bonobo.execution import events
@ -34,7 +35,7 @@ class ConsoleOutputPlugin(Plugin):
isatty = False isatty = False
# Whether we're on windows, or a real operating system. # Whether we're on windows, or a real operating system.
iswindows = (sys.platform == 'win32') iswindows = sys.platform == 'win32'
def __init__(self): def __init__(self):
self.isatty = self._stdout.isatty() self.isatty = self._stdout.isatty()
@ -95,27 +96,32 @@ class ConsoleOutputPlugin(Plugin):
liveliness_color = alive_color if node.alive else dead_color liveliness_color = alive_color if node.alive else dead_color
liveliness_prefix = ' {}{}{} '.format(liveliness_color, node.status, Style.RESET_ALL) liveliness_prefix = ' {}{}{} '.format(liveliness_color, node.status, Style.RESET_ALL)
_line = ''.join(( _line = ''.join(
liveliness_prefix, (
node.name, liveliness_prefix,
name_suffix, node.name,
' ', name_suffix,
node.get_statistics_as_string(), ' ',
' ', node.get_statistics_as_string(),
node.get_flags_as_string(), ' ',
Style.RESET_ALL, node.get_flags_as_string(),
' ', Style.RESET_ALL,
)) ' ',
)
)
print(prefix + _line + CLEAR_EOL, file=self._stderr) print(prefix + _line + CLEAR_EOL, file=self._stderr)
if append: if append:
# todo handle multiline # todo handle multiline
print( print(
''.join(( ''.join(
' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append), (
CLEAR_EOL ' `-> ',
)), ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append),
file=self._stderr CLEAR_EOL,
)
),
file=self._stderr,
) )
t_cnt += 1 t_cnt += 1
@ -128,16 +134,17 @@ class ConsoleOutputPlugin(Plugin):
if self.counter % 10 and self._append_cache: if self.counter % 10 and self._append_cache:
append = self._append_cache append = self._append_cache
else: else:
self._append_cache = append = (('Memory', '{0:.2f} Mb'.format(memory_usage())), self._append_cache = append = (
# ('Total time', '{0} s'.format(execution_time(harness))), ('Memory', '{0:.2f} Mb'.format(memory_usage())),
) # ('Total time', '{0} s'.format(execution_time(harness))),
)
else: else:
append = () append = ()
self.write(context, prefix=self.prefix, append=append, rewind=rewind) self.write(context, prefix=self.prefix, append=append, rewind=rewind)
self.counter += 1 self.counter += 1
class IOBuffer(): class IOBuffer:
""" """
The role of IOBuffer is to overcome the problem of multiple threads wanting to write to stdout at the same time. It The role of IOBuffer is to overcome the problem of multiple threads wanting to write to stdout at the same time. It
works a bit like a videogame: there are two buffers, one that is used to write, and one which is used to read from. works a bit like a videogame: there are two buffers, one that is used to write, and one which is used to read from.
@ -164,5 +171,6 @@ class IOBuffer():
def memory_usage(): def memory_usage():
import os, psutil import os, psutil
process = psutil.Process(os.getpid()) process = psutil.Process(os.getpid())
return process.memory_info()[0] / float(2**20) return process.memory_info()[0] / float(2 ** 20)

View File

@ -1,8 +1,7 @@
import mimetypes import mimetypes
import os import os
from bonobo import JsonReader, CsvReader, PickleReader, FileReader, FileWriter, PickleWriter, CsvWriter, JsonWriter from bonobo import CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter, PickleReader, PickleWriter
FILETYPE_CSV = 'text/csv' FILETYPE_CSV = 'text/csv'
FILETYPE_JSON = 'application/json' FILETYPE_JSON = 'application/json'

View File

@ -1,5 +1,4 @@
import logging import logging
import os import os
from bonobo.errors import ValidationError from bonobo.errors import ValidationError
@ -92,17 +91,14 @@ LOGGING_LEVEL = Setting(
'LOGGING_LEVEL', 'LOGGING_LEVEL',
formatter=logging._checkLevel, formatter=logging._checkLevel,
validator=logging._checkLevel, validator=logging._checkLevel,
default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO,
) )
# Input/Output format for transformations # Input/Output format for transformations
IOFORMAT_ARG0 = 'arg0' IOFORMAT_ARG0 = 'arg0'
IOFORMAT_KWARGS = 'kwargs' IOFORMAT_KWARGS = 'kwargs'
IOFORMATS = { IOFORMATS = {IOFORMAT_ARG0, IOFORMAT_KWARGS}
IOFORMAT_ARG0,
IOFORMAT_KWARGS,
}
IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__) IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__)

View File

@ -1,5 +1,3 @@
from bonobo.structs.graphs import Graph from bonobo.structs.graphs import Graph
__all__ = [ __all__ = ['Graph']
'Graph',
]

View File

@ -3,11 +3,12 @@ import json
from collections import namedtuple from collections import namedtuple
from copy import copy from copy import copy
from bonobo.constants import BEGIN
from bonobo.util import get_name
from graphviz import ExecutableNotFound from graphviz import ExecutableNotFound
from graphviz.dot import Digraph from graphviz.dot import Digraph
from bonobo.constants import BEGIN
from bonobo.util import get_name
GraphRange = namedtuple('GraphRange', ['graph', 'input', 'output']) GraphRange = namedtuple('GraphRange', ['graph', 'input', 'output'])
@ -15,6 +16,7 @@ class Graph:
""" """
Represents a directed graph of nodes. Represents a directed graph of nodes.
""" """
name = '' name = ''
def __init__(self, *chain): def __init__(self, *chain):

View File

@ -16,7 +16,7 @@ from bonobo.util.inspect import (
istuple, istuple,
istype, istype,
) )
from bonobo.util.objects import (get_name, get_attribute_or_create, ValueHolder) from bonobo.util.objects import get_name, get_attribute_or_create, ValueHolder
# Bonobo's util API # Bonobo's util API
__all__ = [ __all__ = [

View File

@ -12,14 +12,14 @@ class ApiHelper:
if graph: if graph:
# This function must comply to the "graph" API interface, meaning it can bahave like bonobo.run. # This function must comply to the "graph" API interface, meaning it can bahave like bonobo.run.
from inspect import signature from inspect import signature
parameters = list(signature(x).parameters) parameters = list(signature(x).parameters)
required_parameters = {'plugins', 'services', 'strategy'} required_parameters = {'plugins', 'services', 'strategy'}
assert len(parameters assert (
) > 0 and parameters[0] == 'graph', 'First parameter of a graph api function must be "graph".' len(parameters) > 0 and parameters[0] == 'graph'
assert required_parameters.intersection( ), 'First parameter of a graph api function must be "graph".'
parameters assert required_parameters.intersection(parameters) == required_parameters, (
) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join( 'Graph api functions must define the following parameters: ' + ', '.join(sorted(required_parameters))
sorted(required_parameters)
) )
self.__all__.append(get_name(x)) self.__all__.append(get_name(x))

View File

@ -73,7 +73,9 @@ class {typename}(tuple):
_field_template = '''\ _field_template = '''\
{name} = _property(_itemgetter({index:d}), doc={doc!r}) {name} = _property(_itemgetter({index:d}), doc={doc!r})
'''.strip('\n') '''.strip(
'\n'
)
_reserved = frozenset( _reserved = frozenset(
['_', '_cls', '_attrs', '_fields', 'get', '_asdict', '_replace', '_make', 'self', '_self', 'tuple'] + dir(tuple) ['_', '_cls', '_attrs', '_fields', 'get', '_asdict', '_replace', '_make', 'self', '_self', 'tuple'] + dir(tuple)
@ -150,16 +152,19 @@ def BagType(typename, fields, *, verbose=False, module=None):
attrs=attrs, attrs=attrs,
num_fields=len(fields), num_fields=len(fields),
arg_list=repr(attrs).replace("'", "")[1:-1], arg_list=repr(attrs).replace("'", "")[1:-1],
repr_fmt=', '.join(('%r' if isinstance(fields[index], int) else '{name}=%r').format(name=name) repr_fmt=', '.join(
for index, name in enumerate(attrs)), ('%r' if isinstance(fields[index], int) else '{name}=%r').format(name=name)
for index, name in enumerate(attrs)
),
field_defs='\n'.join( field_defs='\n'.join(
_field_template.format( _field_template.format(
index=index, index=index,
name=name, name=name,
doc='Alias for ' + doc='Alias for '
('field #{}'.format(index) if isinstance(fields[index], int) else repr(fields[index])) + ('field #{}'.format(index) if isinstance(fields[index], int) else repr(fields[index])),
) for index, name in enumerate(attrs) )
) for index, name in enumerate(attrs)
),
) )
# Execute the template string in a temporary namespace and support # Execute the template string in a temporary namespace and support

View File

@ -26,7 +26,7 @@ def ensure_tuple(tuple_or_mixed, *, cls=tuple):
if isinstance(tuple_or_mixed, tuple): if isinstance(tuple_or_mixed, tuple):
return tuple.__new__(cls, tuple_or_mixed) return tuple.__new__(cls, tuple_or_mixed)
return tuple.__new__(cls, (tuple_or_mixed, )) return tuple.__new__(cls, (tuple_or_mixed,))
def cast(type_): def cast(type_):

View File

@ -9,7 +9,7 @@ def deprecated_alias(alias, func):
warnings.warn( warnings.warn(
"Call to deprecated function alias {}, use {} instead.".format(alias, func.__name__), "Call to deprecated function alias {}, use {} instead.".format(alias, func.__name__),
category=DeprecationWarning, category=DeprecationWarning,
stacklevel=2 stacklevel=2,
) )
warnings.simplefilter('default', DeprecationWarning) # reset filter warnings.simplefilter('default', DeprecationWarning) # reset filter
return func(*args, **kwargs) return func(*args, **kwargs)

View File

@ -58,6 +58,7 @@ def get_argument_parser(parser=None):
""" """
if parser is None: if parser is None:
import argparse import argparse
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
# Store globally to be able to warn the user about the fact he's probably wrong not to pass a parser to # Store globally to be able to warn the user about the fact he's probably wrong not to pass a parser to
@ -94,6 +95,7 @@ def parse_args(mixed=None):
) )
# use the api from bonobo namespace, in case a command patched it. # use the api from bonobo namespace, in case a command patched it.
import bonobo import bonobo
mixed = bonobo.get_argument_parser() mixed = bonobo.get_argument_parser()
if isinstance(mixed, argparse.ArgumentParser): if isinstance(mixed, argparse.ArgumentParser):

View File

@ -9,6 +9,7 @@ def isconfigurable(mixed):
:return: bool :return: bool
""" """
from bonobo.config.configurables import Configurable from bonobo.config.configurables import Configurable
return isinstance(mixed, Configurable) return isinstance(mixed, Configurable)
@ -47,6 +48,7 @@ def isoption(mixed):
""" """
from bonobo.config.options import Option from bonobo.config.options import Option
return isinstance(mixed, Option) return isinstance(mixed, Option)
@ -58,6 +60,7 @@ def ismethod(mixed):
:return: bool :return: bool
""" """
from bonobo.config.options import Method from bonobo.config.options import Method
return isinstance(mixed, Method) return isinstance(mixed, Method)
@ -69,6 +72,7 @@ def iscontextprocessor(x):
:return: bool :return: bool
""" """
from bonobo.config.processors import ContextProcessor from bonobo.config.processors import ContextProcessor
return isinstance(x, ContextProcessor) return isinstance(x, ContextProcessor)
@ -102,15 +106,7 @@ def istuple(mixed):
return isinstance(mixed, tuple) return isinstance(mixed, tuple)
ConfigurableInspection = namedtuple( ConfigurableInspection = namedtuple('ConfigurableInspection', ['type', 'instance', 'options', 'processors', 'partial'])
'ConfigurableInspection', [
'type',
'instance',
'options',
'processors',
'partial',
]
)
ConfigurableInspection.__enter__ = lambda self: self ConfigurableInspection.__enter__ = lambda self: self
ConfigurableInspection.__exit__ = lambda *exc_details: None ConfigurableInspection.__exit__ = lambda *exc_details: None
@ -141,10 +137,4 @@ def inspect_node(mixed, *, _partial=None):
'Not a Configurable, nor a Configurable instance and not even a partially configured Configurable. Check your inputs.' 'Not a Configurable, nor a Configurable instance and not even a partially configured Configurable. Check your inputs.'
) )
return ConfigurableInspection( return ConfigurableInspection(typ, inst, list(typ.__options__), list(typ.__processors__), _partial)
typ,
inst,
list(typ.__options__),
list(typ.__processors__),
_partial,
)

View File

@ -142,10 +142,10 @@ class ValueHolder:
return divmod(other, self._value) return divmod(other, self._value)
def __pow__(self, other): def __pow__(self, other):
return self._value**other return self._value ** other
def __rpow__(self, other): def __rpow__(self, other):
return other**self._value return other ** self._value
def __ipow__(self, other): def __ipow__(self, other):
self._value **= other self._value **= other

View File

@ -1,2 +1,2 @@
CLEAR_EOL = '\033[0K' CLEAR_EOL = '\033[0K'
MOVE_CURSOR_UP = lambda n: '\033[{}A'.format(n) MOVE_CURSOR_UP = lambda n: '\033[{}A'.format(n)

View File

@ -4,12 +4,12 @@ import io
import os import os
import runpy import runpy
import sys import sys
from contextlib import contextmanager, redirect_stdout, redirect_stderr from contextlib import contextmanager, redirect_stderr, redirect_stdout
from unittest.mock import patch from unittest.mock import patch
import pytest import pytest
from bonobo import open_fs, __main__, get_examples_path from bonobo import __main__, get_examples_path, open_fs
from bonobo.commands import entrypoint from bonobo.commands import entrypoint
from bonobo.constants import Token from bonobo.constants import Token
from bonobo.execution.contexts.graph import GraphExecutionContext from bonobo.execution.contexts.graph import GraphExecutionContext
@ -112,19 +112,13 @@ def runner_module(args):
all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module]) all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module])
all_environ_targets = pytest.mark.parametrize( all_environ_targets = pytest.mark.parametrize(
'target', [ 'target', [(get_examples_path('environ.py'),), ('-m', 'bonobo.examples.environ')]
(get_examples_path('environ.py'), ),
(
'-m',
'bonobo.examples.environ',
),
]
) )
@all_runners @all_runners
@all_environ_targets @all_environ_targets
class EnvironmentTestCase(): class EnvironmentTestCase:
def run_quiet(self, runner, *args): def run_quiet(self, runner, *args):
return runner('run', '--quiet', *args) return runner('run', '--quiet', *args)
@ -216,12 +210,12 @@ class ReaderTest(ConfigurableNodeTest):
self.tmpdir = tmpdir self.tmpdir = tmpdir
def get_create_args(self, *args): def get_create_args(self, *args):
return (self.filename, ) + args return (self.filename,) + args
def test_customizable_output_type_transform_not_a_type(self): def test_customizable_output_type_transform_not_a_type(self):
context = self.NodeExecutionContextType( context = self.NodeExecutionContextType(
self.create(*self.get_create_args(), output_type=str.upper, **self.get_create_kwargs()), self.create(*self.get_create_args(), output_type=str.upper, **self.get_create_kwargs()),
services=self.services services=self.services,
) )
with pytest.raises(TypeError): with pytest.raises(TypeError):
context.start() context.start()
@ -229,9 +223,9 @@ class ReaderTest(ConfigurableNodeTest):
def test_customizable_output_type_transform_not_a_tuple(self): def test_customizable_output_type_transform_not_a_tuple(self):
context = self.NodeExecutionContextType( context = self.NodeExecutionContextType(
self.create( self.create(
*self.get_create_args(), output_type=type('UpperString', (str, ), {}), **self.get_create_kwargs() *self.get_create_args(), output_type=type('UpperString', (str,), {}), **self.get_create_kwargs()
), ),
services=self.services services=self.services,
) )
with pytest.raises(TypeError): with pytest.raises(TypeError):
context.start() context.start()
@ -256,7 +250,7 @@ class WriterTest(ConfigurableNodeTest):
self.tmpdir = tmpdir self.tmpdir = tmpdir
def get_create_args(self, *args): def get_create_args(self, *args):
return (self.filename, ) + args return (self.filename,) + args
def readlines(self): def readlines(self):
with self.fs.open(self.filename) as fp: with self.fs.open(self.filename) as fp:

View File

@ -1,8 +1,9 @@
# flake8: noqa # flake8: noqa
from pygments.style import Style from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, \ from pygments.token import (
Number, Operator, Generic, Whitespace, Punctuation, Other, Literal Comment, Error, Generic, Keyword, Literal, Name, Number, Operator, Other, Punctuation, String, Whitespace
)
# Originally based on FlaskyStyle which was based on 'tango'. # Originally based on FlaskyStyle which was based on 'tango'.
@ -12,7 +13,7 @@ class Alabaster(Style):
styles = { styles = {
# No corresponding class for the following: # No corresponding class for the following:
#Text: "", # class: '' # Text: "", # class: ''
Whitespace: "underline #f8f8f8", # class: 'w' Whitespace: "underline #f8f8f8", # class: 'w'
Error: "#a40000 border:#ef2929", # class: 'err' Error: "#a40000 border:#ef2929", # class: 'err'
Other: "#000000", # class 'x' Other: "#000000", # class 'x'
@ -28,7 +29,6 @@ class Alabaster(Style):
Operator: "#582800", # class: 'o' Operator: "#582800", # class: 'o'
Operator.Word: "bold #004461", # class: 'ow' - like keywords Operator.Word: "bold #004461", # class: 'ow' - like keywords
Punctuation: "bold #000000", # class: 'p' Punctuation: "bold #000000", # class: 'p'
# because special names such as Name.Class, Name.Function, etc. # because special names such as Name.Class, Name.Function, etc.
# are not recognized as such later in the parsing, we choose them # are not recognized as such later in the parsing, we choose them
# to look the same as ordinary variables. # to look the same as ordinary variables.

View File

@ -5,10 +5,11 @@ import datetime
import os import os
import sys import sys
import bonobo
sys.path.insert(0, os.path.abspath('..')) sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('_themes')) sys.path.insert(0, os.path.abspath('_themes'))
import bonobo
# -- General configuration ------------------------------------------------ # -- General configuration ------------------------------------------------
@ -63,11 +64,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
autoclass_content = 'both' autoclass_content = 'both'
autodoc_member_order = 'groupwise' autodoc_member_order = 'groupwise'
autodoc_default_flags = [ autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance']
'members',
'undoc-members',
'show-inheritance',
]
add_module_names = False add_module_names = False
pygments_style = 'sphinx' pygments_style = 'sphinx'
@ -112,7 +109,7 @@ html_sidebars = {
'sourcelink.html', 'sourcelink.html',
'searchbox.html', 'searchbox.html',
'sidebarinfos.html', 'sidebarinfos.html',
] ],
} }
html_theme_path = ['_themes'] html_theme_path = ['_themes']
@ -137,15 +134,12 @@ latex_elements = {
# The paper size ('letterpaper' or 'a4paper'). # The paper size ('letterpaper' or 'a4paper').
# #
# 'papersize': 'letterpaper', # 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt'). # The font size ('10pt', '11pt' or '12pt').
# #
# 'pointsize': '10pt', # 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble. # Additional stuff for the LaTeX preamble.
# #
# 'preamble': '', # 'preamble': '',
# Latex figure (float) alignment # Latex figure (float) alignment
# #
# 'figure_align': 'htbp', # 'figure_align': 'htbp',
@ -154,9 +148,7 @@ latex_elements = {
# Grouping the document tree into LaTeX files. List of tuples # Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, # (source start file, target name, title,
# author, documentclass [howto, manual, or own class]). # author, documentclass [howto, manual, or own class]).
latex_documents = [ latex_documents = [(master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual')]
(master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual'),
]
# -- Options for manual page output --------------------------------------- # -- Options for manual page output ---------------------------------------
@ -171,9 +163,14 @@ man_pages = [(master_doc, 'bonobo', 'Bonobo Documentation', [author], 1)]
# dir menu entry, description, category) # dir menu entry, description, category)
texinfo_documents = [ texinfo_documents = [
( (
master_doc, 'Bonobo', 'Bonobo Documentation', author, 'Bonobo', 'One line description of project.', master_doc,
'Miscellaneous' 'Bonobo',
), 'Bonobo Documentation',
author,
'Bonobo',
'One line description of project.',
'Miscellaneous',
)
] ]
# -- Options for Epub output ---------------------------------------------- # -- Options for Epub output ----------------------------------------------
@ -209,4 +206,6 @@ rst_epilog = """
.. |longversion| replace:: v.{version} .. |longversion| replace:: v.{version}
""".format(version=version, ) """.format(
version=version
)

View File

@ -1,9 +1,10 @@
-e .[dev] -e .[dev]
-r requirements.txt -r requirements.txt
alabaster==0.7.10 alabaster==0.7.11
arrow==0.12.1 arrow==0.12.1
atomicwrites==1.1.5
attrs==18.1.0 attrs==18.1.0
babel==2.5.3 babel==2.6.0
binaryornot==0.4.4 binaryornot==0.4.4
certifi==2018.4.16 certifi==2018.4.16
chardet==3.0.4 chardet==3.0.4
@ -12,29 +13,29 @@ cookiecutter==1.5.1
coverage==4.5.1 coverage==4.5.1
docutils==0.14 docutils==0.14
future==0.16.0 future==0.16.0
idna==2.6 idna==2.7
imagesize==1.0.0 imagesize==1.0.0
jinja2-time==0.2.0 jinja2-time==0.2.0
jinja2==2.10 jinja2==2.10
markupsafe==1.0 markupsafe==1.0
more-itertools==4.1.0 more-itertools==4.3.0
packaging==17.1 packaging==17.1
pluggy==0.6.0 pathlib2==2.3.2
pluggy==0.7.1
poyo==0.4.1 poyo==0.4.1
py==1.5.3 py==1.5.4
pygments==2.2.0 pygments==2.2.0
pyparsing==2.2.0 pyparsing==2.2.0
pytest-cov==2.5.1 pytest-cov==2.5.1
pytest-timeout==1.2.1 pytest-timeout==1.3.1
pytest==3.5.1 pytest==3.7.1
python-dateutil==2.7.3 python-dateutil==2.7.3
pytz==2018.4 pytz==2018.5
requests==2.18.4 requests==2.19.1
six==1.11.0 six==1.11.0
snowballstemmer==1.2.1 snowballstemmer==1.2.1
sphinx-sitemap==0.2 sphinx-sitemap==0.2
sphinx==1.7.4 sphinx==1.7.6
sphinxcontrib-websupport==1.0.1 sphinxcontrib-websupport==1.1.0
urllib3==1.22 urllib3==1.23
whichcraft==0.4.1 whichcraft==0.4.1
yapf==0.22.0

View File

@ -5,26 +5,26 @@ bonobo-docker==0.6.0
certifi==2018.4.16 certifi==2018.4.16
chardet==3.0.4 chardet==3.0.4
colorama==0.3.9 colorama==0.3.9
docker-pycreds==0.2.3 docker-pycreds==0.3.0
docker==2.7.0 docker==2.7.0
fs==2.0.23 fs==2.0.27
graphviz==0.8.3 graphviz==0.8.4
idna==2.6 idna==2.7
jinja2==2.10 jinja2==2.10
markupsafe==1.0 markupsafe==1.0
mondrian==0.7.0 mondrian==0.7.0
packaging==17.1 packaging==17.1
pbr==4.0.3 pbr==4.2.0
psutil==5.4.5 psutil==5.4.6
pyparsing==2.2.0 pyparsing==2.2.0
python-slugify==1.2.5 python-slugify==1.2.5
pytz==2018.4 pytz==2018.5
requests==2.18.4 requests==2.19.1
semantic-version==2.6.0 semantic-version==2.6.0
six==1.11.0 six==1.11.0
stevedore==1.28.0 stevedore==1.29.0
typing==3.6.4 typing==3.6.4
unidecode==1.0.22 unidecode==1.0.22
urllib3==1.22 urllib3==1.23
websocket-client==0.47.0 websocket-client==0.48.0
whistle==1.0.1 whistle==1.0.1

View File

@ -8,9 +8,9 @@ entrypoints==0.2.3
html5lib==1.0.1 html5lib==1.0.1
ipykernel==4.8.2 ipykernel==4.8.2
ipython-genutils==0.2.0 ipython-genutils==0.2.0
ipython==6.4.0 ipython==6.5.0
ipywidgets==6.0.1 ipywidgets==6.0.1
jedi==0.12.0 jedi==0.12.1
jinja2==2.10 jinja2==2.10
jsonschema==2.6.0 jsonschema==2.6.0
jupyter-client==5.2.3 jupyter-client==5.2.3
@ -21,23 +21,24 @@ markupsafe==1.0
mistune==0.8.3 mistune==0.8.3
nbconvert==5.3.1 nbconvert==5.3.1
nbformat==4.4.0 nbformat==4.4.0
notebook==5.5.0 notebook==5.6.0
pandocfilters==1.4.2 pandocfilters==1.4.2
parso==0.2.0 parso==0.3.1
pexpect==4.5.0 pexpect==4.6.0
pickleshare==0.7.4 pickleshare==0.7.4
prometheus-client==0.3.1
prompt-toolkit==1.0.15 prompt-toolkit==1.0.15
ptyprocess==0.5.2 ptyprocess==0.6.0
pygments==2.2.0 pygments==2.2.0
python-dateutil==2.7.3 python-dateutil==2.7.3
pyzmq==17.0.0 pyzmq==17.1.2
qtconsole==4.3.1 qtconsole==4.3.1
send2trash==1.5.0 send2trash==1.5.0
simplegeneric==0.8.1 simplegeneric==0.8.1
six==1.11.0 six==1.11.0
terminado==0.8.1 terminado==0.8.1
testpath==0.3.1 testpath==0.3.1
tornado==5.0.2 tornado==5.1
traitlets==4.3.2 traitlets==4.3.2
wcwidth==0.1.7 wcwidth==0.1.7
webencodings==0.5.1 webencodings==0.5.1

View File

@ -5,23 +5,23 @@ bonobo-sqlalchemy==0.6.0
certifi==2018.4.16 certifi==2018.4.16
chardet==3.0.4 chardet==3.0.4
colorama==0.3.9 colorama==0.3.9
fs==2.0.23 fs==2.0.27
graphviz==0.8.3 graphviz==0.8.4
idna==2.6 idna==2.7
jinja2==2.10 jinja2==2.10
markupsafe==1.0 markupsafe==1.0
mondrian==0.7.0 mondrian==0.7.0
packaging==17.1 packaging==17.1
pbr==4.0.3 pbr==4.2.0
psutil==5.4.5 psutil==5.4.6
pyparsing==2.2.0 pyparsing==2.2.0
python-slugify==1.2.5 python-slugify==1.2.5
pytz==2018.4 pytz==2018.5
requests==2.18.4 requests==2.19.1
six==1.11.0 six==1.11.0
sqlalchemy==1.2.7 sqlalchemy==1.2.10
stevedore==1.28.0 stevedore==1.29.0
typing==3.6.4 typing==3.6.4
unidecode==1.0.22 unidecode==1.0.22
urllib3==1.22 urllib3==1.23
whistle==1.0.1 whistle==1.0.1

View File

@ -3,22 +3,22 @@ appdirs==1.4.3
certifi==2018.4.16 certifi==2018.4.16
chardet==3.0.4 chardet==3.0.4
colorama==0.3.9 colorama==0.3.9
fs==2.0.23 fs==2.0.27
graphviz==0.8.3 graphviz==0.8.4
idna==2.6 idna==2.7
jinja2==2.10 jinja2==2.10
markupsafe==1.0 markupsafe==1.0
mondrian==0.7.0 mondrian==0.7.0
packaging==17.1 packaging==17.1
pbr==4.0.3 pbr==4.2.0
psutil==5.4.5 psutil==5.4.6
pyparsing==2.2.0 pyparsing==2.2.0
python-slugify==1.2.5 python-slugify==1.2.5
pytz==2018.4 pytz==2018.5
requests==2.18.4 requests==2.19.1
six==1.11.0 six==1.11.0
stevedore==1.28.0 stevedore==1.29.0
typing==3.6.4 typing==3.6.4
unidecode==1.0.22 unidecode==1.0.22
urllib3==1.22 urllib3==1.23
whistle==1.0.1 whistle==1.0.1

View File

@ -1,11 +1,12 @@
# Generated by Medikit 0.6.1 on 2018-05-21. # Generated by Medikit 0.6.3 on 2018-08-11.
# All changes will be overriden. # All changes will be overriden.
# Edit Projectfile and run “make update” (or “medikit update”) to regenerate. # Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
from setuptools import setup, find_packages
from codecs import open from codecs import open
from os import path from os import path
from setuptools import find_packages, setup
here = path.abspath(path.dirname(__file__)) here = path.abspath(path.dirname(__file__))
# Py3 compatibility hacks, borrowed from IPython. # Py3 compatibility hacks, borrowed from IPython.
@ -44,14 +45,17 @@ else:
setup( setup(
author='Romain Dorgueil', author='Romain Dorgueil',
author_email='romain@dorgueil.net', author_email='romain@dorgueil.net',
data_files=[( data_files=[
'share/jupyter/nbextensions/bonobo-jupyter', [ (
'bonobo/contrib/jupyter/static/extension.js', 'bonobo/contrib/jupyter/static/index.js', 'share/jupyter/nbextensions/bonobo-jupyter',
'bonobo/contrib/jupyter/static/index.js.map' [
] 'bonobo/contrib/jupyter/static/extension.js',
)], 'bonobo/contrib/jupyter/static/index.js',
description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' 'bonobo/contrib/jupyter/static/index.js.map',
'python 3.5+.'), ],
)
],
description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' 'python 3.5+.'),
license='Apache License, Version 2.0', license='Apache License, Version 2.0',
name='bonobo', name='bonobo',
version=version, version=version,
@ -60,26 +64,42 @@ setup(
packages=find_packages(exclude=['ez_setup', 'example', 'test']), packages=find_packages(exclude=['ez_setup', 'example', 'test']),
include_package_data=True, include_package_data=True,
install_requires=[ install_requires=[
'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (~= 2.9)', 'mondrian (~= 0.7)', 'packaging (~= 17.0)', 'fs (~= 2.0)',
'psutil (~= 5.4)', 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)', 'stevedore (~= 1.27)', 'whistle (~= 1.0)' 'graphviz (>= 0.8, < 0.9)',
'jinja2 (~= 2.9)',
'mondrian (~= 0.7)',
'packaging (~= 17.0)',
'psutil (~= 5.4)',
'python-slugify (~= 1.2.0)',
'requests (~= 2.0)',
'stevedore (~= 1.27)',
'whistle (~= 1.0)',
], ],
extras_require={ extras_require={
'dev': [ 'dev': [
'cookiecutter (>= 1.5, < 1.6)', 'coverage (~= 4.4)', 'pytest (~= 3.4)', 'pytest-cov (~= 2.5)', 'cookiecutter (>= 1.5, < 1.6)',
'pytest-timeout (>= 1, < 2)', 'sphinx (~= 1.7)', 'sphinx-sitemap (>= 0.2, < 0.3)', 'yapf' 'coverage (~= 4.4)',
'pytest (~= 3.4)',
'pytest-cov (~= 2.5)',
'pytest-timeout (>= 1, < 2)',
'sphinx (~= 1.7)',
'sphinx-sitemap (>= 0.2, < 0.3)',
], ],
'docker': ['bonobo-docker (~= 0.6.0a1)'], 'docker': ['bonobo-docker (~= 0.6.0a1)'],
'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'], 'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'],
'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)'] 'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)'],
}, },
entry_points={ entry_points={
'bonobo.commands': [ 'bonobo.commands': [
'convert = bonobo.commands.convert:ConvertCommand', 'download = bonobo.commands.download:DownloadCommand', 'convert = bonobo.commands.convert:ConvertCommand',
'examples = bonobo.commands.examples:ExamplesCommand', 'init = bonobo.commands.init:InitCommand', 'download = bonobo.commands.download:DownloadCommand',
'inspect = bonobo.commands.inspect:InspectCommand', 'run = bonobo.commands.run:RunCommand', 'examples = bonobo.commands.examples:ExamplesCommand',
'version = bonobo.commands.version:VersionCommand' 'init = bonobo.commands.init:InitCommand',
'inspect = bonobo.commands.inspect:InspectCommand',
'run = bonobo.commands.run:RunCommand',
'version = bonobo.commands.version:VersionCommand',
], ],
'console_scripts': ['bonobo = bonobo.commands:entrypoint'] 'console_scripts': ['bonobo = bonobo.commands:entrypoint'],
}, },
url='https://www.bonobo-project.org/', url='https://www.bonobo-project.org/',
download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version),

View File

@ -9,17 +9,11 @@ def test_entrypoint():
for command in pkg_resources.iter_entry_points('bonobo.commands'): for command in pkg_resources.iter_entry_points('bonobo.commands'):
commands[command.name] = command commands[command.name] = command
assert not { assert not {'convert', 'init', 'inspect', 'run', 'version'}.difference(set(commands))
'convert',
'init',
'inspect',
'run',
'version',
}.difference(set(commands))
@all_runners @all_runners
def test_no_command(runner): def test_no_command(runner):
_, err, exc = runner(catch_errors=True) _, err, exc = runner(catch_errors=True)
assert type(exc) == SystemExit assert type(exc) == SystemExit
assert 'error: the following arguments are required: command' in err assert 'error: the following arguments are required: command' in err

View File

@ -27,8 +27,9 @@ def test_download_works_for_examples(runner):
fout = io.BytesIO() fout = io.BytesIO()
fout.close = lambda: None fout.close = lambda: None
with patch('bonobo.commands.download._open_url') as mock_open_url, \ with patch('bonobo.commands.download._open_url') as mock_open_url, patch(
patch('bonobo.commands.download.open') as mock_open: 'bonobo.commands.download.open'
) as mock_open:
mock_open_url.return_value = MockResponse() mock_open_url.return_value = MockResponse()
mock_open.return_value = fout mock_open.return_value = fout
runner('download', 'examples/datasets/coffeeshops.txt') runner('download', 'examples/datasets/coffeeshops.txt')
@ -41,4 +42,4 @@ def test_download_works_for_examples(runner):
@all_runners @all_runners
def test_download_fails_non_example(runner): def test_download_fails_non_example(runner):
with pytest.raises(ValueError): with pytest.raises(ValueError):
runner('download', 'something/entirely/different.txt') runner('download', 'something/entirely/different.txt')

View File

@ -6,21 +6,14 @@ from bonobo.util.testing import EnvironmentTestCase
@pytest.fixture @pytest.fixture
def env1(tmpdir): def env1(tmpdir):
env_file = tmpdir.join('.env_one') env_file = tmpdir.join('.env_one')
env_file.write('\n'.join(( env_file.write('\n'.join(('SECRET=unknown', 'PASSWORD=sweet', 'PATH=first')))
'SECRET=unknown',
'PASSWORD=sweet',
'PATH=first',
)))
return str(env_file) return str(env_file)
@pytest.fixture @pytest.fixture
def env2(tmpdir): def env2(tmpdir):
env_file = tmpdir.join('.env_two') env_file = tmpdir.join('.env_two')
env_file.write('\n'.join(( env_file.write('\n'.join(('PASSWORD=bitter', "PATH='second'")))
'PASSWORD=bitter',
"PATH='second'",
)))
return str(env_file) return str(env_file)
@ -71,7 +64,15 @@ class TestEnvFileCombinations(EnvironmentTestCase):
def test_run_with_both_env_files_then_overrides(self, runner, target, env1, env2): def test_run_with_both_env_files_then_overrides(self, runner, target, env1, env2):
env = self.run_environ( env = self.run_environ(
runner, *target, '--default-env-file', env1, '--env-file', env2, '--env', 'PASSWORD=mine', '--env', runner,
*target,
'--default-env-file',
env1,
'--env-file',
env2,
'--env',
'PASSWORD=mine',
'--env',
'SECRET=s3cr3t' 'SECRET=s3cr3t'
) )
assert env.get('SECRET') == 's3cr3t' assert env.get('SECRET') == 's3cr3t'

View File

@ -17,4 +17,4 @@ def test_version(runner):
out, err = runner('version', '-qq') out, err = runner('version', '-qq')
out = out.strip() out = out.strip()
assert not out.startswith('bonobo ') assert not out.startswith('bonobo ')
assert __version__ in out assert __version__ in out

View File

@ -50,10 +50,7 @@ def test_define_with_decorator():
calls = [] calls = []
def my_handler(*args, **kwargs): def my_handler(*args, **kwargs):
calls.append(( calls.append((args, kwargs))
args,
kwargs,
))
Concrete = MethodBasedConfigurable(my_handler) Concrete = MethodBasedConfigurable(my_handler)
@ -77,10 +74,7 @@ def test_late_binding_method_decoration():
@MethodBasedConfigurable(foo='foo') @MethodBasedConfigurable(foo='foo')
def Concrete(*args, **kwargs): def Concrete(*args, **kwargs):
calls.append(( calls.append((args, kwargs))
args,
kwargs,
))
assert callable(Concrete.handler) assert callable(Concrete.handler)
t = Concrete(bar='baz') t = Concrete(bar='baz')
@ -95,10 +89,7 @@ def test_define_with_argument():
calls = [] calls = []
def concrete_handler(*args, **kwargs): def concrete_handler(*args, **kwargs):
calls.append(( calls.append((args, kwargs))
args,
kwargs,
))
t = MethodBasedConfigurable(concrete_handler, 'foo', bar='baz') t = MethodBasedConfigurable(concrete_handler, 'foo', bar='baz')
assert callable(t.handler) assert callable(t.handler)
@ -112,10 +103,7 @@ def test_define_with_inheritance():
class Inheriting(MethodBasedConfigurable): class Inheriting(MethodBasedConfigurable):
def handler(self, *args, **kwargs): def handler(self, *args, **kwargs):
calls.append(( calls.append((args, kwargs))
args,
kwargs,
))
t = Inheriting('foo', bar='baz') t = Inheriting('foo', bar='baz')
assert callable(t.handler) assert callable(t.handler)
@ -132,10 +120,7 @@ def test_inheritance_then_decorate():
@Inheriting @Inheriting
def Concrete(*args, **kwargs): def Concrete(*args, **kwargs):
calls.append(( calls.append((args, kwargs))
args,
kwargs,
))
assert callable(Concrete.handler) assert callable(Concrete.handler)
t = Concrete('foo', bar='baz') t = Concrete('foo', bar='baz')

View File

@ -40,7 +40,7 @@ def test_partial():
assert len(ci.options) == 4 assert len(ci.options) == 4
assert len(ci.processors) == 1 assert len(ci.processors) == 1
assert ci.partial assert ci.partial
assert ci.partial[0] == (f1, ) assert ci.partial[0] == (f1,)
assert not len(ci.partial[1]) assert not len(ci.partial[1])
# instanciate a more complete partial instance ... # instanciate a more complete partial instance ...
@ -53,10 +53,7 @@ def test_partial():
assert len(ci.options) == 4 assert len(ci.options) == 4
assert len(ci.processors) == 1 assert len(ci.processors) == 1
assert ci.partial assert ci.partial
assert ci.partial[0] == ( assert ci.partial[0] == (f1, f2)
f1,
f2,
)
assert not len(ci.partial[1]) assert not len(ci.partial[1])
c = C('foo') c = C('foo')

View File

@ -1,7 +1,7 @@
from operator import attrgetter from operator import attrgetter
from bonobo.config import Configurable from bonobo.config import Configurable
from bonobo.config.processors import ContextProcessor, resolve_processors, ContextCurrifier, use_context_processor from bonobo.config.processors import ContextCurrifier, ContextProcessor, resolve_processors, use_context_processor
class CP1(Configurable): class CP1(Configurable):

View File

@ -4,11 +4,11 @@ import time
import pytest import pytest
from bonobo.config import Configurable, Container, Exclusive, Service, use from bonobo.config import Configurable, Container, Exclusive, Service, use
from bonobo.config.services import validate_service_name, create_container from bonobo.config.services import create_container, validate_service_name
from bonobo.util import get_name from bonobo.util import get_name
class PrinterInterface(): class PrinterInterface:
def print(self, *args): def print(self, *args):
raise NotImplementedError() raise NotImplementedError()
@ -21,14 +21,11 @@ class ConcretePrinter(PrinterInterface):
return ';'.join((self.prefix, *args)) return ';'.join((self.prefix, *args))
SERVICES = Container( SERVICES = Container(printer0=ConcretePrinter(prefix='0'), printer1=ConcretePrinter(prefix='1'))
printer0=ConcretePrinter(prefix='0'),
printer1=ConcretePrinter(prefix='1'),
)
class MyServiceDependantConfigurable(Configurable): class MyServiceDependantConfigurable(Configurable):
printer = Service(PrinterInterface, ) printer = Service(PrinterInterface)
def __call__(self, *args, printer: PrinterInterface): def __call__(self, *args, printer: PrinterInterface):
return printer.print(*args) return printer.print(*args)
@ -80,7 +77,7 @@ def test_exclusive():
vcr.append(' '.join((prefix, str(i)))) vcr.append(' '.join((prefix, str(i))))
time.sleep(0.05) time.sleep(0.05)
threads = [threading.Thread(target=record, args=(str(i), )) for i in range(5)] threads = [threading.Thread(target=record, args=(str(i),)) for i in range(5)]
for thread in threads: for thread in threads:
thread.start() thread.start()
@ -90,8 +87,32 @@ def test_exclusive():
thread.join() thread.join()
assert vcr.tape == [ assert vcr.tape == [
'hello', '0 0', '0 1', '0 2', '0 3', '0 4', '1 0', '1 1', '1 2', '1 3', '1 4', '2 0', '2 1', '2 2', '2 3', 'hello',
'2 4', '3 0', '3 1', '3 2', '3 3', '3 4', '4 0', '4 1', '4 2', '4 3', '4 4' '0 0',
'0 1',
'0 2',
'0 3',
'0 4',
'1 0',
'1 1',
'1 2',
'1 3',
'1 4',
'2 0',
'2 1',
'2 2',
'2 3',
'2 4',
'3 0',
'3 1',
'3 2',
'3 3',
'3 4',
'4 0',
'4 1',
'4 2',
'4 3',
'4 4',
] ]
@ -118,10 +139,7 @@ def test_create_container_empty_values(services):
def test_create_container_override(): def test_create_container_override():
c = create_container({ c = create_container({'http': 'http', 'fs': 'fs'})
'http': 'http',
'fs': 'fs',
})
assert len(c) == 2 assert len(c) == 2
assert 'fs' in c and c['fs'] == 'fs' assert 'fs' in c and c['fs'] == 'fs'
assert 'http' in c and c['http'] == 'http' assert 'http' in c and c['http'] == 'http'

View File

@ -3,10 +3,10 @@ from unittest.mock import MagicMock
import pytest import pytest
from bonobo import Graph from bonobo import Graph
from bonobo.constants import EMPTY, NOT_MODIFIED, INHERIT from bonobo.constants import EMPTY, INHERIT, NOT_MODIFIED
from bonobo.execution.contexts.node import NodeExecutionContext, split_token from bonobo.execution.contexts.node import NodeExecutionContext, split_token
from bonobo.execution.strategies import NaiveStrategy from bonobo.execution.strategies import NaiveStrategy
from bonobo.util.testing import BufferingNodeExecutionContext, BufferingGraphExecutionContext from bonobo.util.testing import BufferingGraphExecutionContext, BufferingNodeExecutionContext
def test_node_string(): def test_node_string():
@ -18,7 +18,7 @@ def test_node_string():
output = context.get_buffer() output = context.get_buffer()
assert len(output) == 1 assert len(output) == 1
assert output[0] == ('foo', ) assert output[0] == ('foo',)
def g(): def g():
yield 'foo' yield 'foo'
@ -29,8 +29,8 @@ def test_node_string():
output = context.get_buffer() output = context.get_buffer()
assert len(output) == 2 assert len(output) == 2
assert output[0] == ('foo', ) assert output[0] == ('foo',)
assert output[1] == ('bar', ) assert output[1] == ('bar',)
def test_node_bytes(): def test_node_bytes():
@ -42,7 +42,7 @@ def test_node_bytes():
output = context.get_buffer() output = context.get_buffer()
assert len(output) == 1 assert len(output) == 1
assert output[0] == (b'foo', ) assert output[0] == (b'foo',)
def g(): def g():
yield b'foo' yield b'foo'
@ -53,8 +53,8 @@ def test_node_bytes():
output = context.get_buffer() output = context.get_buffer()
assert len(output) == 2 assert len(output) == 2
assert output[0] == (b'foo', ) assert output[0] == (b'foo',)
assert output[1] == (b'bar', ) assert output[1] == (b'bar',)
def test_node_dict(): def test_node_dict():
@ -65,7 +65,7 @@ def test_node_dict():
context.write_sync(EMPTY) context.write_sync(EMPTY)
output = context.get_buffer() output = context.get_buffer()
assert len(output) == 1 assert len(output) == 1
assert output[0] == ({'id': 1, 'name': 'foo'}, ) assert output[0] == ({'id': 1, 'name': 'foo'},)
def g(): def g():
yield {'id': 1, 'name': 'foo'} yield {'id': 1, 'name': 'foo'}
@ -75,8 +75,8 @@ def test_node_dict():
context.write_sync(EMPTY) context.write_sync(EMPTY)
output = context.get_buffer() output = context.get_buffer()
assert len(output) == 2 assert len(output) == 2
assert output[0] == ({'id': 1, 'name': 'foo'}, ) assert output[0] == ({'id': 1, 'name': 'foo'},)
assert output[1] == ({'id': 2, 'name': 'bar'}, ) assert output[1] == ({'id': 2, 'name': 'bar'},)
def test_node_dict_chained(): def test_node_dict_chained():
@ -93,7 +93,7 @@ def test_node_dict_chained():
output = context.get_buffer() output = context.get_buffer()
assert len(output) == 1 assert len(output) == 1
assert output[0] == ({'id': 1, 'name': 'FOO'}, ) assert output[0] == ({'id': 1, 'name': 'FOO'},)
def g(): def g():
yield {'id': 1, 'name': 'foo'} yield {'id': 1, 'name': 'foo'}
@ -104,8 +104,8 @@ def test_node_dict_chained():
output = context.get_buffer() output = context.get_buffer()
assert len(output) == 2 assert len(output) == 2
assert output[0] == ({'id': 1, 'name': 'FOO'}, ) assert output[0] == ({'id': 1, 'name': 'FOO'},)
assert output[1] == ({'id': 2, 'name': 'BAR'}, ) assert output[1] == ({'id': 2, 'name': 'BAR'},)
def test_node_tuple(): def test_node_tuple():
@ -229,7 +229,7 @@ def test_node_lifecycle_with_kill():
def test_split_token(): def test_split_token():
assert split_token(('foo', 'bar')) == (set(), ('foo', 'bar')) assert split_token(('foo', 'bar')) == (set(), ('foo', 'bar'))
assert split_token(()) == (set(), ()) assert split_token(()) == (set(), ())
assert split_token('') == (set(), ('', )) assert split_token('') == (set(), ('',))
def test_split_token_duplicate(): def test_split_token_duplicate():
@ -249,10 +249,10 @@ def test_split_token_not_modified():
with pytest.raises(ValueError): with pytest.raises(ValueError):
split_token((INHERIT, NOT_MODIFIED)) split_token((INHERIT, NOT_MODIFIED))
assert split_token(NOT_MODIFIED) == ({NOT_MODIFIED}, ()) assert split_token(NOT_MODIFIED) == ({NOT_MODIFIED}, ())
assert split_token((NOT_MODIFIED, )) == ({NOT_MODIFIED}, ()) assert split_token((NOT_MODIFIED,)) == ({NOT_MODIFIED}, ())
def test_split_token_inherit(): def test_split_token_inherit():
assert split_token(INHERIT) == ({INHERIT}, ()) assert split_token(INHERIT) == ({INHERIT}, ())
assert split_token((INHERIT, )) == ({INHERIT}, ()) assert split_token((INHERIT,)) == ({INHERIT}, ())
assert split_token((INHERIT, 'foo', 'bar')) == ({INHERIT}, ('foo', 'bar')) assert split_token((INHERIT, 'foo', 'bar')) == ({INHERIT}, ('foo', 'bar'))

View File

@ -14,16 +14,11 @@ class ResponseMock:
return {} return {}
else: else:
self.count += 1 self.count += 1
return { return {'records': self.json_value}
'records': self.json_value,
}
def test_read_from_opendatasoft_api(): def test_read_from_opendatasoft_api():
extract = OpenDataSoftAPI(dataset='test-a-set') extract = OpenDataSoftAPI(dataset='test-a-set')
with patch('requests.get', return_value=ResponseMock([ with patch('requests.get', return_value=ResponseMock([{'fields': {'foo': 'bar'}}, {'fields': {'foo': 'zab'}}])):
{'fields': {'foo': 'bar'}},
{'fields': {'foo': 'zab'}},
])):
for line in extract('http://example.com/', ValueHolder(0)): for line in extract('http://example.com/', ValueHolder(0)):
assert 'foo' in line assert 'foo' in line

View File

@ -1,10 +1,7 @@
from bonobo.constants import INHERIT from bonobo.constants import INHERIT
from bonobo.util.testing import BufferingNodeExecutionContext from bonobo.util.testing import BufferingNodeExecutionContext
messages = [ messages = [('Hello',), ('Goodbye',)]
('Hello', ),
('Goodbye', ),
]
def append(*args): def append(*args):
@ -15,7 +12,7 @@ def test_inherit():
with BufferingNodeExecutionContext(append) as context: with BufferingNodeExecutionContext(append) as context:
context.write_sync(*messages) context.write_sync(*messages)
assert context.get_buffer() == list(map(lambda x: x + ('!', ), messages)) assert context.get_buffer() == list(map(lambda x: x + ('!',), messages))
def test_inherit_bag_tuple(): def test_inherit_bag_tuple():
@ -24,4 +21,4 @@ def test_inherit_bag_tuple():
context.write_sync(*messages) context.write_sync(*messages)
assert context.get_output_fields() == ('message', '0') assert context.get_output_fields() == ('message', '0')
assert context.get_buffer() == list(map(lambda x: x + ('!', ), messages)) assert context.get_buffer() == list(map(lambda x: x + ('!',), messages))

View File

@ -7,10 +7,7 @@ def useless(*args, **kwargs):
def test_not_modified(): def test_not_modified():
input_messages = [ input_messages = [('foo', 'bar'), ('foo', 'baz')]
('foo', 'bar'),
('foo', 'baz'),
]
with BufferingNodeExecutionContext(useless) as context: with BufferingNodeExecutionContext(useless) as context:
context.write_sync(*input_messages) context.write_sync(*input_messages)

View File

@ -6,8 +6,9 @@ import pytest
from bonobo import CsvReader, CsvWriter from bonobo import CsvReader, CsvWriter
from bonobo.constants import EMPTY from bonobo.constants import EMPTY
from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext, WriterTest, ConfigurableNodeTest, \ from bonobo.util.testing import (
ReaderTest BufferingNodeExecutionContext, ConfigurableNodeTest, FilesystemTester, ReaderTest, WriterTest
)
csv_tester = FilesystemTester('csv') csv_tester = FilesystemTester('csv')
csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar' csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar'
@ -23,15 +24,10 @@ def test_read_csv_from_file_kwargs(tmpdir):
with BufferingNodeExecutionContext(CsvReader(filename, **defaults), services=services) as context: with BufferingNodeExecutionContext(CsvReader(filename, **defaults), services=services) as context:
context.write_sync(EMPTY) context.write_sync(EMPTY)
assert context.get_buffer_args_as_dicts() == [{ assert context.get_buffer_args_as_dicts() == [
'a': 'a foo', {'a': 'a foo', 'b': 'b foo', 'c': 'c foo'},
'b': 'b foo', {'a': 'a bar', 'b': 'b bar', 'c': 'c bar'},
'c': 'c foo', ]
}, {
'a': 'a bar',
'b': 'b bar',
'c': 'c bar',
}]
### ###
@ -50,22 +46,11 @@ LL = ('i', 'have', 'more', 'values')
class CsvReaderTest(Csv, ReaderTest, TestCase): class CsvReaderTest(Csv, ReaderTest, TestCase):
input_data = '\n'.join(( input_data = '\n'.join(('id,name', '1,John Doe', '2,Jane Doe', ',DPR', '42,Elon Musk'))
'id,name',
'1,John Doe',
'2,Jane Doe',
',DPR',
'42,Elon Musk',
))
def check_output(self, context, *, prepend=None): def check_output(self, context, *, prepend=None):
out = context.get_buffer() out = context.get_buffer()
assert out == (prepend or list()) + [ assert out == (prepend or list()) + [('1', 'John Doe'), ('2', 'Jane Doe'), ('', 'DPR'), ('42', 'Elon Musk')]
('1', 'John Doe'),
('2', 'Jane Doe'),
('', 'DPR'),
('42', 'Elon Musk'),
]
@incontext() @incontext()
def test_nofields(self, context): def test_nofields(self, context):
@ -80,12 +65,7 @@ class CsvReaderTest(Csv, ReaderTest, TestCase):
context.stop() context.stop()
self.check_output(context, prepend=[('id', 'name')]) self.check_output(context, prepend=[('id', 'name')])
@incontext( @incontext(output_fields=('x', 'y'), skip=1)
output_fields=(
'x',
'y',
), skip=1
)
def test_output_fields(self, context): def test_output_fields(self, context):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
@ -107,11 +87,7 @@ class CsvWriterTest(Csv, WriterTest, TestCase):
context.write_sync(('a', 'b'), ('c', 'd')) context.write_sync(('a', 'b'), ('c', 'd'))
context.stop() context.stop()
assert self.readlines() == ( assert self.readlines() == ('foo,bar', 'a,b', 'c,d')
'foo,bar',
'a,b',
'c,d',
)
@incontext() @incontext()
def test_fields_from_type(self, context): def test_fields_from_type(self, context):
@ -127,30 +103,21 @@ class CsvWriterTest(Csv, WriterTest, TestCase):
context.write_sync((L1, L2), (L3, L4)) context.write_sync((L1, L2), (L3, L4))
context.stop() context.stop()
assert self.readlines() == ( assert self.readlines() == ('a,hey', 'b,bee', 'c,see', 'd,dee')
'a,hey',
'b,bee',
'c,see',
'd,dee',
)
@incontext() @incontext()
def test_nofields_multiple_args_length_mismatch(self, context): def test_nofields_multiple_args_length_mismatch(self, context):
# if length of input vary, then we get a TypeError (unrecoverable) # if length of input vary, then we get a TypeError (unrecoverable)
with pytest.raises(TypeError): with pytest.raises(TypeError):
context.write_sync((L1, L2), (L3, )) context.write_sync((L1, L2), (L3,))
@incontext() @incontext()
def test_nofields_single_arg(self, context): def test_nofields_single_arg(self, context):
# single args are just dumped, shapes can vary. # single args are just dumped, shapes can vary.
context.write_sync((L1, ), (LL, ), (L3, )) context.write_sync((L1,), (LL,), (L3,))
context.stop() context.stop()
assert self.readlines() == ( assert self.readlines() == ('a,hey', 'i,have,more,values', 'c,see')
'a,hey',
'i,have,more,values',
'c,see',
)
@incontext() @incontext()
def test_nofields_empty_args(self, context): def test_nofields_empty_args(self, context):

View File

@ -21,10 +21,7 @@ def test_file_writer_contextless(tmpdir):
@pytest.mark.parametrize( @pytest.mark.parametrize(
'lines,output', 'lines,output',
[ [(('ACME',), 'ACME'), (('Foo', 'Bar', 'Baz'), 'Foo\nBar\nBaz')], # one line... # more than one line...
(('ACME', ), 'ACME'), # one line...
(('Foo', 'Bar', 'Baz'), 'Foo\nBar\nBaz'), # more than one line...
]
) )
def test_file_writer_in_context(tmpdir, lines, output): def test_file_writer_in_context(tmpdir, lines, output):
fs, filename, services = txt_tester.get_services_for_writer(tmpdir) fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
@ -44,5 +41,5 @@ def test_file_reader(tmpdir):
output = context.get_buffer() output = context.get_buffer()
assert len(output) == 2 assert len(output) == 2
assert output[0] == ('Hello', ) assert output[0] == ('Hello',)
assert output[1] == ('World', ) assert output[1] == ('World',)

View File

@ -4,10 +4,9 @@ from unittest import TestCase
import pytest import pytest
from bonobo import JsonReader, JsonWriter from bonobo import JsonReader, JsonWriter, LdjsonReader, LdjsonWriter
from bonobo import LdjsonReader, LdjsonWriter
from bonobo.constants import EMPTY from bonobo.constants import EMPTY
from bonobo.util.testing import WriterTest, ReaderTest, ConfigurableNodeTest from bonobo.util.testing import ConfigurableNodeTest, ReaderTest, WriterTest
FOOBAR = {'foo': 'bar'} FOOBAR = {'foo': 'bar'}
OD_ABC = OrderedDict((('a', 'A'), ('b', 'B'), ('c', 'C'))) OD_ABC = OrderedDict((('a', 'A'), ('b', 'B'), ('c', 'C')))
@ -34,14 +33,7 @@ class JsonReaderDictsTest(Json, ReaderTest, TestCase):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)]
({
"foo": "bar"
}, ),
({
"baz": "boz"
}, ),
]
class JsonReaderListsTest(Json, ReaderTest, TestCase): class JsonReaderListsTest(Json, ReaderTest, TestCase):
@ -52,20 +44,14 @@ class JsonReaderListsTest(Json, ReaderTest, TestCase):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
([1, 2, 3], ),
([4, 5, 6], ),
]
@incontext(output_type=tuple) @incontext(output_type=tuple)
def test_output_type(self, context): def test_output_type(self, context):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
([1, 2, 3], ),
([4, 5, 6], ),
]
class JsonReaderStringsTest(Json, ReaderTest, TestCase): class JsonReaderStringsTest(Json, ReaderTest, TestCase):
@ -76,22 +62,14 @@ class JsonReaderStringsTest(Json, ReaderTest, TestCase):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [('foo',), ('bar',), ('baz',)]
('foo', ),
('bar', ),
('baz', ),
]
@incontext(output_type=tuple) @incontext(output_type=tuple)
def test_output_type(self, context): def test_output_type(self, context):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [('foo',), ('bar',), ('baz',)]
('foo', ),
('bar', ),
('baz', ),
]
class JsonWriterTest(Json, WriterTest, TestCase): class JsonWriterTest(Json, WriterTest, TestCase):
@ -101,10 +79,7 @@ class JsonWriterTest(Json, WriterTest, TestCase):
context.write_sync(('a', 'b'), ('c', 'd')) context.write_sync(('a', 'b'), ('c', 'd'))
context.stop() context.stop()
assert self.readlines() == ( assert self.readlines() == ('[{"foo": "a", "bar": "b"},', '{"foo": "c", "bar": "d"}]')
'[{"foo": "a", "bar": "b"},',
'{"foo": "c", "bar": "d"}]',
)
@incontext() @incontext()
def test_fields_from_type(self, context): def test_fields_from_type(self, context):
@ -112,10 +87,7 @@ class JsonWriterTest(Json, WriterTest, TestCase):
context.write_sync((1, 2), (3, 4)) context.write_sync((1, 2), (3, 4))
context.stop() context.stop()
assert self.readlines() == ( assert self.readlines() == ('[{"x": 1, "y": 2},', '{"x": 3, "y": 4}]')
'[{"x": 1, "y": 2},',
'{"x": 3, "y": 4}]',
)
@incontext() @incontext()
def test_nofields_multiple_args(self, context): def test_nofields_multiple_args(self, context):
@ -144,11 +116,7 @@ class JsonWriterTest(Json, WriterTest, TestCase):
context.write_sync(FOOBAR, OD_ABC, FOOBAZ) context.write_sync(FOOBAR, OD_ABC, FOOBAZ)
context.stop() context.stop()
assert self.readlines() == ( assert self.readlines() == ('[{"foo": "bar"},', '{"a": "A", "b": "B", "c": "C"},', '{"foo": "baz"}]')
'[{"foo": "bar"},',
'{"a": "A", "b": "B", "c": "C"},',
'{"foo": "baz"}]',
)
@incontext() @incontext()
def test_nofields_empty_args(self, context): def test_nofields_empty_args(self, context):
@ -156,7 +124,7 @@ class JsonWriterTest(Json, WriterTest, TestCase):
context.write_sync(EMPTY, EMPTY, EMPTY) context.write_sync(EMPTY, EMPTY, EMPTY)
context.stop() context.stop()
assert self.readlines() == ('[]', ) assert self.readlines() == ('[]',)
### ###
@ -178,14 +146,7 @@ class LdjsonReaderDictsTest(Ldjson, ReaderTest, TestCase):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)]
({
"foo": "bar"
}, ),
({
"baz": "boz"
}, ),
]
class LdjsonReaderListsTest(Ldjson, ReaderTest, TestCase): class LdjsonReaderListsTest(Ldjson, ReaderTest, TestCase):
@ -196,20 +157,14 @@ class LdjsonReaderListsTest(Ldjson, ReaderTest, TestCase):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
([1, 2, 3], ),
([4, 5, 6], ),
]
@incontext(output_type=tuple) @incontext(output_type=tuple)
def test_output_type(self, context): def test_output_type(self, context):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
([1, 2, 3], ),
([4, 5, 6], ),
]
class LdjsonReaderStringsTest(Ldjson, ReaderTest, TestCase): class LdjsonReaderStringsTest(Ldjson, ReaderTest, TestCase):
@ -220,22 +175,14 @@ class LdjsonReaderStringsTest(Ldjson, ReaderTest, TestCase):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [('foo',), ('bar',), ('baz',)]
('foo', ),
('bar', ),
('baz', ),
]
@incontext(output_type=tuple) @incontext(output_type=tuple)
def test_output_type(self, context): def test_output_type(self, context):
context.write_sync(EMPTY) context.write_sync(EMPTY)
context.stop() context.stop()
assert context.get_buffer() == [ assert context.get_buffer() == [('foo',), ('bar',), ('baz',)]
('foo', ),
('bar', ),
('baz', ),
]
class LdjsonWriterTest(Ldjson, WriterTest, TestCase): class LdjsonWriterTest(Ldjson, WriterTest, TestCase):
@ -253,10 +200,7 @@ class LdjsonWriterTest(Ldjson, WriterTest, TestCase):
context.write_sync((1, 2), (3, 4)) context.write_sync((1, 2), (3, 4))
context.stop() context.stop()
assert self.readlines() == ( assert self.readlines() == ('{"x": 1, "y": 2}', '{"x": 3, "y": 4}')
'{"x": 1, "y": 2}',
'{"x": 3, "y": 4}',
)
@incontext() @incontext()
def test_nofields_multiple_args(self, context): def test_nofields_multiple_args(self, context):
@ -285,11 +229,7 @@ class LdjsonWriterTest(Ldjson, WriterTest, TestCase):
context.write_sync(FOOBAR, OD_ABC, FOOBAZ) context.write_sync(FOOBAR, OD_ABC, FOOBAZ)
context.stop() context.stop()
assert self.readlines() == ( assert self.readlines() == ('{"foo": "bar"}', '{"a": "A", "b": "B", "c": "C"}', '{"foo": "baz"}')
'{"foo": "bar"}',
'{"a": "A", "b": "B", "c": "C"}',
'{"foo": "baz"}',
)
@incontext() @incontext()
def test_nofields_empty_args(self, context): def test_nofields_empty_args(self, context):

View File

@ -32,7 +32,4 @@ def test_read_pickled_list_from_file(tmpdir):
output = context.get_buffer() output = context.get_buffer()
assert context.get_output_fields() == ('a', 'b', 'c') assert context.get_output_fields() == ('a', 'b', 'c')
assert output == [ assert output == [('a foo', 'b foo', 'c foo'), ('a bar', 'b bar', 'c bar')]
('a foo', 'b foo', 'c foo'),
('a bar', 'b bar', 'c bar'),
]

View File

@ -5,9 +5,9 @@ from unittest.mock import MagicMock
import pytest import pytest
import bonobo import bonobo
from bonobo.constants import NOT_MODIFIED, EMPTY from bonobo.constants import EMPTY, NOT_MODIFIED
from bonobo.util import ensure_tuple, ValueHolder from bonobo.util import ValueHolder, ensure_tuple
from bonobo.util.testing import BufferingNodeExecutionContext, StaticNodeTest, ConfigurableNodeTest from bonobo.util.testing import BufferingNodeExecutionContext, ConfigurableNodeTest, StaticNodeTest
class CountTest(StaticNodeTest, TestCase): class CountTest(StaticNodeTest, TestCase):
@ -26,7 +26,7 @@ class CountTest(StaticNodeTest, TestCase):
def test_execution(self): def test_execution(self):
with self.execute() as context: with self.execute() as context:
context.write_sync(*([EMPTY] * 42)) context.write_sync(*([EMPTY] * 42))
assert context.get_buffer() == [(42, )] assert context.get_buffer() == [(42,)]
class IdentityTest(StaticNodeTest, TestCase): class IdentityTest(StaticNodeTest, TestCase):
@ -98,14 +98,11 @@ def test_fixedwindow():
with BufferingNodeExecutionContext(bonobo.FixedWindow(2)) as context: with BufferingNodeExecutionContext(bonobo.FixedWindow(2)) as context:
context.write_sync(*range(9)) context.write_sync(*range(9))
assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), ( assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), (8, None)]
8,
None,
)]
with BufferingNodeExecutionContext(bonobo.FixedWindow(1)) as context: with BufferingNodeExecutionContext(bonobo.FixedWindow(1)) as context:
context.write_sync(*range(3)) context.write_sync(*range(3))
assert context.get_buffer() == [(0, ), (1, ), (2, )] assert context.get_buffer() == [(0,), (1,), (2,)]
def test_methodcaller(): def test_methodcaller():

View File

@ -1,10 +1,11 @@
from unittest.mock import MagicMock from unittest.mock import MagicMock
from whistle import EventDispatcher
import bonobo import bonobo
from bonobo.execution import events from bonobo.execution import events
from bonobo.execution.contexts.graph import GraphExecutionContext from bonobo.execution.contexts.graph import GraphExecutionContext
from bonobo.plugins.console import ConsoleOutputPlugin from bonobo.plugins.console import ConsoleOutputPlugin
from whistle import EventDispatcher
def test_register_unregister(): def test_register_unregister():

View File

@ -1,7 +1,7 @@
import pytest
from unittest.mock import sentinel from unittest.mock import sentinel
import pytest
from bonobo.constants import BEGIN from bonobo.constants import BEGIN
from bonobo.structs import Graph from bonobo.structs import Graph
@ -48,24 +48,14 @@ def test_graph_add_chain():
def test_graph_topological_sort(): def test_graph_topological_sort():
g = Graph() g = Graph()
g.add_chain( g.add_chain(sentinel.a1, sentinel.a2, sentinel.a3, _input=None, _output=None)
sentinel.a1,
sentinel.a2,
sentinel.a3,
_input=None,
_output=None,
)
assert g.topologically_sorted_indexes == (0, 1, 2) assert g.topologically_sorted_indexes == (0, 1, 2)
assert g[0] == sentinel.a1 assert g[0] == sentinel.a1
assert g[1] == sentinel.a2 assert g[1] == sentinel.a2
assert g[2] == sentinel.a3 assert g[2] == sentinel.a3
g.add_chain( g.add_chain(sentinel.b1, sentinel.b2, _output=sentinel.a2)
sentinel.b1,
sentinel.b2,
_output=sentinel.a2,
)
assert g.topologically_sorted_indexes[-2:] == (1, 2) assert g.topologically_sorted_indexes[-2:] == (1, 2)
assert g.topologically_sorted_indexes.index(3) < g.topologically_sorted_indexes.index(4) assert g.topologically_sorted_indexes.index(3) < g.topologically_sorted_indexes.index(4)

View File

@ -19,7 +19,7 @@ from queue import Empty
import pytest import pytest
from bonobo.constants import BEGIN, END from bonobo.constants import BEGIN, END
from bonobo.errors import InactiveWritableError, InactiveReadableError from bonobo.errors import InactiveReadableError, InactiveWritableError
from bonobo.structs.inputs import Input from bonobo.structs.inputs import Input

View File

@ -10,7 +10,7 @@ def generate_integers():
def square(i): def square(i):
return i**2 return i ** 2
def results(f, context): def results(f, context):

View File

@ -147,14 +147,14 @@ class TestBagType(unittest.TestCase):
self.assertEqual(Zero()._asdict(), {}) self.assertEqual(Zero()._asdict(), {})
self.assertEqual(Zero()._fields, ()) self.assertEqual(Zero()._fields, ())
Dot = BagType('Dot', ('d', )) Dot = BagType('Dot', ('d',))
self.assertEqual(Dot(1), (1, )) self.assertEqual(Dot(1), (1,))
self.assertEqual(Dot._make([1]), (1, )) self.assertEqual(Dot._make([1]), (1,))
self.assertEqual(Dot(1).d, 1) self.assertEqual(Dot(1).d, 1)
self.assertEqual(repr(Dot(1)), 'Dot(d=1)') self.assertEqual(repr(Dot(1)), 'Dot(d=1)')
self.assertEqual(Dot(1)._asdict(), {'d': 1}) self.assertEqual(Dot(1)._asdict(), {'d': 1})
self.assertEqual(Dot(1)._replace(d=999), (999, )) self.assertEqual(Dot(1)._replace(d=999), (999,))
self.assertEqual(Dot(1)._fields, ('d', )) self.assertEqual(Dot(1)._fields, ('d',))
n = 5000 if sys.version_info >= (3, 7) else 254 n = 5000 if sys.version_info >= (3, 7) else 254
names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n)))
@ -178,7 +178,7 @@ class TestBagType(unittest.TestCase):
def test_pickle(self): def test_pickle(self):
p = TBag(x=10, y=20, z=30) p = TBag(x=10, y=20, z=30)
for module in (pickle, ): for module in (pickle,):
loads = getattr(module, 'loads') loads = getattr(module, 'loads')
dumps = getattr(module, 'dumps') dumps = getattr(module, 'dumps')
for protocol in range(-1, module.HIGHEST_PROTOCOL + 1): for protocol in range(-1, module.HIGHEST_PROTOCOL + 1):
@ -206,25 +206,191 @@ class TestBagType(unittest.TestCase):
# Broader test of all interesting names taken from the code, old # Broader test of all interesting names taken from the code, old
# template, and an example # template, and an example
words = { words = {
'Alias', 'At', 'AttributeError', 'Build', 'Bypass', 'Create', 'Encountered', 'Expected', 'Field', 'For', 'Alias',
'Got', 'Helper', 'IronPython', 'Jython', 'KeyError', 'Make', 'Modify', 'Note', 'OrderedDict', 'Point', 'At',
'Return', 'Returns', 'Type', 'TypeError', 'Used', 'Validate', 'ValueError', 'Variables', 'a', 'accessible', 'AttributeError',
'add', 'added', 'all', 'also', 'an', 'arg_list', 'args', 'arguments', 'automatically', 'be', 'build', 'Build',
'builtins', 'but', 'by', 'cannot', 'class_namespace', 'classmethod', 'cls', 'collections', 'convert', 'Bypass',
'copy', 'created', 'creation', 'd', 'debugging', 'defined', 'dict', 'dictionary', 'doc', 'docstring', 'Create',
'docstrings', 'duplicate', 'effect', 'either', 'enumerate', 'environments', 'error', 'example', 'exec', 'f', 'Encountered',
'f_globals', 'field', 'field_names', 'fields', 'formatted', 'frame', 'function', 'functions', 'generate', 'Expected',
'getter', 'got', 'greater', 'has', 'help', 'identifiers', 'indexable', 'instance', 'instantiate', 'Field',
'interning', 'introspection', 'isidentifier', 'isinstance', 'itemgetter', 'iterable', 'join', 'keyword', 'For',
'keywords', 'kwds', 'len', 'like', 'list', 'map', 'maps', 'message', 'metadata', 'method', 'methods', 'Got',
'module', 'module_name', 'must', 'name', 'named', 'namedtuple', 'namedtuple_', 'names', 'namespace', 'Helper',
'needs', 'new', 'nicely', 'num_fields', 'number', 'object', 'of', 'operator', 'option', 'p', 'particular', 'IronPython',
'pickle', 'pickling', 'plain', 'pop', 'positional', 'property', 'r', 'regular', 'rename', 'replace', 'Jython',
'replacing', 'repr', 'repr_fmt', 'representation', 'result', 'reuse_itemgetter', 's', 'seen', 'sequence', 'KeyError',
'set', 'side', 'specified', 'split', 'start', 'startswith', 'step', 'str', 'string', 'strings', 'subclass', 'Make',
'sys', 'targets', 'than', 'the', 'their', 'this', 'to', 'tuple_new', 'type', 'typename', 'underscore', 'Modify',
'unexpected', 'unpack', 'up', 'use', 'used', 'user', 'valid', 'values', 'variable', 'verbose', 'where', 'Note',
'which', 'work', 'x', 'y', 'z', 'zip' 'OrderedDict',
'Point',
'Return',
'Returns',
'Type',
'TypeError',
'Used',
'Validate',
'ValueError',
'Variables',
'a',
'accessible',
'add',
'added',
'all',
'also',
'an',
'arg_list',
'args',
'arguments',
'automatically',
'be',
'build',
'builtins',
'but',
'by',
'cannot',
'class_namespace',
'classmethod',
'cls',
'collections',
'convert',
'copy',
'created',
'creation',
'd',
'debugging',
'defined',
'dict',
'dictionary',
'doc',
'docstring',
'docstrings',
'duplicate',
'effect',
'either',
'enumerate',
'environments',
'error',
'example',
'exec',
'f',
'f_globals',
'field',
'field_names',
'fields',
'formatted',
'frame',
'function',
'functions',
'generate',
'getter',
'got',
'greater',
'has',
'help',
'identifiers',
'indexable',
'instance',
'instantiate',
'interning',
'introspection',
'isidentifier',
'isinstance',
'itemgetter',
'iterable',
'join',
'keyword',
'keywords',
'kwds',
'len',
'like',
'list',
'map',
'maps',
'message',
'metadata',
'method',
'methods',
'module',
'module_name',
'must',
'name',
'named',
'namedtuple',
'namedtuple_',
'names',
'namespace',
'needs',
'new',
'nicely',
'num_fields',
'number',
'object',
'of',
'operator',
'option',
'p',
'particular',
'pickle',
'pickling',
'plain',
'pop',
'positional',
'property',
'r',
'regular',
'rename',
'replace',
'replacing',
'repr',
'repr_fmt',
'representation',
'result',
'reuse_itemgetter',
's',
'seen',
'sequence',
'set',
'side',
'specified',
'split',
'start',
'startswith',
'step',
'str',
'string',
'strings',
'subclass',
'sys',
'targets',
'than',
'the',
'their',
'this',
'to',
'tuple_new',
'type',
'typename',
'underscore',
'unexpected',
'unpack',
'up',
'use',
'used',
'user',
'valid',
'values',
'variable',
'verbose',
'where',
'which',
'work',
'x',
'y',
'z',
'zip',
} }
sorted_words = tuple(sorted(words)) sorted_words = tuple(sorted(words))
T = BagType('T', sorted_words) T = BagType('T', sorted_words)
@ -252,7 +418,7 @@ class TestBagType(unittest.TestCase):
self.assertEqual(t.__getnewargs__(), values) self.assertEqual(t.__getnewargs__(), values)
def test_repr(self): def test_repr(self):
A = BagType('A', ('x', )) A = BagType('A', ('x',))
self.assertEqual(repr(A(1)), 'A(x=1)') self.assertEqual(repr(A(1)), 'A(x=1)')
# repr should show the name of the subclass # repr should show the name of the subclass
@ -273,6 +439,18 @@ class TestBagType(unittest.TestCase):
def test_annoying_attribute_names(self): def test_annoying_attribute_names(self):
self._create( self._create(
'__slots__', '__getattr__', '_attrs', '_fields', '__new__', '__getnewargs__', '__repr__', '_make', 'get', '__slots__',
'_replace', '_asdict', '_cls', 'self', 'tuple' '__getattr__',
'_attrs',
'_fields',
'__new__',
'__getnewargs__',
'__repr__',
'_make',
'get',
'_replace',
'_asdict',
'_cls',
'self',
'tuple',
) )

View File

@ -1,7 +1,7 @@
import pytest import pytest
from bonobo.util import sortedlist, ensure_tuple from bonobo.util import ensure_tuple, sortedlist
from bonobo.util.collections import tuplize, cast from bonobo.util.collections import cast, tuplize
def test_sortedlist(): def test_sortedlist():
@ -14,8 +14,8 @@ def test_sortedlist():
def test_ensure_tuple(): def test_ensure_tuple():
assert ensure_tuple('a') == ('a', ) assert ensure_tuple('a') == ('a',)
assert ensure_tuple(('a', )) == ('a', ) assert ensure_tuple(('a',)) == ('a',)
assert ensure_tuple(()) is () assert ensure_tuple(()) is ()

View File

@ -2,7 +2,7 @@ import operator
import pytest import pytest
from bonobo.util.objects import Wrapper, get_name, ValueHolder, get_attribute_or_create from bonobo.util.objects import ValueHolder, Wrapper, get_attribute_or_create, get_name
from bonobo.util.testing import optional_contextmanager from bonobo.util.testing import optional_contextmanager
@ -65,10 +65,7 @@ def test_valueholder_notequal():
assert not (x != 42) assert not (x != 42)
@pytest.mark.parametrize('rlo,rhi', [ @pytest.mark.parametrize('rlo,rhi', [(1, 2), ('a', 'b')])
(1, 2),
('a', 'b'),
])
def test_valueholder_ordering(rlo, rhi): def test_valueholder_ordering(rlo, rhi):
vlo, vhi = ValueHolder(rlo), ValueHolder(rhi) vlo, vhi = ValueHolder(rlo), ValueHolder(rhi)
@ -129,15 +126,27 @@ def test_get_attribute_or_create():
unsupported_operations = { unsupported_operations = {
int: {operator.matmul}, int: {operator.matmul},
str: { str: {
operator.sub, operator.mul, operator.matmul, operator.floordiv, operator.truediv, operator.mod, divmod, operator.sub,
operator.pow, operator.lshift, operator.rshift, operator.and_, operator.xor, operator.or_ operator.mul,
operator.matmul,
operator.floordiv,
operator.truediv,
operator.mod,
divmod,
operator.pow,
operator.lshift,
operator.rshift,
operator.and_,
operator.xor,
operator.or_,
}, },
} }
@pytest.mark.parametrize('x,y', [(5, 3), (0, 10), (0, 0), (1, 1), ('foo', 'bar'), ('', 'baz!')]) @pytest.mark.parametrize('x,y', [(5, 3), (0, 10), (0, 0), (1, 1), ('foo', 'bar'), ('', 'baz!')])
@pytest.mark.parametrize( @pytest.mark.parametrize(
'operation,inplace_operation', [ 'operation,inplace_operation',
[
(operator.add, operator.iadd), (operator.add, operator.iadd),
(operator.sub, operator.isub), (operator.sub, operator.isub),
(operator.mul, operator.imul), (operator.mul, operator.imul),
@ -152,7 +161,7 @@ unsupported_operations = {
(operator.and_, operator.iand), (operator.and_, operator.iand),
(operator.xor, operator.ixor), (operator.xor, operator.ixor),
(operator.or_, operator.ior), (operator.or_, operator.ior),
] ],
) )
def test_valueholder_integer_operations(x, y, operation, inplace_operation): def test_valueholder_integer_operations(x, y, operation, inplace_operation):
v = ValueHolder(x) v = ValueHolder(x)

View File

@ -15,4 +15,4 @@ def test_resolve_options():
def test_resolve_transformations(): def test_resolve_transformations():
assert _resolve_transformations(('PrettyPrinter', )) == (bonobo.PrettyPrinter, ) assert _resolve_transformations(('PrettyPrinter',)) == (bonobo.PrettyPrinter,)

View File

@ -3,10 +3,7 @@ from bonobo.util.statistics import WithStatistics
class MyThingWithStats(WithStatistics): class MyThingWithStats(WithStatistics):
def get_statistics(self, *args, **kwargs): def get_statistics(self, *args, **kwargs):
return ( return (('foo', 42), ('bar', 69))
('foo', 42),
('bar', 69),
)
def test_with_statistics(): def test_with_statistics():