paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:hartym/bonobo into develop

Browse Source
This commit is contained in:
Romain Dorgueil
2018-08-11 16:13:26 +02:00
parent 7a82604fbf a140e6506e
commit 52887a297f
44 changed files with 258 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.codacy.yml
View File

@ -1,4 +1,11 @@
---
exclude_paths:
- bonobo/examples/
- bonobo/ext/
- benchmarks/**
- bin/**
- bonobo/contrib/jupyter/**.js
- bonobo/examples/**
- bonobo/ext/**
- bonobo/util/testing.py
- docs/**
- setup.py
- tests/**

17
.editorconfig Normal file
View File

@ -0,0 +1,17 @@
root = true
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
[*.py]
indent = ' '
indent_size = 4
indent_style = space
line_length = 120
multi_line_output = 5
[Makefile]
indent_style = tab

24
Makefile
View File

@ -1,4 +1,4 @@
# Generated by Medikit 0.6.3 on 2018-07-29.
# Generated by Medikit 0.6.3 on 2018-08-11.
# All changes will be overriden.
# Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
@ -26,8 +26,6 @@ SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build
SPHINX_OPTIONS ?=
SPHINX_SOURCEDIR ?= docs
SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
YAPF ?= $(PYTHON) -m yapf
YAPF_OPTIONS ?= -rip
SPHINX_AUTOBUILD ?= $(PYTHON_DIRNAME)/sphinx-autobuild
MEDIKIT ?= $(PYTHON) -m medikit
MEDIKIT_UPDATE_OPTIONS ?=
@ -44,7 +42,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else
@printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_INLINE) -r $(PYTHON_REQUIREMENTS_FILE)
@mkdir -p .medikit; touch $@
endif
@ -62,7 +60,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else
@printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DEV_INLINE) -r $(PYTHON_REQUIREMENTS_DEV_FILE)
@mkdir -p .medikit; touch $@
endif
@ -79,7 +77,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else
@printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DOCKER_INLINE) -r $(PYTHON_REQUIREMENTS_DOCKER_FILE)
@mkdir -p .medikit; touch $@
endif
@ -93,7 +91,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else
@printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_JUPYTER_INLINE) -r $(PYTHON_REQUIREMENTS_JUPYTER_FILE)
@mkdir -p .medikit; touch $@
endif
@ -107,7 +105,7 @@ else ifneq ($(QUICK),)
@printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target)
else
@printf "Applying \033[36m%s\033[0m target...\n" $(target)
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel
$(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_SQLALCHEMY_INLINE) -r $(PYTHON_REQUIREMENTS_SQLALCHEMY_FILE)
@mkdir -p .medikit; touch $@
endif
@ -118,15 +116,15 @@ test: install-dev ## Runs the test suite.
$(SPHINX_SOURCEDIR): install-dev ##
$(SPHINX_BUILD) -b html -D latex_paper_size=a4 $(SPHINX_OPTIONS) $(SPHINX_SOURCEDIR) $(SPHINX_BUILDDIR)/html
format: install-dev ## Reformats the whole python codebase using yapf.
$(YAPF) $(YAPF_OPTIONS) .
$(YAPF) $(YAPF_OPTIONS) Projectfile
watch-$(SPHINX_SOURCEDIR): ##
$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)
format: ## Reformats the whole codebase using our standards (requires black and isort).
black -l 120 --skip-string-normalization .
isort -rc -o mondrian -o whistle -y .
medikit: # Checks installed medikit version and updates it if it is outdated.
@$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=10.0" "medikit>=$(MEDIKIT_VERSION)"
@$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=18.0" "medikit>=$(MEDIKIT_VERSION)"
update: medikit ## Update project artifacts using medikit.
$(MEDIKIT) update $(MEDIKIT_UPDATE_OPTIONS)

28
Projectfile
View File

@ -6,7 +6,6 @@ make = require('make')
pytest = require('pytest')
python = require('python')
sphinx = require('sphinx')
yapf = require('yapf')
python.setup(
name='bonobo',
@ -47,7 +46,7 @@ python.add_requirements(
'fs ~=2.0',
'graphviz >=0.8,<0.9',
'jinja2 ~=2.9',
'mondrian ~=0.7',
'mondrian ~=0.8',
'packaging ~=17.0',
'psutil ~=5.4',
'python-slugify ~=1.2.0',
@ -74,14 +73,29 @@ python.add_requirements(
@listen(make.on_generate)
def on_make_generate(event):
event.makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild'
event.makefile.add_target(
makefile = event.makefile
# Sphinx
makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild'
makefile.add_target(
'watch-$(SPHINX_SOURCEDIR)',
'''
$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)
''',
'$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)',
phony=True
)
# Formating
makefile.add_target(
'format',
'''
black -l 120 --skip-string-normalization .
isort -rc -o mondrian -o whistle -y .
''',
phony=True,
doc='Reformats the whole codebase using our standards (requires black and isort).'
)
# vim: ft=python:

5
benchmarks/parameters.py
View File

@ -47,10 +47,9 @@ if __name__ == '__main__':
for i in 1, 2, 3:
print(
'j{}'.format(i),
timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i))
'j{}'.format(i), timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i))
)
print(
'k{}'.format(i),
timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i))
timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i)),
)

22
bin/update_apidoc.py
View File

@ -1,6 +1,6 @@
import os
from jinja2 import Environment, DictLoader
from jinja2 import DictLoader, Environment
__path__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__), '..'))
@ -18,11 +18,7 @@ class Module:
return '<{} ({})>'.format(self.title, self.name)
def asdict(self):
return {
'name': self.name,
'title': self.title,
'automodule_options': self.automodule_options,
}
return {'name': self.name, 'title': self.title, 'automodule_options': self.automodule_options}
def get_path(self):
return os.path.join(__path__, apidoc_root, *self.name.split('.')) + '.rst'
@ -45,9 +41,9 @@ def underlined_filter(txt, chr):
env = Environment(
loader=DictLoader({
'module':
'''
loader=DictLoader(
{
'module': '''
{{ (':mod:`'~title~' <'~name~'>`') | underlined('=') }}
.. currentmodule:: {{ name }}
@ -56,8 +52,12 @@ env = Environment(
.. automodule:: {{ name }}
{% for opt in automodule_options %} :{{ opt }}:{{ "\n" }}{% endfor %}
''' [1:-1] + '\n'
})
'''[
1:-1
]
+ '\n'
}
)
)
env.filters['underlined'] = underlined_filter

8
bonobo/__init__.py
View File

@ -7,7 +7,8 @@
import sys
assert sys.version_info >= (3, 5), "Python 3.5+ is required to use Bonobo."
if sys.version_info < (3, 5):
raise RuntimeError('Python 3.5+ is required to use Bonobo.')
from bonobo._api import (
run,
@ -58,15 +59,14 @@ __version__ = __version__
def _repr_html_():
"""This allows to easily display a version snippet in Jupyter."""
from bonobo.util.pkgs import bonobo_packages
from bonobo.commands.version import get_versions
return (
'<div style="padding: 8px;">'
' <div style="float: left; width: 20px; height: 20px;">{}</div>'
' <pre style="white-space: nowrap; padding-left: 8px">{}</pre>'
"</div>"
).format(__logo__, "<br/>".join(get_versions(all=True)))
'</div>'
).format(__logo__, '<br/>'.join(get_versions(all=True)))
del sys

2
bonobo/_api.py
View File

@ -187,7 +187,7 @@ def get_examples_path(*pathsegments):
import os
import pathlib
return str(pathlib.Path(os.path.dirname(__file__), "examples", *pathsegments))
return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments))
@api.register

2
bonobo/_version.py
View File

@ -1 +1 @@
__version__ = "0.6.2"
__version__ = '0.6.3'

4
bonobo/commands/__init__.py
View File

@ -26,7 +26,9 @@ def entrypoint(args=None):
commands = {}
def register_extension(ext, commands=commands):
def register_extension(ext):
nonlocal commands
try:
parser = subparsers.add_parser(ext.name)
if isinstance(ext.plugin, type) and issubclass(ext.plugin, BaseCommand):

4
bonobo/commands/base.py
View File

@ -7,6 +7,7 @@ from contextlib import contextmanager
import bonobo.util.environ
from bonobo.util import get_name
from bonobo.util.environ import get_argument_parser, parse_args
from mondrian import humanizer
class BaseCommand:
@ -60,7 +61,8 @@ class BaseGraphCommand(BaseCommand):
return options
def handle(self, file, mod, **options):
options = self.parse_options(**options)
with humanizer.humanize():
options = self.parse_options(**options)
with self.read(file, mod, **options) as (graph, graph_execution_options, options):
return self.do_handle(graph, **graph_execution_options, **options)

5
bonobo/commands/convert.py
View File

@ -2,6 +2,7 @@ import bonobo
from bonobo.commands import BaseCommand
from bonobo.registry import READER, WRITER, default_registry
from bonobo.util.resolvers import _resolve_options, _resolve_transformations
from mondrian import humanizer
class ConvertCommand(BaseCommand):
@ -48,6 +49,7 @@ class ConvertCommand(BaseCommand):
help="Add a named option to the writer factory.",
)
@humanizer.humanize()
def handle(
self,
input_filename,
@ -60,6 +62,8 @@ class ConvertCommand(BaseCommand):
limit=None,
transformation=None,
):
graph = bonobo.Graph()
reader_factory = default_registry.get_reader_factory_for(input_filename, format=reader)
reader_kwargs = _resolve_options((option or []) + (reader_option or []))
@ -78,7 +82,6 @@ class ConvertCommand(BaseCommand):
transformations += _resolve_transformations(transformation)
graph = bonobo.Graph()
graph.add_chain(
reader_factory(input_filename, **reader_kwargs),
*transformations,

40
bonobo/commands/init.py
View File

@ -3,6 +3,7 @@ import os
from jinja2 import Environment, FileSystemLoader
from bonobo.commands import BaseCommand
from mondrian import humanizer
class InitCommand(BaseCommand):
@ -30,12 +31,16 @@ class InitCommand(BaseCommand):
with open(filename, "w+") as f:
f.write(template.render(name=name))
self.logger.info("Generated {} using template {!r}.".format(filename, template_name))
print(
humanizer.Success(
"Generated {} using template {!r}.".format(filename, template_name)
)
)
def create_package(self, *, filename):
name, ext = os.path.splitext(filename)
if ext != "":
raise ValueError("Package names should not have an extension.")
_, ext = os.path.splitext(filename)
if ext != '':
raise ValueError('Package names should not have an extension.')
try:
import medikit.commands
@ -52,18 +57,23 @@ class InitCommand(BaseCommand):
self.logger.info('Generated "{}" package with medikit.'.format(package_name))
self.create_file_from_template(template="default", filename=os.path.join(filename, package_name, "__main__.py"))
print('Your "{}" package has been created.'.format(package_name))
print()
print("Install it...")
print()
print(" pip install --editable {}".format(filename))
print()
print("Then maybe run the example...")
print()
print(" python -m {}".format(package_name))
print()
print("Enjoy!")
print(
humanizer.Success(
'Package "{}" has been created.'.format(package_name),
'',
"Install it...",
'',
" $ `pip install --editable {}`".format(filename),
'',
"Then maybe run the example...",
'',
" $ `python -m {}`".format(package_name),
'',
"Enjoy!"
)
)
@humanizer.humanize()
def handle(self, *, template, filename, package=False, force=False):
if os.path.exists(filename) and not force:
raise FileExistsError("Target filename already exists, use --force to override.")

2
bonobo/commands/version.py
View File

@ -1,4 +1,5 @@
from bonobo.commands import BaseCommand
from mondrian import humanizer
def get_versions(*, all=False, quiet=None):
@ -21,6 +22,7 @@ def get_versions(*, all=False, quiet=None):
class VersionCommand(BaseCommand):
@humanizer.humanize()
def handle(self, *, all=False, quiet=False):
for line in get_versions(all=all, quiet=quiet):
print(line)

2
bonobo/config/processors.py
View File

@ -133,7 +133,7 @@ class ContextCurrifier:
try:
# todo yield from ? how to ?
processor.send(self._stack_values.pop())
except StopIteration as exc:
except StopIteration:
# This is normal, and wanted.
pass
else:

23
bonobo/config/services.py
View File

@ -21,32 +21,32 @@ class Service(Option):
identifier. For example, you can create a Configurable that has a "database" Service in its attribute, meaning that
you'll define which database to use, by name, when creating the instance of this class, then provide an
implementation when running the graph using a strategy.
Example::
import bonobo
class QueryExtractor(bonobo.Configurable):
database = bonobo.Service(default='sqlalchemy.engine.default')
graph = bonobo.Graph(
QueryExtractor(database='sqlalchemy.engine.secondary'),
*more_transformations,
)
if __name__ == '__main__':
engine = create_engine('... dsn ...')
bonobo.run(graph, services={
'sqlalchemy.engine.secondary': engine
})
The main goal is not to tie transformations to actual dependencies, so the same can be run in different contexts
(stages like preprod, prod, or tenants like client1, client2, or anything you want).
.. attribute:: name
Service name will be used to retrieve the implementation at runtime.
"""
def __init__(self, name, __doc__=None):
@ -66,8 +66,13 @@ class Service(Option):
class Container(dict):
def __new__(cls, *args, **kwargs):
if len(args) == 1:
assert not len(kwargs), "only one usage at a time, my dear."
if not (args[0]):
if len(kwargs):
raise ValueError(
'You can either use {} with one positional argument or with keyword arguments, not both.'.format(
cls.__name__
)
)
if not args[0]:
return super().__new__(cls)
if isinstance(args[0], cls):
return cls

7
bonobo/contrib/django/commands.py
View File

@ -4,11 +4,11 @@ from types import GeneratorType
from colorama import Back, Fore, Style
from django.core.management import BaseCommand
from django.core.management.base import OutputWrapper
from mondrian import term
import bonobo
from bonobo.plugins.console import ConsoleOutputPlugin
from bonobo.util.term import CLEAR_EOL
from mondrian import term
from .utils import create_or_update
@ -59,8 +59,9 @@ class ETLCommand(BaseCommand):
graph_coll = (graph_coll,)
for i, graph in enumerate(graph_coll):
assert isinstance(graph, bonobo.Graph), "Invalid graph provided."
print(term.lightwhite("{}. {}".format(i + 1, graph.name)))
if not isinstance(graph, bonobo.Graph):
raise ValueError('Expected a Graph instance, got {!r}.'.format(graph))
print(term.lightwhite('{}. {}'.format(i + 1, graph.name)))
result = bonobo.run(graph, services=services, strategy=strategy)
results.append(result)
print(term.lightblack(" ... return value: " + str(result)))

5
bonobo/examples/datasets/coffeeshops.py
View File

@ -8,6 +8,11 @@ from bonobo.structs.graphs import PartialGraph
def get_graph(graph=None, *, _limit=(), _print=()):
"""
Extracts a list of cafes with on euro in Paris, renames the name, address and zipcode fields,
reorders the fields and formats to json and csv files.
"""
graph = graph or bonobo.Graph()
producer = (

2
bonobo/examples/datasets/fablabs.py
View File

@ -1,5 +1,5 @@
"""
Extracts a list of fablabs in the world, restrict to the ones in france, then format it both for a nice console output
Extracts a list of fablabs in the world, restricted to the ones in france, then format its both for a nice console output
and a flat txt file.
.. graphviz::

9
bonobo/execution/contexts/base.py
View File

@ -2,16 +2,17 @@ import logging
import sys
from contextlib import contextmanager
from mondrian import term
from bonobo.util import deprecated
from bonobo.util.objects import Wrapper, get_name
from mondrian import term
@contextmanager
def recoverable(error_handler):
try:
yield
except Exception as exc: # pylint: disable=broad-except
except Exception: # pylint: disable=broad-except
error_handler(*sys.exc_info(), level=logging.ERROR)
@ -19,9 +20,9 @@ def recoverable(error_handler):
def unrecoverable(error_handler):
try:
yield
except Exception as exc: # pylint: disable=broad-except
except Exception: # pylint: disable=broad-except
error_handler(*sys.exc_info(), level=logging.ERROR)
raise # raise unrecoverableerror from exc ?
raise # raise unrecoverableerror from x ?
class Lifecycle:

4
bonobo/execution/contexts/node.py
View File

@ -190,7 +190,7 @@ class NodeExecutionContext(BaseContext, WithStatistics):
if self._stack:
try:
self._stack.teardown()
except Exception as exc:
except Exception:
self.fatal(sys.exc_info())
super().stop()
@ -207,7 +207,7 @@ class NodeExecutionContext(BaseContext, WithStatistics):
if self._input_type is not None:
raise RuntimeError("Cannot override input type, already have %r.", self._input_type)
if type(input_type) is not type:
if not isinstance(input_type, type):
raise UnrecoverableTypeError("Input types must be regular python types.")
if not issubclass(input_type, tuple):

2
bonobo/execution/strategies/__init__.py
View File

@ -28,7 +28,7 @@ DEFAULT_STRATEGY = "threadpool"
def create_strategy(name=None):
"""
Create a strategy, or just returns it if it's already one.
:param name:
:return: Strategy
"""

5
bonobo/nodes/basics.py
View File

@ -3,6 +3,8 @@ import html
import itertools
import pprint
from mondrian import term
from bonobo import settings
from bonobo.config import Configurable, Method, Option, use_context, use_no_input, use_raw_input
from bonobo.config.functools import transformation_factory
@ -10,7 +12,6 @@ from bonobo.config.processors import ContextProcessor, use_context_processor
from bonobo.constants import NOT_MODIFIED
from bonobo.util.objects import ValueHolder
from bonobo.util.term import CLEAR_EOL
from mondrian import term
__all__ = [
"FixedWindow",
@ -57,8 +58,6 @@ class Limit(Configurable):
def Tee(f):
from bonobo.constants import NOT_MODIFIED
@functools.wraps(f)
def wrapped(*args, **kwargs):
nonlocal f

4
bonobo/nodes/filter.py
View File

@ -11,9 +11,9 @@ class Filter(Configurable):
.. attribute:: filter
A callable used to filter lines.
If the callable returns a true-ish value, the input will be passed unmodified to the next items.
Otherwise, it'll be burnt.
"""

4
bonobo/plugins/__init__.py
View File

@ -2,13 +2,13 @@ class Plugin:
"""
A plugin is an extension to the core behavior of bonobo. If you're writing transformations, you should not need
to use this interface.
For examples, you can read bonobo.plugins.console.ConsoleOutputPlugin, or bonobo.plugins.jupyter.JupyterOutputPlugin
that respectively permits an interactive output on an ANSI console and a rich output in a jupyter notebook. Note
that you most probably won't instanciate them by yourself at runtime, as it's the default behaviour of bonobo to use
them if your in a compatible context (aka an interactive terminal for the console plugin, or a jupyter notebook for
the notebook plugin.)
Warning: THE PLUGIN API IS PRE-ALPHA AND WILL EVOLVE BEFORE 1.0, DO NOT RELY ON IT BEING STABLE!
"""

4
bonobo/settings.py
View File

@ -7,7 +7,7 @@ from bonobo.errors import ValidationError
def to_bool(s):
if s is None:
return False
if type(s) is bool:
if isinstance(s, bool):
return s
if len(s):
if s.lower() in ("f", "false", "n", "no", "0"):
@ -31,7 +31,7 @@ class Setting:
def __init__(self, name, default=None, validator=None, formatter=None):
self.name = name
if default:
if default is not None:
self.default = default if callable(default) else lambda: default
else:
self.default = lambda: None

1
bonobo/structs/graphs.py
View File

@ -36,7 +36,6 @@ class GraphCursor:
"Expected something looking like a node, but got an Ellipsis (...). Did you forget to complete the graph?"
)
if len(nodes):
chain = self.graph.add_chain(*nodes, _input=self.last)
return GraphCursor(chain.graph, first=self.first, last=chain.output)

4
bonobo/util/bags.py
View File

@ -125,7 +125,7 @@ def BagType(typename, fields, *, verbose=False, module=None):
# message or automatically replace the field name with a valid name.
attrs = tuple(map(_uniquify(_make_valid_attr_name), fields))
if type(fields) is str:
if isinstance(fields, str):
raise TypeError("BagType does not support providing fields as a string.")
fields = list(map(str, fields))
typename = str(typename)
@ -133,6 +133,8 @@ def BagType(typename, fields, *, verbose=False, module=None):
for i, name in enumerate([typename] + fields):
if type(name) is not str:
raise TypeError("Type names and field names must be strings, got {name!r}".format(name=name))
if not isinstance(name, str):
raise TypeError('Type names and field names must be strings, got {name!r}'.format(name=name))
if not i:
if not name.isidentifier():
raise ValueError("Type names must be valid identifiers: {name!r}".format(name=name))

2
bonobo/util/objects.py
View File

@ -24,7 +24,7 @@ class ValueHolder:
For the sake of concistency, all operator methods have been implemented (see https://docs.python.org/3/reference/datamodel.html) or
at least all in a certain category, but it feels like a more correct method should exist, like with a getattr-something on the
value. Let's see later.
"""
def __init__(self, value):

2
bonobo/util/term.py
View File

@ -1,2 +1,2 @@
CLEAR_EOL = "\033[0K"
MOVE_CURSOR_UP = lambda n: "\033[{}A".format(n)
MOVE_CURSOR_UP = '\033[{}A'.format

8
docs/_templates/alabaster/support.py vendored
View File

@ -1,8 +1,9 @@
# flake8: noqa
from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, \
Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
from pygments.token import (
Comment, Error, Generic, Keyword, Literal, Name, Number, Operator, Other, Punctuation, String, Whitespace
)
# Originally based on FlaskyStyle which was based on 'tango'.
@ -12,7 +13,7 @@ class Alabaster(Style):
styles = {
# No corresponding class for the following:
#Text: "", # class: ''
# Text: "", # class: ''
Whitespace: "underline #f8f8f8", # class: 'w'
Error: "#a40000 border:#ef2929", # class: 'err'
Other: "#000000", # class 'x'
@ -28,7 +29,6 @@ class Alabaster(Style):
Operator: "#582800", # class: 'o'
Operator.Word: "bold #004461", # class: 'ow' - like keywords
Punctuation: "bold #000000", # class: 'p'
# because special names such as Name.Class, Name.Function, etc.
# are not recognized as such later in the parsing, we choose them
# to look the same as ordinary variables.

33
docs/conf.py
View File

@ -5,10 +5,11 @@ import datetime
import os
import sys
import bonobo
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('_themes'))
import bonobo
# -- General configuration ------------------------------------------------
@ -63,11 +64,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
autoclass_content = 'both'
autodoc_member_order = 'groupwise'
autodoc_default_flags = [
'members',
'undoc-members',
'show-inheritance',
]
autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance']
add_module_names = False
pygments_style = 'sphinx'
@ -112,7 +109,7 @@ html_sidebars = {
'sourcelink.html',
'searchbox.html',
'sidebarinfos.html',
]
],
}
html_theme_path = ['_themes']
@ -137,15 +134,12 @@ latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
@ -154,9 +148,7 @@ latex_elements = {
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual'),
]
latex_documents = [(master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual')]
# -- Options for manual page output ---------------------------------------
@ -171,9 +163,14 @@ man_pages = [(master_doc, 'bonobo', 'Bonobo Documentation', [author], 1)]
# dir menu entry, description, category)
texinfo_documents = [
(
master_doc, 'Bonobo', 'Bonobo Documentation', author, 'Bonobo', 'One line description of project.',
'Miscellaneous'
),
master_doc,
'Bonobo',
'Bonobo Documentation',
author,
'Bonobo',
'One line description of project.',
'Miscellaneous',
)
]
# -- Options for Epub output ----------------------------------------------
@ -209,4 +206,6 @@ rst_epilog = """
.. |longversion| replace:: v.{version}
""".format(version=version, )
""".format(
version=version
)

14
docs/reference/examples.rst
View File

@ -5,7 +5,19 @@ There are a few examples bundled with **bonobo**.
You'll find them under the :mod:`bonobo.examples` package, and you can run them directly as modules:
$ bonobo run -m bonobo.examples...module
.. code-block:: shell-session
$ bonobo run -m bonobo.examples.module
or
.. code-block:: shell-session
$ python -m bonobo.examples.module
.. toctree::
:maxdepth: 4

50
docs/reference/examples/tutorials.rst
View File

@ -1,50 +0,0 @@
Examples from the tutorial
==========================
Examples from :doc:`/tutorial/tut01`
::::::::::::::::::::::::::::::::::::
Example 1
---------
.. automodule:: bonobo.examples.tutorials.tut01e01
:members:
:undoc-members:
:show-inheritance:
Example 2
---------
.. automodule:: bonobo.examples.tutorials.tut01e02
:members:
:undoc-members:
:show-inheritance:
Examples from :doc:`/tutorial/tut02`
::::::::::::::::::::::::::::::::::::
Example 1: Read
---------------
.. automodule:: bonobo.examples.tutorials.tut02e01_read
:members:
:undoc-members:
:show-inheritance:
Example 2: Write
----------------
.. automodule:: bonobo.examples.tutorials.tut02e02_write
:members:
:undoc-members:
:show-inheritance:
Example 3: Write as map
-----------------------
.. automodule:: bonobo.examples.tutorials.tut02e03_writeasmap
:members:
:undoc-members:
:show-inheritance:

6
requirements-dev.txt
View File

@ -18,8 +18,9 @@ imagesize==1.0.0
jinja2-time==0.2.0
jinja2==2.10
markupsafe==1.0
more-itertools==4.2.0
more-itertools==4.3.0
packaging==17.1
pathlib2==2.3.2
pluggy==0.7.1
poyo==0.4.1
py==1.5.4
@ -27,7 +28,7 @@ pygments==2.2.0
pyparsing==2.2.0
pytest-cov==2.5.1
pytest-timeout==1.3.1
pytest==3.6.4
pytest==3.7.1
python-dateutil==2.7.3
pytz==2018.5
requests==2.19.1
@ -38,4 +39,3 @@ sphinx==1.7.6
sphinxcontrib-websupport==1.1.0
urllib3==1.23
whichcraft==0.4.1
yapf==0.22.0

4
requirements-docker.txt
View File

@ -7,12 +7,12 @@ chardet==3.0.4
colorama==0.3.9
docker-pycreds==0.3.0
docker==2.7.0
fs==2.0.26
fs==2.0.27
graphviz==0.8.4
idna==2.7
jinja2==2.10
markupsafe==1.0
mondrian==0.7.0
mondrian==0.8.0
packaging==17.1
pbr==4.2.0
psutil==5.4.6

6
requirements-jupyter.txt
View File

@ -8,7 +8,7 @@ entrypoints==0.2.3
html5lib==1.0.1
ipykernel==4.8.2
ipython-genutils==0.2.0
ipython==6.4.0
ipython==6.5.0
ipywidgets==6.0.1
jedi==0.12.1
jinja2==2.10
@ -26,12 +26,12 @@ pandocfilters==1.4.2
parso==0.3.1
pexpect==4.6.0
pickleshare==0.7.4
prometheus-client==0.3.0
prometheus-client==0.3.1
prompt-toolkit==1.0.15
ptyprocess==0.6.0
pygments==2.2.0
python-dateutil==2.7.3
pyzmq==17.1.0
pyzmq==17.1.2
qtconsole==4.3.1
send2trash==1.5.0
simplegeneric==0.8.1

4
requirements-sqlalchemy.txt
View File

@ -5,12 +5,12 @@ bonobo-sqlalchemy==0.6.0
certifi==2018.4.16
chardet==3.0.4
colorama==0.3.9
fs==2.0.26
fs==2.0.27
graphviz==0.8.4
idna==2.7
jinja2==2.10
markupsafe==1.0
mondrian==0.7.0
mondrian==0.8.0
packaging==17.1
pbr==4.2.0
psutil==5.4.6

4
requirements.txt
View File

@ -4,12 +4,12 @@ cached-property==1.4.3
certifi==2018.4.16
chardet==3.0.4
colorama==0.3.9
fs==2.0.26
fs==2.0.27
graphviz==0.8.4
idna==2.7
jinja2==2.10
markupsafe==1.0
mondrian==0.7.0
mondrian==0.8.0
packaging==17.1
pbr==4.2.0
psutil==5.4.6

41
setup.py
View File

@ -1,4 +1,4 @@
# Generated by Medikit 0.6.3 on 2018-07-29.
# Generated by Medikit 0.6.3 on 2018-08-11.
# All changes will be overriden.
# Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
@ -44,14 +44,14 @@ else:
setup(
author='Romain Dorgueil',
author_email='romain@dorgueil.net',
data_files=[(
'share/jupyter/nbextensions/bonobo-jupyter', [
'bonobo/contrib/jupyter/static/extension.js', 'bonobo/contrib/jupyter/static/index.js',
'bonobo/contrib/jupyter/static/index.js.map'
]
)],
description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for '
'python 3.5+.'),
data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [
'bonobo/contrib/jupyter/static/extension.js',
'bonobo/contrib/jupyter/static/index.js',
'bonobo/contrib/jupyter/static/index.js.map'
])],
description=(
'Bonobo, a simple, modern and atomic extract-transform-load toolkit for '
'python 3.5+.'),
license='Apache License, Version 2.0',
name='bonobo',
python_requires='>=3.5',
@ -61,14 +61,17 @@ setup(
packages=find_packages(exclude=['ez_setup', 'example', 'test']),
include_package_data=True,
install_requires=[
'cached-property (~= 1.4)', 'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (~= 2.9)', 'mondrian (~= 0.7)',
'packaging (~= 17.0)', 'psutil (~= 5.4)', 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)',
'cached-property (~= 1.4)', 'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)',
'jinja2 (~= 2.9)', 'mondrian (~= 0.8)', 'packaging (~= 17.0)',
'psutil (~= 5.4)', 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)',
'stevedore (~= 1.27)', 'whistle (~= 1.0)'
],
extras_require={
'dev': [
'cookiecutter (>= 1.5, < 1.6)', 'coverage (~= 4.4)', 'pytest (~= 3.4)', 'pytest-cov (~= 2.5)',
'pytest-timeout (>= 1, < 2)', 'sphinx (~= 1.7)', 'sphinx-sitemap (>= 0.2, < 0.3)', 'yapf'
'cookiecutter (>= 1.5, < 1.6)', 'coverage (~= 4.4)',
'pytest (~= 3.4)', 'pytest-cov (~= 2.5)',
'pytest-timeout (>= 1, < 2)', 'sphinx (~= 1.7)',
'sphinx-sitemap (>= 0.2, < 0.3)'
],
'docker': ['bonobo-docker (~= 0.6.0a1)'],
'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'],
@ -76,13 +79,17 @@ setup(
},
entry_points={
'bonobo.commands': [
'convert = bonobo.commands.convert:ConvertCommand', 'download = bonobo.commands.download:DownloadCommand',
'examples = bonobo.commands.examples:ExamplesCommand', 'init = bonobo.commands.init:InitCommand',
'inspect = bonobo.commands.inspect:InspectCommand', 'run = bonobo.commands.run:RunCommand',
'convert = bonobo.commands.convert:ConvertCommand',
'download = bonobo.commands.download:DownloadCommand',
'examples = bonobo.commands.examples:ExamplesCommand',
'init = bonobo.commands.init:InitCommand',
'inspect = bonobo.commands.inspect:InspectCommand',
'run = bonobo.commands.run:RunCommand',
'version = bonobo.commands.version:VersionCommand'
],
'console_scripts': ['bonobo = bonobo.commands:entrypoint']
},
url='https://www.bonobo-project.org/',
download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version),
download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.
format(version=version),
)

16
tests/config/test_services.py
View File

@ -130,7 +130,21 @@ def test_requires():
assert svcargs["output"] == vcr.append
@pytest.mark.parametrize("services", [None, {}])
def test_constructor():
c1 = Container(foo='foo', bar='bar')
assert 2 == len(c1)
c2 = Container({'foo': 'foo', 'bar': 'bar'})
assert 2 == len(c2)
assert c1['foo'] == c2['foo']
assert c1['bar'] == c2['bar']
with pytest.raises(ValueError):
Container({'bar': 'bar'}, foo='foo')
@pytest.mark.parametrize('services', [None, {}])
def test_create_container_empty_values(services):
c = create_container(services)
assert len(c) == 2

6
tests/nodes/io/test_csv.py
View File

@ -63,9 +63,9 @@ class CsvReaderTest(Csv, ReaderTest, TestCase):
def test_output_type(self, context):
context.write_sync(EMPTY)
context.stop()
self.check_output(context, prepend=[("id", "name")])
self.check_output(context, prepend=[('id', 'name')])
@incontext(output_fields=("x", "y"), skip=1)
@incontext(output_fields=('x', 'y'), skip=1)
def test_output_fields(self, context):
context.write_sync(EMPTY)
context.stop()
@ -103,7 +103,7 @@ class CsvWriterTest(Csv, WriterTest, TestCase):
context.write_sync((L1, L2), (L3, L4))
context.stop()
assert self.readlines() == ("a,hey", "b,bee", "c,see", "d,dee")
assert self.readlines() == ('a,hey', 'b,bee', 'c,see', 'd,dee')
@incontext()
def test_nofields_multiple_args_length_mismatch(self, context):

1
tests/nodes/io/test_json.py
View File

@ -14,6 +14,7 @@ FOOBAZ = {"foo": "baz"}
incontext = ConfigurableNodeTest.incontext
###
# Standard JSON Readers / Writers
###

10
tests/test_settings.py
View File

@ -5,6 +5,7 @@ from unittest.mock import patch
import pytest
from bonobo import settings
from bonobo.errors import ValidationError
TEST_SETTING = "TEST_SETTING"
@ -38,6 +39,15 @@ def test_setting():
s.clear()
assert s.get() == "hello"
s = settings.Setting(TEST_SETTING, default=0, validator=lambda x: x == 42)
with pytest.raises(ValidationError):
assert s.get() is 0
s.set(42)
with pytest.raises(ValidationError):
s.set(21)
def test_default_settings():
settings.clear_all()