paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

style: switching from yapf to isort/black

Browse Source
This commit is contained in:
Romain Dorgueil
2018-08-11 06:34:37 +02:00
parent ebba06822b
commit d1c9beae97
93 changed files with 805 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bonobo/__init__.py
View File

@ -31,9 +31,7 @@ def _repr_html_():
' <div style="float: left; width: 20px; height: 20px;">{}</div>'
' <pre style="white-space: nowrap; padding-left: 8px">{}</pre>'
'</div>'
).format(
__logo__, '<br/>'.join(get_versions(all=True))
)
).format(__logo__, '<br/>'.join(get_versions(all=True)))
del sys

11
bonobo/_api.py
View File

@ -8,11 +8,11 @@ to another is maximal.
"""
from bonobo.execution.strategies import create_strategy
from bonobo.nodes import __all__ as _all_nodes
from bonobo.nodes import *
from bonobo.nodes import __all__ as _all_nodes
from bonobo.structs import Graph
from bonobo.util.api import ApiHelper
from bonobo.util.environ import parse_args, get_argument_parser
from bonobo.util.environ import get_argument_parser, parse_args
__all__ = []
@ -44,14 +44,17 @@ def run(graph, *, plugins=None, services=None, strategy=None):
plugins = plugins or []
from bonobo import settings
settings.check()
if not settings.QUIET.get(): # pragma: no cover
if _is_interactive_console():
import mondrian
mondrian.setup(excepthook=True)
from bonobo.plugins.console import ConsoleOutputPlugin
if ConsoleOutputPlugin not in plugins:
plugins.append(ConsoleOutputPlugin)
@ -60,6 +63,7 @@ def run(graph, *, plugins=None, services=None, strategy=None):
from bonobo.contrib.jupyter import JupyterOutputPlugin
except ImportError:
import logging
logging.warning(
'Failed to load jupyter widget. Easiest way is to install the optional "jupyter" '
'dependencies with «pip install bonobo[jupyter]», but you can also install a specific '
@ -70,6 +74,7 @@ def run(graph, *, plugins=None, services=None, strategy=None):
plugins.append(JupyterOutputPlugin)
import logging
logging.getLogger().setLevel(settings.LOGGING_LEVEL.get())
strategy = create_strategy(strategy)
return strategy.execute(graph, plugins=plugins, services=services)
@ -158,6 +163,7 @@ api.register_group(
def _is_interactive_console():
import sys
return sys.stdout.isatty()
@ -172,6 +178,7 @@ def _is_jupyter_notebook():
def get_examples_path(*pathsegments):
import os
import pathlib
return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments))

1
bonobo/commands/__init__.py
View File

@ -42,6 +42,7 @@ def entrypoint(args=None):
logger.exception('Error while loading command {}.'.format(ext.name))
from stevedore import ExtensionManager
mgr = ExtensionManager(namespace='bonobo.commands')
mgr.map(register_extension)

1
bonobo/commands/base.py
View File

@ -41,6 +41,7 @@ class BaseGraphCommand(BaseCommand):
Base class for CLI commands that depends on a graph definition, either from a file or from a module.
"""
required = True
handler = None

45
bonobo/commands/convert.py
View File

@ -1,7 +1,7 @@
import bonobo
from bonobo.commands import BaseCommand
from bonobo.registry import READER, WRITER, default_registry
from bonobo.util.resolvers import _resolve_transformations, _resolve_options
from bonobo.util.resolvers import _resolve_options, _resolve_transformations
class ConvertCommand(BaseCommand):
@ -11,21 +11,14 @@ class ConvertCommand(BaseCommand):
parser.add_argument(
'--' + READER,
'-r',
help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.'
help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.',
)
parser.add_argument(
'--' + WRITER,
'-w',
help=
'Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).'
)
parser.add_argument(
'--limit',
'-l',
type=int,
help='Adds a Limit() after the reader instance.',
default=None,
help='Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).',
)
parser.add_argument('--limit', '-l', type=int, help='Adds a Limit() after the reader instance.', default=None)
parser.add_argument(
'--transformation',
'-t',
@ -56,16 +49,16 @@ class ConvertCommand(BaseCommand):
)
def handle(
self,
input_filename,
output_filename,
reader=None,
reader_option=None,
writer=None,
writer_option=None,
option=None,
limit=None,
transformation=None,
self,
input_filename,
output_filename,
reader=None,
reader_option=None,
writer=None,
writer_option=None,
option=None,
limit=None,
transformation=None,
):
reader_factory = default_registry.get_reader_factory_for(input_filename, format=reader)
reader_kwargs = _resolve_options((option or []) + (reader_option or []))
@ -75,13 +68,13 @@ class ConvertCommand(BaseCommand):
writer_args = ()
else:
writer_factory = default_registry.get_writer_factory_for(output_filename, format=writer)
writer_args = (output_filename, )
writer_args = (output_filename,)
writer_kwargs = _resolve_options((option or []) + (writer_option or []))
transformations = ()
if limit:
transformations += (bonobo.Limit(limit), )
transformations += (bonobo.Limit(limit),)
transformations += _resolve_transformations(transformation)
@ -92,8 +85,4 @@ class ConvertCommand(BaseCommand):
writer_factory(*writer_args, **writer_kwargs),
)
return bonobo.run(
graph, services={
'fs': bonobo.open_fs(),
}
)
return bonobo.run(graph, services={'fs': bonobo.open_fs()})

2
bonobo/commands/run.py
View File

@ -19,6 +19,7 @@ class RunCommand(BaseGraphCommand):
def parse_options(self, *, quiet=False, verbose=False, install=False, **options):
from bonobo import settings
settings.QUIET.set_if_true(quiet)
settings.DEBUG.set_if_true(verbose)
self.install = install
@ -65,4 +66,5 @@ def _install_requirements(requirements):
# python interpreter.
pip.utils.pkg_resources = importlib.reload(pip.utils.pkg_resources)
import site
importlib.reload(site)

3
bonobo/commands/version.py
View File

@ -32,10 +32,11 @@ class VersionCommand(BaseCommand):
def _format_version(mod, *, name=None, quiet=False):
from bonobo.util.pkgs import bonobo_packages
args = {
'name': name or mod.__name__,
'version': mod.__version__,
'location': bonobo_packages[name or mod.__name__].location
'location': bonobo_packages[name or mod.__name__].location,
}
if not quiet:

23
bonobo/config/configurables.py
View File

@ -1,9 +1,7 @@
from bonobo.errors import AbstractError
from bonobo.util import isoption, iscontextprocessor, sortedlist, get_name
from bonobo.util import get_name, iscontextprocessor, isoption, sortedlist
__all__ = [
'Configurable',
]
__all__ = ['Configurable']
get_creation_counter = lambda v: v._creation_counter
@ -64,10 +62,7 @@ class ConfigurableMeta(type):
return cls.__processors_cache
def __repr__(self):
return ' '.join((
'<Configurable',
super(ConfigurableMeta, self).__repr__().split(' ', 1)[1],
))
return ' '.join(('<Configurable', super(ConfigurableMeta, self).__repr__().split(' ', 1)[1]))
try:
@ -156,8 +151,10 @@ class Configurable(metaclass=ConfigurableMeta):
if len(extraneous):
raise TypeError(
'{}() got {} unexpected option{}: {}.'.format(
cls.__name__, len(extraneous), 's'
if len(extraneous) > 1 else '', ', '.join(map(repr, sorted(extraneous)))
cls.__name__,
len(extraneous),
's' if len(extraneous) > 1 else '',
', '.join(map(repr, sorted(extraneous))),
)
)
@ -167,8 +164,10 @@ class Configurable(metaclass=ConfigurableMeta):
if _final:
raise TypeError(
'{}() missing {} required option{}: {}.'.format(
cls.__name__, len(missing), 's'
if len(missing) > 1 else '', ', '.join(map(repr, sorted(missing)))
cls.__name__,
len(missing),
's' if len(missing) > 1 else '',
', '.join(map(repr, sorted(missing))),
)
)
return PartiallyConfigured(cls, *args, **kwargs)

1
bonobo/config/functools.py
View File

@ -1,5 +1,4 @@
import functools
import itertools

9
bonobo/config/options.py
View File

@ -113,8 +113,9 @@ class RemovedOption(Option):
def clean(self, value):
if value != self.value:
raise ValueError(
'Removed options cannot change value, {!r} must now be {!r} (and you should remove setting the value explicitely, as it is deprecated and will be removed quite soon.'.
format(self.name, self.value)
'Removed options cannot change value, {!r} must now be {!r} (and you should remove setting the value explicitely, as it is deprecated and will be removed quite soon.'.format(
self.name, self.value
)
)
return self.value
@ -195,9 +196,7 @@ class Method(Option):
if not callable(value):
raise TypeError(
'Option {!r} ({}) is expecting a callable value, got {!r} object: {!r}.'.format(
self.name,
type(self).__name__,
type(value).__name__, value
self.name, type(self).__name__, type(value).__name__, value
)
)
inst._options_values[self.name] = self.type(value) if self.type else value

20
bonobo/config/processors.py
View File

@ -101,15 +101,17 @@ class ContextCurrifier:
try:
bound = self._bind(_input)
except TypeError as exc:
raise UnrecoverableTypeError((
'Input of {wrapped!r} does not bind to the node signature.\n'
'Args: {args}\n'
'Input: {input}\n'
'Kwargs: {kwargs}\n'
'Signature: {sig}'
).format(
wrapped=self.wrapped, args=self.args, input=_input, kwargs=self.kwargs, sig=signature(self.wrapped)
)) from exc
raise UnrecoverableTypeError(
(
'Input of {wrapped!r} does not bind to the node signature.\n'
'Args: {args}\n'
'Input: {input}\n'
'Kwargs: {kwargs}\n'
'Signature: {sig}'
).format(
wrapped=self.wrapped, args=self.args, input=_input, kwargs=self.kwargs, sig=signature(self.wrapped)
)
) from exc
return self.wrapped(*bound.args, **bound.kwargs)
def setup(self, *context):

3
bonobo/config/services.py
View File

@ -112,10 +112,12 @@ def create_container(services=None, factory=Container):
if not 'fs' in container:
import bonobo
container.setdefault('fs', bonobo.open_fs())
if not 'http' in container:
import requests
container.setdefault('http', requests)
return container
@ -139,6 +141,7 @@ class Exclusive(ContextDecorator):
ensure that.
"""
_locks = {}
def __init__(self, wrapped):

5
bonobo/contrib/django/__init__.py
View File

@ -9,7 +9,4 @@ This module contains all tools for Bonobo and Django to interract nicely.
from .utils import create_or_update
from .commands import ETLCommand
__all__ = [
'ETLCommand',
'create_or_update',
]
__all__ = ['ETLCommand', 'create_or_update']

11
bonobo/contrib/django/commands.py
View File

@ -1,13 +1,14 @@
from logging import getLogger
from types import GeneratorType
from colorama import Back, Fore, Style
from django.core.management import BaseCommand
from django.core.management.base import OutputWrapper
from mondrian import term
import bonobo
from bonobo.plugins.console import ConsoleOutputPlugin
from bonobo.util.term import CLEAR_EOL
from colorama import Fore, Back, Style
from django.core.management import BaseCommand
from django.core.management.base import OutputWrapper
from mondrian import term
from .utils import create_or_update
@ -55,7 +56,7 @@ class ETLCommand(BaseCommand):
graph_coll = self.get_graph(*args, **options)
if not isinstance(graph_coll, GeneratorType):
graph_coll = (graph_coll, )
graph_coll = (graph_coll,)
for i, graph in enumerate(graph_coll):
assert isinstance(graph, bonobo.Graph), 'Invalid graph provided.'

4
bonobo/contrib/google/__init__.py
View File

@ -41,14 +41,14 @@ def get_credentials(*, scopes):
return credentials
def get_google_spreadsheets_api_client(scopes=('https://www.googleapis.com/auth/spreadsheets', )):
def get_google_spreadsheets_api_client(scopes=('https://www.googleapis.com/auth/spreadsheets',)):
credentials = get_credentials(scopes=scopes)
http = credentials.authorize(httplib2.Http())
discoveryUrl = 'https://sheets.googleapis.com/$discovery/rest?version=v4'
return discovery.build('sheets', 'v4', http=http, discoveryServiceUrl=discoveryUrl, cache_discovery=False)
def get_google_people_api_client(scopes=('https://www.googleapis.com/auth/contacts', )):
def get_google_people_api_client(scopes=('https://www.googleapis.com/auth/contacts',)):
credentials = get_credentials(scopes=scopes)
http = credentials.authorize(httplib2.Http())
discoveryUrl = 'https://people.googleapis.com/$discovery/rest?version=v1'

4
bonobo/contrib/jupyter/__init__.py
View File

@ -5,6 +5,4 @@ def _jupyter_nbextension_paths():
return [{'section': 'notebook', 'src': 'static', 'dest': 'bonobo-jupyter', 'require': 'bonobo-jupyter/extension'}]
__all__ = [
'JupyterOutputPlugin',
]
__all__ = ['JupyterOutputPlugin']

10
bonobo/contrib/opendatasoft/__init__.py
View File

@ -44,15 +44,9 @@ class OpenDataSoftAPI(Configurable):
break
for row in records:
yield {
**row.get('fields', {}),
'geometry': row.get('geometry', {}),
'recordid': row.get('recordid'),
}
yield {**row.get('fields', {}), 'geometry': row.get('geometry', {}), 'recordid': row.get('recordid')}
start += self.rows
__all__ = [
'OpenDataSoftAPI',
]
__all__ = ['OpenDataSoftAPI']

8
bonobo/errors.py
View File

@ -16,10 +16,7 @@ class InactiveWritableError(InactiveIOError):
class ValidationError(RuntimeError):
def __init__(self, inst, message):
super(ValidationError, self).__init__(
'Validation error in {class_name}: {message}'.format(
class_name=type(inst).__name__,
message=message,
)
'Validation error in {class_name}: {message}'.format(class_name=type(inst).__name__, message=message)
)
@ -42,8 +39,7 @@ class AbstractError(UnrecoverableError, NotImplementedError):
def __init__(self, method):
super().__init__(
'Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format(
class_name=get_name(method.__self__),
method_name=get_name(method),
class_name=get_name(method.__self__), method_name=get_name(method)
)
)

19
bonobo/examples/__init__.py
View File

@ -4,19 +4,9 @@ import bonobo
def get_argument_parser(parser=None):
parser = bonobo.get_argument_parser(parser=parser)
parser.add_argument('--limit', '-l', type=int, default=None, help='If set, limits the number of processed lines.')
parser.add_argument(
'--limit',
'-l',
type=int,
default=None,
help='If set, limits the number of processed lines.'
)
parser.add_argument(
'--print',
'-p',
action='store_true',
default=False,
help='If set, pretty prints before writing to output file.'
'--print', '-p', action='store_true', default=False, help='If set, pretty prints before writing to output file.'
)
return parser
@ -26,7 +16,4 @@ def get_graph_options(options):
_limit = options.pop('limit', None)
_print = options.pop('print', False)
return {
'_limit': (bonobo.Limit(_limit), ) if _limit else (),
'_print': (bonobo.PrettyPrinter(), ) if _print else (),
}
return {'_limit': (bonobo.Limit(_limit),) if _limit else (), '_print': (bonobo.PrettyPrinter(),) if _print else ()}

8
bonobo/examples/clock.py
View File

@ -1,7 +1,8 @@
import bonobo
import datetime
import time
import bonobo
def extract():
"""Placeholder, change, rename, remove... """
@ -13,10 +14,7 @@ def extract():
def get_graph():
graph = bonobo.Graph()
graph.add_chain(
extract,
print,
)
graph.add_chain(extract, print)
return graph

31
bonobo/examples/datasets/__main__.py
View File

@ -4,26 +4,18 @@ import bonobo
from bonobo import examples
from bonobo.examples.datasets.coffeeshops import get_graph as get_coffeeshops_graph
from bonobo.examples.datasets.fablabs import get_graph as get_fablabs_graph
from bonobo.examples.datasets.services import get_services, get_datasets_dir, get_minor_version
from bonobo.examples.datasets.services import get_datasets_dir, get_minor_version, get_services
graph_factories = {
'coffeeshops': get_coffeeshops_graph,
'fablabs': get_fablabs_graph,
}
graph_factories = {'coffeeshops': get_coffeeshops_graph, 'fablabs': get_fablabs_graph}
if __name__ == '__main__':
parser = examples.get_argument_parser()
parser.add_argument(
'--target', '-t', choices=graph_factories.keys(), nargs='+'
)
parser.add_argument('--target', '-t', choices=graph_factories.keys(), nargs='+')
parser.add_argument('--sync', action='store_true', default=False)
with bonobo.parse_args(parser) as options:
graph_options = examples.get_graph_options(options)
graph_names = list(
options['target']
if options['target'] else sorted(graph_factories.keys())
)
graph_names = list(options['target'] if options['target'] else sorted(graph_factories.keys()))
# Create a graph with all requested subgraphs
graph = bonobo.Graph()
@ -43,18 +35,9 @@ if __name__ == '__main__':
for filename in files:
local_path = os.path.join(root, filename)
relative_path = os.path.relpath(local_path, local_dir)
s3_path = os.path.join(
get_minor_version(), relative_path
)
s3_path = os.path.join(get_minor_version(), relative_path)
try:
s3.head_object(
Bucket='bonobo-examples', Key=s3_path
)
s3.head_object(Bucket='bonobo-examples', Key=s3_path)
except:
s3.upload_file(
local_path,
'bonobo-examples',
s3_path,
ExtraArgs={'ACL': 'public-read'}
)
s3.upload_file(local_path, 'bonobo-examples', s3_path, ExtraArgs={'ACL': 'public-read'})

39
bonobo/examples/datasets/coffeeshops.py
View File

@ -13,48 +13,26 @@ def get_graph(graph=None, *, _limit=(), _print=()):
graph = graph or bonobo.Graph()
producer = graph.add_chain(
ODSReader(
dataset='liste-des-cafes-a-un-euro',
netloc='opendata.paris.fr'
),
ODSReader(dataset='liste-des-cafes-a-un-euro', netloc='opendata.paris.fr'),
*_limit,
bonobo.UnpackItems(0),
bonobo.Rename(
name='nom_du_cafe',
address='adresse',
zipcode='arrondissement'
),
bonobo.Rename(name='nom_du_cafe', address='adresse', zipcode='arrondissement'),
bonobo.Format(city='Paris', country='France'),
bonobo.OrderFields(
[
'name', 'address', 'zipcode', 'city', 'country',
'geometry', 'geoloc'
]
),
bonobo.OrderFields(['name', 'address', 'zipcode', 'city', 'country', 'geometry', 'geoloc']),
*_print,
)
# Comma separated values.
graph.add_chain(
bonobo.CsvWriter(
'coffeeshops.csv',
fields=['name', 'address', 'zipcode', 'city'],
delimiter=','
),
bonobo.CsvWriter('coffeeshops.csv', fields=['name', 'address', 'zipcode', 'city'], delimiter=','),
_input=producer.output,
)
# Standard JSON
graph.add_chain(
bonobo.JsonWriter(path='coffeeshops.json'),
_input=producer.output,
)
graph.add_chain(bonobo.JsonWriter(path='coffeeshops.json'), _input=producer.output)
# Line-delimited JSON
graph.add_chain(
bonobo.LdjsonWriter(path='coffeeshops.ldjson'),
_input=producer.output,
)
graph.add_chain(bonobo.LdjsonWriter(path='coffeeshops.ldjson'), _input=producer.output)
return graph
@ -63,7 +41,4 @@ if __name__ == '__main__':
parser = examples.get_argument_parser()
with bonobo.parse_args(parser) as options:
bonobo.run(
get_graph(**examples.get_graph_options(options)),
services=get_services()
)
bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())

17
bonobo/examples/datasets/fablabs.py
View File

@ -24,9 +24,7 @@ from bonobo.examples.datasets.services import get_services
try:
import pycountry
except ImportError as exc:
raise ImportError(
'You must install package "pycountry" to run this example.'
) from exc
raise ImportError('You must install package "pycountry" to run this example.') from exc
API_DATASET = 'fablabs@public-us'
ROWS = 100
@ -39,12 +37,8 @@ def _getlink(x):
def normalize(row):
result = {
**row,
'links':
list(filter(None, map(_getlink, json.loads(row.get('links'))))),
'country':
pycountry.countries.get(
alpha_2=row.get('country_code', '').upper()
).name,
'links': list(filter(None, map(_getlink, json.loads(row.get('links'))))),
'country': pycountry.countries.get(alpha_2=row.get('country_code', '').upper()).name,
}
return result
@ -66,7 +60,4 @@ if __name__ == '__main__':
parser = examples.get_argument_parser()
with bonobo.parse_args(parser) as options:
bonobo.run(
get_graph(**examples.get_graph_options(options)),
services=get_services()
)
bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())

4
bonobo/examples/datasets/services.py
View File

@ -9,9 +9,7 @@ def get_minor_version():
def get_datasets_dir(*dirs):
home_dir = os.path.expanduser('~')
target_dir = os.path.join(
home_dir, '.cache/bonobo', get_minor_version(), *dirs
)
target_dir = os.path.join(home_dir, '.cache/bonobo', get_minor_version(), *dirs)
os.makedirs(target_dir, exist_ok=True)
return target_dir

5
bonobo/examples/files/_services.py
View File

@ -2,7 +2,4 @@ from bonobo import get_examples_path, open_fs
def get_services():
return {
'fs': open_fs(get_examples_path()),
'fs.output': open_fs(),
}
return {'fs': open_fs(get_examples_path()), 'fs.output': open_fs()}

23
bonobo/examples/files/csv_handlers.py
View File

@ -5,8 +5,8 @@ from bonobo.examples.files._services import get_services
def get_graph(*, _limit=None, _print=False):
return bonobo.Graph(
bonobo.CsvReader('datasets/coffeeshops.txt'),
*((bonobo.Limit(_limit), ) if _limit else ()),
*((bonobo.PrettyPrinter(), ) if _print else ()),
*((bonobo.Limit(_limit),) if _limit else ()),
*((bonobo.PrettyPrinter(),) if _print else ()),
bonobo.CsvWriter('coffeeshops.csv', fs='fs.output')
)
@ -14,23 +14,10 @@ def get_graph(*, _limit=None, _print=False):
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
parser.add_argument('--limit', '-l', type=int, default=None, help='If set, limits the number of processed lines.')
parser.add_argument(
'--limit',
'-l',
type=int,
default=None,
help='If set, limits the number of processed lines.'
)
parser.add_argument(
'--print',
'-p',
action='store_true',
default=False,
help='If set, pretty prints before writing to output file.'
'--print', '-p', action='store_true', default=False, help='If set, pretty prints before writing to output file.'
)
with bonobo.parse_args(parser) as options:
bonobo.run(
get_graph(_limit=options['limit'], _print=options['print']),
services=get_services()
)
bonobo.run(get_graph(_limit=options['limit'], _print=options['print']), services=get_services())

34
bonobo/examples/files/json_handlers.py
View File

@ -5,22 +5,13 @@ from bonobo.examples.files._services import get_services
def get_graph(*, _limit=None, _print=False):
graph = bonobo.Graph()
trunk = graph.add_chain(
bonobo.JsonReader('datasets/theaters.json'),
*((bonobo.Limit(_limit), ) if _limit else ()),
)
trunk = graph.add_chain(bonobo.JsonReader('datasets/theaters.json'), *((bonobo.Limit(_limit),) if _limit else ()))
if _print:
graph.add_chain(bonobo.PrettyPrinter(), _input=trunk.output)
graph.add_chain(
bonobo.JsonWriter('theaters.json', fs='fs.output'),
_input=trunk.output
)
graph.add_chain(
bonobo.LdjsonWriter('theaters.ldjson', fs='fs.output'),
_input=trunk.output
)
graph.add_chain(bonobo.JsonWriter('theaters.json', fs='fs.output'), _input=trunk.output)
graph.add_chain(bonobo.LdjsonWriter('theaters.ldjson', fs='fs.output'), _input=trunk.output)
return graph
@ -28,23 +19,10 @@ def get_graph(*, _limit=None, _print=False):
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
parser.add_argument('--limit', '-l', type=int, default=None, help='If set, limits the number of processed lines.')
parser.add_argument(
'--limit',
'-l',
type=int,
default=None,
help='If set, limits the number of processed lines.'
)
parser.add_argument(
'--print',
'-p',
action='store_true',
default=False,
help='If set, pretty prints before writing to output file.'
'--print', '-p', action='store_true', default=False, help='If set, pretty prints before writing to output file.'
)
with bonobo.parse_args(parser) as options:
bonobo.run(
get_graph(_limit=options['limit'], _print=options['print']),
services=get_services()
)
bonobo.run(get_graph(_limit=options['limit'], _print=options['print']), services=get_services())

15
bonobo/examples/files/pickle_handlers.py
View File

@ -35,9 +35,7 @@ from bonobo import examples
def cleanse_sms(category, sms):
if category == 'spam':
sms_clean = '**MARKED AS SPAM** ' + sms[0:50] + (
'...' if len(sms) > 50 else ''
)
sms_clean = '**MARKED AS SPAM** ' + sms[0:50] + ('...' if len(sms) > 50 else '')
elif category == 'ham':
sms_clean = sms
else:
@ -62,16 +60,11 @@ def get_graph(*, _limit=(), _print=()):
def get_services():
from ._services import get_services
return {
**get_services(), 'fs':
TarFS(bonobo.get_examples_path('datasets/spam.tgz'))
}
return {**get_services(), 'fs': TarFS(bonobo.get_examples_path('datasets/spam.tgz'))}
if __name__ == '__main__':
parser = examples.get_argument_parser()
with bonobo.parse_args(parser) as options:
bonobo.run(
get_graph(**examples.get_graph_options(options)),
services=get_services()
)
bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())

5
bonobo/examples/files/text_handlers.py
View File

@ -23,7 +23,4 @@ def get_graph(*, _limit=(), _print=()):
if __name__ == '__main__':
parser = examples.get_argument_parser()
with bonobo.parse_args(parser) as options:
bonobo.run(
get_graph(**examples.get_graph_options(options)),
services=get_services()
)
bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())

6
bonobo/execution/contexts/__init__.py
View File

@ -9,8 +9,4 @@ from bonobo.execution.contexts.graph import GraphExecutionContext
from bonobo.execution.contexts.node import NodeExecutionContext
from bonobo.execution.contexts.plugin import PluginExecutionContext
__all__ = [
'GraphExecutionContext',
'NodeExecutionContext',
'PluginExecutionContext',
]
__all__ = ['GraphExecutionContext', 'NodeExecutionContext', 'PluginExecutionContext']

3
bonobo/execution/contexts/graph.py
View File

@ -1,12 +1,13 @@
from functools import partial
from time import sleep
from whistle import EventDispatcher
from bonobo.config import create_container
from bonobo.constants import BEGIN, END
from bonobo.execution import events
from bonobo.execution.contexts.node import NodeExecutionContext
from bonobo.execution.contexts.plugin import PluginExecutionContext
from whistle import EventDispatcher
class GraphExecutionContext:

9
bonobo/execution/contexts/node.py
View File

@ -7,11 +7,11 @@ from types import GeneratorType
from bonobo.config import create_container
from bonobo.config.processors import ContextCurrifier
from bonobo.constants import NOT_MODIFIED, BEGIN, END, TICK_PERIOD, Token, Flag, INHERIT
from bonobo.constants import BEGIN, END, INHERIT, NOT_MODIFIED, TICK_PERIOD, Flag, Token
from bonobo.errors import InactiveReadableError, UnrecoverableError, UnrecoverableTypeError
from bonobo.execution.contexts.base import BaseContext
from bonobo.structs.inputs import Input
from bonobo.util import get_name, isconfigurabletype, ensure_tuple
from bonobo.util import ensure_tuple, get_name, isconfigurabletype
from bonobo.util.bags import BagType
from bonobo.util.statistics import WithStatistics
@ -105,10 +105,7 @@ class NodeExecutionContext(BaseContext, WithStatistics):
except Empty:
sleep(TICK_PERIOD) # XXX: How do we determine this constant?
continue
except (
NotImplementedError,
UnrecoverableError,
):
except (NotImplementedError, UnrecoverableError):
self.fatal(sys.exc_info()) # exit loop
except Exception: # pylint: disable=broad-except
self.error(sys.exc_info()) # does not exit loop

4
bonobo/execution/strategies/__init__.py
View File

@ -9,9 +9,7 @@ at home if you want to give it a shot.
from bonobo.execution.strategies.executor import ProcessPoolExecutorStrategy, ThreadPoolExecutorStrategy
from bonobo.execution.strategies.naive import NaiveStrategy
__all__ = [
'create_strategy',
]
__all__ = ['create_strategy']
STRATEGIES = {
'naive': NaiveStrategy,

1
bonobo/execution/strategies/base.py
View File

@ -6,6 +6,7 @@ class Strategy:
Base class for execution strategies.
"""
GraphExecutionContextType = GraphExecutionContext
def __init__(self, GraphExecutionContextType=None):

28
bonobo/nodes/basics.py
View File

@ -3,14 +3,15 @@ import html
import itertools
import pprint
from mondrian import term
from bonobo import settings
from bonobo.config import Configurable, Option, Method, use_raw_input, use_context, use_no_input
from bonobo.config import Configurable, Method, Option, use_context, use_no_input, use_raw_input
from bonobo.config.functools import transformation_factory
from bonobo.config.processors import ContextProcessor, use_context_processor
from bonobo.constants import NOT_MODIFIED
from bonobo.util.objects import ValueHolder
from bonobo.util.term import CLEAR_EOL
from mondrian import term
__all__ = [
'FixedWindow',
@ -43,6 +44,7 @@ class Limit(Configurable):
TODO: simplify into a closure building factory?
"""
limit = Option(positional=True, default=10)
@ContextProcessor
@ -69,7 +71,7 @@ def Tee(f):
def _shorten(s, w):
if w and len(s) > w:
s = s[0:w - 3] + '...'
s = s[0 : w - 3] + '...'
return s
@ -80,17 +82,19 @@ class PrettyPrinter(Configurable):
required=False,
__doc__='''
If set, truncates the output values longer than this to this width.
'''
''',
)
filter = Method(
default=
(lambda self, index, key, value: (value is not None) and (not isinstance(key, str) or not key.startswith('_'))),
default=(
lambda self, index, key, value: (value is not None)
and (not isinstance(key, str) or not key.startswith('_'))
),
__doc__='''
A filter that determine what to print.
Default is to ignore any key starting with an underscore and none values.
'''
''',
)
@ContextProcessor
@ -99,6 +103,7 @@ class PrettyPrinter(Configurable):
yield context
if context._jupyter_html is not None:
from IPython.display import display, HTML
display(HTML('\n'.join(['<table>'] + context._jupyter_html + ['</table>'])))
def __call__(self, context, *args, **kwargs):
@ -153,16 +158,11 @@ class PrettyPrinter(Configurable):
if not context._jupyter_html:
context._jupyter_html = [
'<thead><tr>',
*map('<th>{}</th>'.format, map(html.escape, map(str,
context.get_input_fields() or range(len(args))))),
*map('<th>{}</th>'.format, map(html.escape, map(str, context.get_input_fields() or range(len(args))))),
'</tr></thead>',
]
context._jupyter_html += [
'<tr>',
*map('<td>{}</td>'.format, map(html.escape, map(repr, args))),
'</tr>',
]
context._jupyter_html += ['<tr>', *map('<td>{}</td>'.format, map(html.escape, map(repr, args))), '</tr>']
@use_no_input

3
bonobo/nodes/filter.py
View File

@ -1,6 +1,5 @@
from bonobo.constants import NOT_MODIFIED
from bonobo.config import Configurable, Method
from bonobo.constants import NOT_MODIFIED
class Filter(Configurable):

37
bonobo/nodes/io/base.py
View File

@ -13,22 +13,39 @@ class FileHandler(Configurable):
"""
path = Option(
str, required=True, positional=True, __doc__='''
str,
required=True,
positional=True,
__doc__='''
Path to use within the provided filesystem.
'''
''',
) # type: str
eol = Option(str, default='\n', __doc__='''
eol = Option(
str,
default='\n',
__doc__='''
Character to use as line separator.
''') # type: str
mode = Option(str, __doc__='''
''',
) # type: str
mode = Option(
str,
__doc__='''
What mode to use for open() call.
''') # type: str
encoding = Option(str, default='utf-8', __doc__='''
''',
) # type: str
encoding = Option(
str,
default='utf-8',
__doc__='''
Encoding.
''') # type: str
fs = Service('fs', __doc__='''
''',
) # type: str
fs = Service(
'fs',
__doc__='''
The filesystem instance to use.
''') # type: str
''',
) # type: str
@ContextProcessor
def file(self, context, *, fs):

6
bonobo/nodes/io/csv.py
View File

@ -1,6 +1,6 @@
import csv
from bonobo.config import Option, use_raw_input, use_context
from bonobo.config import Option, use_context, use_raw_input
from bonobo.config.options import Method, RenamedOption
from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.base import FileHandler
@ -62,7 +62,7 @@ class CsvReader(FileReader, CsvHandler):
default=0,
__doc__='''
If set and greater than zero, the reader will skip this amount of lines.
'''
''',
)
@Method(
@ -72,7 +72,7 @@ class CsvReader(FileReader, CsvHandler):
iterable.
Defaults to builtin csv.reader(...), but can be overriden to fit your special needs.
'''
''',
)
def reader_factory(self, file):
return csv.reader(file, **self.get_dialect_kwargs())

22
bonobo/nodes/io/file.py
View File

@ -1,4 +1,4 @@
from bonobo.config import Option, ContextProcessor, use_context
from bonobo.config import ContextProcessor, Option, use_context
from bonobo.constants import NOT_MODIFIED
from bonobo.errors import UnrecoverableError
from bonobo.nodes.io.base import FileHandler, Reader, Writer
@ -12,9 +12,13 @@ class FileReader(Reader, FileHandler):
present. Extending it is usually the right way to create more specific file readers (like json, csv, etc.)
"""
mode = Option(str, default='r', __doc__='''
mode = Option(
str,
default='r',
__doc__='''
What mode to use for open() call.
''') # type: str
''',
) # type: str
output_fields = Option(
ensure_tuple,
@ -22,14 +26,14 @@ class FileReader(Reader, FileHandler):
__doc__='''
Specify the field names of output lines.
Mutually exclusive with "output_type".
'''
''',
)
output_type = Option(
required=False,
__doc__='''
Specify the type of output lines.
Mutually exclusive with "output_fields".
'''
''',
)
@ContextProcessor
@ -72,9 +76,13 @@ class FileWriter(Writer, FileHandler):
usually the right way to create more specific file writers (like json, csv, etc.)
"""
mode = Option(str, default='w+', __doc__='''
mode = Option(
str,
default='w+',
__doc__='''
What mode to use for open() call.
''') # type: str
''',
) # type: str
def write(self, file, context, line, *, fs):
"""

56
bonobo/plugins/console.py
View File

@ -1,8 +1,9 @@
import io
import sys
from contextlib import redirect_stdout, redirect_stderr
from contextlib import redirect_stderr, redirect_stdout
from colorama import Style, Fore, init as initialize_colorama_output_wrappers
from colorama import Fore, Style
from colorama import init as initialize_colorama_output_wrappers
from bonobo import settings
from bonobo.execution import events
@ -34,7 +35,7 @@ class ConsoleOutputPlugin(Plugin):
isatty = False
# Whether we're on windows, or a real operating system.
iswindows = (sys.platform == 'win32')
iswindows = sys.platform == 'win32'
def __init__(self):
self.isatty = self._stdout.isatty()
@ -95,27 +96,32 @@ class ConsoleOutputPlugin(Plugin):
liveliness_color = alive_color if node.alive else dead_color
liveliness_prefix = ' {}{}{} '.format(liveliness_color, node.status, Style.RESET_ALL)
_line = ''.join((
liveliness_prefix,
node.name,
name_suffix,
' ',
node.get_statistics_as_string(),
' ',
node.get_flags_as_string(),
Style.RESET_ALL,
' ',
))
_line = ''.join(
(
liveliness_prefix,
node.name,
name_suffix,
' ',
node.get_statistics_as_string(),
' ',
node.get_flags_as_string(),
Style.RESET_ALL,
' ',
)
)
print(prefix + _line + CLEAR_EOL, file=self._stderr)
if append:
# todo handle multiline
print(
''.join((
' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append),
CLEAR_EOL
)),
file=self._stderr
''.join(
(
' `-> ',
' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append),
CLEAR_EOL,
)
),
file=self._stderr,
)
t_cnt += 1
@ -128,16 +134,17 @@ class ConsoleOutputPlugin(Plugin):
if self.counter % 10 and self._append_cache:
append = self._append_cache
else:
self._append_cache = append = (('Memory', '{0:.2f} Mb'.format(memory_usage())),
# ('Total time', '{0} s'.format(execution_time(harness))),
)
self._append_cache = append = (
('Memory', '{0:.2f} Mb'.format(memory_usage())),
# ('Total time', '{0} s'.format(execution_time(harness))),
)
else:
append = ()
self.write(context, prefix=self.prefix, append=append, rewind=rewind)
self.counter += 1
class IOBuffer():
class IOBuffer:
"""
The role of IOBuffer is to overcome the problem of multiple threads wanting to write to stdout at the same time. It
works a bit like a videogame: there are two buffers, one that is used to write, and one which is used to read from.
@ -164,5 +171,6 @@ class IOBuffer():
def memory_usage():
import os, psutil
process = psutil.Process(os.getpid())
return process.memory_info()[0] / float(2**20)
return process.memory_info()[0] / float(2 ** 20)

3
bonobo/registry.py
View File

@ -1,8 +1,7 @@
import mimetypes
import os
from bonobo import JsonReader, CsvReader, PickleReader, FileReader, FileWriter, PickleWriter, CsvWriter, JsonWriter
from bonobo import CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter, PickleReader, PickleWriter
FILETYPE_CSV = 'text/csv'
FILETYPE_JSON = 'application/json'

8
bonobo/settings.py
View File

@ -1,5 +1,4 @@
import logging
import os
from bonobo.errors import ValidationError
@ -92,17 +91,14 @@ LOGGING_LEVEL = Setting(
'LOGGING_LEVEL',
formatter=logging._checkLevel,
validator=logging._checkLevel,
default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO
default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO,
)
# Input/Output format for transformations
IOFORMAT_ARG0 = 'arg0'
IOFORMAT_KWARGS = 'kwargs'
IOFORMATS = {
IOFORMAT_ARG0,
IOFORMAT_KWARGS,
}
IOFORMATS = {IOFORMAT_ARG0, IOFORMAT_KWARGS}
IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__)

4
bonobo/structs/__init__.py
View File

@ -1,5 +1,3 @@
from bonobo.structs.graphs import Graph
__all__ = [
'Graph',
]
__all__ = ['Graph']

6
bonobo/structs/graphs.py
View File

@ -3,11 +3,12 @@ import json
from collections import namedtuple
from copy import copy
from bonobo.constants import BEGIN
from bonobo.util import get_name
from graphviz import ExecutableNotFound
from graphviz.dot import Digraph
from bonobo.constants import BEGIN
from bonobo.util import get_name
GraphRange = namedtuple('GraphRange', ['graph', 'input', 'output'])
@ -15,6 +16,7 @@ class Graph:
"""
Represents a directed graph of nodes.
"""
name = ''
def __init__(self, *chain):

2
bonobo/util/__init__.py
View File

@ -16,7 +16,7 @@ from bonobo.util.inspect import (
istuple,
istype,
)
from bonobo.util.objects import (get_name, get_attribute_or_create, ValueHolder)
from bonobo.util.objects import get_name, get_attribute_or_create, ValueHolder
# Bonobo's util API
__all__ = [

12
bonobo/util/api.py
View File

@ -12,14 +12,14 @@ class ApiHelper:
if graph:
# This function must comply to the "graph" API interface, meaning it can bahave like bonobo.run.
from inspect import signature
parameters = list(signature(x).parameters)
required_parameters = {'plugins', 'services', 'strategy'}
assert len(parameters
) > 0 and parameters[0] == 'graph', 'First parameter of a graph api function must be "graph".'
assert required_parameters.intersection(
parameters
) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join(
sorted(required_parameters)
assert (
len(parameters) > 0 and parameters[0] == 'graph'
), 'First parameter of a graph api function must be "graph".'
assert required_parameters.intersection(parameters) == required_parameters, (
'Graph api functions must define the following parameters: ' + ', '.join(sorted(required_parameters))
)
self.__all__.append(get_name(x))

19
bonobo/util/bags.py
View File

@ -73,7 +73,9 @@ class {typename}(tuple):
_field_template = '''\
{name} = _property(_itemgetter({index:d}), doc={doc!r})
'''.strip('\n')
'''.strip(
'\n'
)
_reserved = frozenset(
['_', '_cls', '_attrs', '_fields', 'get', '_asdict', '_replace', '_make', 'self', '_self', 'tuple'] + dir(tuple)
@ -150,16 +152,19 @@ def BagType(typename, fields, *, verbose=False, module=None):
attrs=attrs,
num_fields=len(fields),
arg_list=repr(attrs).replace("'", "")[1:-1],
repr_fmt=', '.join(('%r' if isinstance(fields[index], int) else '{name}=%r').format(name=name)
for index, name in enumerate(attrs)),
repr_fmt=', '.join(
('%r' if isinstance(fields[index], int) else '{name}=%r').format(name=name)
for index, name in enumerate(attrs)
),
field_defs='\n'.join(
_field_template.format(
index=index,
name=name,
doc='Alias for ' +
('field #{}'.format(index) if isinstance(fields[index], int) else repr(fields[index]))
) for index, name in enumerate(attrs)
)
doc='Alias for '
+ ('field #{}'.format(index) if isinstance(fields[index], int) else repr(fields[index])),
)
for index, name in enumerate(attrs)
),
)
# Execute the template string in a temporary namespace and support

2
bonobo/util/collections.py
View File

@ -26,7 +26,7 @@ def ensure_tuple(tuple_or_mixed, *, cls=tuple):
if isinstance(tuple_or_mixed, tuple):
return tuple.__new__(cls, tuple_or_mixed)
return tuple.__new__(cls, (tuple_or_mixed, ))
return tuple.__new__(cls, (tuple_or_mixed,))
def cast(type_):

2
bonobo/util/compat.py
View File

@ -9,7 +9,7 @@ def deprecated_alias(alias, func):
warnings.warn(
"Call to deprecated function alias {}, use {} instead.".format(alias, func.__name__),
category=DeprecationWarning,
stacklevel=2
stacklevel=2,
)
warnings.simplefilter('default', DeprecationWarning) # reset filter
return func(*args, **kwargs)

2
bonobo/util/environ.py
View File

@ -58,6 +58,7 @@ def get_argument_parser(parser=None):
"""
if parser is None:
import argparse
parser = argparse.ArgumentParser()
# Store globally to be able to warn the user about the fact he's probably wrong not to pass a parser to
@ -94,6 +95,7 @@ def parse_args(mixed=None):
)
# use the api from bonobo namespace, in case a command patched it.
import bonobo
mixed = bonobo.get_argument_parser()
if isinstance(mixed, argparse.ArgumentParser):

22
bonobo/util/inspect.py
View File

@ -9,6 +9,7 @@ def isconfigurable(mixed):
:return: bool
"""
from bonobo.config.configurables import Configurable
return isinstance(mixed, Configurable)
@ -47,6 +48,7 @@ def isoption(mixed):
"""
from bonobo.config.options import Option
return isinstance(mixed, Option)
@ -58,6 +60,7 @@ def ismethod(mixed):
:return: bool
"""
from bonobo.config.options import Method
return isinstance(mixed, Method)
@ -69,6 +72,7 @@ def iscontextprocessor(x):
:return: bool
"""
from bonobo.config.processors import ContextProcessor
return isinstance(x, ContextProcessor)
@ -102,15 +106,7 @@ def istuple(mixed):
return isinstance(mixed, tuple)
ConfigurableInspection = namedtuple(
'ConfigurableInspection', [
'type',
'instance',
'options',
'processors',
'partial',
]
)
ConfigurableInspection = namedtuple('ConfigurableInspection', ['type', 'instance', 'options', 'processors', 'partial'])
ConfigurableInspection.__enter__ = lambda self: self
ConfigurableInspection.__exit__ = lambda *exc_details: None
@ -141,10 +137,4 @@ def inspect_node(mixed, *, _partial=None):
'Not a Configurable, nor a Configurable instance and not even a partially configured Configurable. Check your inputs.'
)
return ConfigurableInspection(
typ,
inst,
list(typ.__options__),
list(typ.__processors__),
_partial,
)
return ConfigurableInspection(typ, inst, list(typ.__options__), list(typ.__processors__), _partial)

4
bonobo/util/objects.py
View File

@ -142,10 +142,10 @@ class ValueHolder:
return divmod(other, self._value)
def __pow__(self, other):
return self._value**other
return self._value ** other
def __rpow__(self, other):
return other**self._value
return other ** self._value
def __ipow__(self, other):
self._value **= other

2
bonobo/util/term.py
View File

@ -1,2 +1,2 @@
CLEAR_EOL = '\033[0K'
MOVE_CURSOR_UP = lambda n: '\033[{}A'.format(n)
MOVE_CURSOR_UP = lambda n: '\033[{}A'.format(n)

24
bonobo/util/testing.py
View File

@ -4,12 +4,12 @@ import io
import os
import runpy
import sys
from contextlib import contextmanager, redirect_stdout, redirect_stderr
from contextlib import contextmanager, redirect_stderr, redirect_stdout
from unittest.mock import patch
import pytest
from bonobo import open_fs, __main__, get_examples_path
from bonobo import __main__, get_examples_path, open_fs
from bonobo.commands import entrypoint
from bonobo.constants import Token
from bonobo.execution.contexts.graph import GraphExecutionContext
@ -112,19 +112,13 @@ def runner_module(args):
all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module])
all_environ_targets = pytest.mark.parametrize(
'target', [
(get_examples_path('environ.py'), ),
(
'-m',
'bonobo.examples.environ',
),
]
'target', [(get_examples_path('environ.py'),), ('-m', 'bonobo.examples.environ')]
)
@all_runners
@all_environ_targets
class EnvironmentTestCase():
class EnvironmentTestCase:
def run_quiet(self, runner, *args):
return runner('run', '--quiet', *args)
@ -216,12 +210,12 @@ class ReaderTest(ConfigurableNodeTest):
self.tmpdir = tmpdir
def get_create_args(self, *args):
return (self.filename, ) + args
return (self.filename,) + args
def test_customizable_output_type_transform_not_a_type(self):
context = self.NodeExecutionContextType(
self.create(*self.get_create_args(), output_type=str.upper, **self.get_create_kwargs()),
services=self.services
services=self.services,
)
with pytest.raises(TypeError):
context.start()
@ -229,9 +223,9 @@ class ReaderTest(ConfigurableNodeTest):
def test_customizable_output_type_transform_not_a_tuple(self):
context = self.NodeExecutionContextType(
self.create(
*self.get_create_args(), output_type=type('UpperString', (str, ), {}), **self.get_create_kwargs()
*self.get_create_args(), output_type=type('UpperString', (str,), {}), **self.get_create_kwargs()
),
services=self.services
services=self.services,
)
with pytest.raises(TypeError):
context.start()
@ -256,7 +250,7 @@ class WriterTest(ConfigurableNodeTest):
self.tmpdir = tmpdir
def get_create_args(self, *args):
return (self.filename, ) + args
return (self.filename,) + args
def readlines(self):
with self.fs.open(self.filename) as fp: