paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge tag '0.5.1' into develop

Browse Source 
0.5.1
This commit is contained in:
Romain Dorgueil
2017-10-21 12:54:46 +02:00
parent 92cc400fe7 c7f39aa851
commit 4f1eeacd45
26 changed files with 161 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
Makefile
View File

@ -1,7 +1,5 @@
# This file has been auto-generated.
# All changes will be lost, see Projectfile.
#
# Updated at 2017-10-05 18:56:33.985014
# This file has been auto-generated by Medikit. All changes will be lost.
# Updated on 2017-10-21.
PACKAGE ?= bonobo
PYTHON ?= $(shell which python)
@ -22,7 +20,7 @@ YAPF ?= $(PYTHON) -m yapf
YAPF_OPTIONS ?= -rip
VERSION ?= $(shell git describe 2>/dev/null || echo dev)
.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev test
.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev test update update-requirements
# Installs the local project dependencies.
install:
@ -40,6 +38,16 @@ install-dev:
clean:
rm -rf build dist *.egg-info
# Update project artifacts using medikit, after installing it eventually.
update:
python -c 'import medikit; print(medikit.__version__)' || pip install medikit;
$(PYTHON) -m medikit update
# Remove requirements files and update project artifacts using medikit, after installing it eventually.
update-requirements:
rm -rf requirements*.txt
$(MAKE) update
test: install-dev
$(PYTEST) $(PYTEST_OPTIONS) tests

4
Projectfile
View File

@ -1,6 +1,6 @@
# bonobo (see github.com/python-edgy/project)
# bonobo's description for medikit
from edgy.project import require
from medikit import require
pytest = require('pytest')
python = require('python')

2
bonobo/_version.py
View File

@ -1 +1 @@
__version__ = '0.5.0'
__version__ = '0.5.1'

5
bonobo/commands/run.py
View File

@ -3,7 +3,10 @@ import os
import bonobo
from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME
DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', )
DEFAULT_GRAPH_FILENAMES = (
'__main__.py',
'main.py',
)
DEFAULT_GRAPH_ATTR = 'get_graph'

5
bonobo/config/configurables.py
View File

@ -50,7 +50,10 @@ class ConfigurableMeta(type):
return (processor for _, processor in cls.__processors)
def __repr__(self):
return ' '.join(('<Configurable', super(ConfigurableMeta, self).__repr__().split(' ', 1)[1], ))
return ' '.join((
'<Configurable',
super(ConfigurableMeta, self).__repr__().split(' ', 1)[1],
))
try:

5
bonobo/config/options.py
View File

@ -151,8 +151,9 @@ class Method(Option):
def __set__(self, inst, value):
if not hasattr(value, '__call__'):
raise TypeError(
'Option of type {!r} is expecting a callable value, got {!r} object (which is not).'.
format(type(self).__name__, type(value).__name__)
'Option of type {!r} is expecting a callable value, got {!r} object (which is not).'.format(
type(self).__name__, type(value).__name__
)
)
inst._options_values[self.name] = self.type(value) if self.type else value

4
bonobo/config/services.py
View File

@ -49,8 +49,8 @@ class Service(Option):
"""
def __init__(self, name):
super().__init__(str, required=False, default=name)
def __init__(self, name, __doc__=None):
super().__init__(str, required=False, default=name, __doc__=__doc__)
def __set__(self, inst, value):
inst._options_values[self.name] = validate_service_name(value)

19
bonobo/examples/datasets/fablabs.py
View File

@ -61,22 +61,27 @@ def display(row):
row.get('city', None)
)
)
), row.get('county', None), row.get('country'),
),
row.get('county', None),
row.get('country'),
)
)
)
print(
' - {}address{}: {address}'.
format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address))
' - {}address{}: {address}'.format(
Fore.BLUE, Style.RESET_ALL, address=', '.join(address)
)
)
print(
' - {}links{}: {links}'.
format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links']))
' - {}links{}: {links}'.format(
Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links'])
)
)
print(
' - {}geometry{}: {geometry}'.
format(Fore.BLUE, Style.RESET_ALL, **row)
' - {}geometry{}: {geometry}'.format(
Fore.BLUE, Style.RESET_ALL, **row
)
)
print(
' - {}source{}: {source}'.format(

29
bonobo/ext/console.py
View File

@ -89,15 +89,32 @@ class ConsoleOutputPlugin(Plugin):
if node.alive:
_line = ''.join(
(
' ', alive_color, '+', Style.RESET_ALL, ' ', node.name, name_suffix, ' ',
node.get_statistics_as_string(), Style.RESET_ALL, ' ',
' ',
alive_color,
'+',
Style.RESET_ALL,
' ',
node.name,
name_suffix,
' ',
node.get_statistics_as_string(),
Style.RESET_ALL,
' ',
)
)
else:
_line = ''.join(
(
' ', dead_color, '-', ' ', node.name, name_suffix, ' ', node.get_statistics_as_string(),
Style.RESET_ALL, ' ',
' ',
dead_color,
'-',
' ',
node.name,
name_suffix,
' ',
node.get_statistics_as_string(),
Style.RESET_ALL,
' ',
)
)
print(prefix + _line + '\033[0K', file=sys.stderr)
@ -107,8 +124,8 @@ class ConsoleOutputPlugin(Plugin):
print(
''.join(
(
' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v)
for k, v in append), CLEAR_EOL
' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append),
CLEAR_EOL
)
),
file=sys.stderr

5
bonobo/nodes/io/csv.py
View File

@ -62,7 +62,10 @@ class CsvReader(FileReader, CsvHandler):
for row in reader:
if len(row) != field_count:
raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count, ))
raise ValueError('Got a line with %d fields, expecting %d.' % (
len(row),
field_count,
))
yield dict(zip(_headers, row))

5
bonobo/nodes/io/pickle.py
View File

@ -53,7 +53,10 @@ class PickleReader(FileReader, PickleHandler):
for i in iterator:
if len(i) != item_count:
raise ValueError('Received an object with %d items, expecting %d.' % (len(i), item_count, ))
raise ValueError('Received an object with %d items, expecting %d.' % (
len(i),
item_count,
))
yield dict(zip(i)) if is_dict else dict(zip(pickle_headers.value, i))

16
bonobo/structs/bags.py
View File

@ -45,7 +45,10 @@ class Bag:
def args(self):
if self._parent is None:
return self._args
return (*self._parent.args, *self._args, )
return (
*self._parent.args,
*self._args,
)
@property
def kwargs(self):
@ -122,11 +125,12 @@ class Bag:
def __repr__(self):
return '<{} ({})>'.format(
type(self).__name__, ', '.
join(itertools.chain(
map(repr, self.args),
('{}={}'.format(k, repr(v)) for k, v in self.kwargs.items()),
))
type(self).__name__, ', '.join(
itertools.chain(
map(repr, self.args),
('{}={}'.format(k, repr(v)) for k, v in self.kwargs.items()),
)
)
)

7
bonobo/util/iterators.py
View File

@ -38,6 +38,11 @@ def tuplize(generator):
def iter_if_not_sequence(mixed):
if isinstance(mixed, (dict, list, str, bytes, )):
if isinstance(mixed, (
dict,
list,
str,
bytes,
)):
raise TypeError(type(mixed).__name__)
return iter(mixed)

1
docs/conf.py
View File

@ -76,6 +76,7 @@ html_theme_options = {
'github_user': 'python-bonobo',
'github_repo': 'bonobo',
'github_button': 'true',
'github_banner': 'true',
'show_powered_by': 'false',
'show_related': 'true',
}

6
docs/tutorial/index.rst
View File

@ -5,7 +5,7 @@ What is Bonobo?
:::::::::::::::
Bonobo is an ETL (Extract-Transform-Load) framework for python 3.5. The goal is to define data-transformations, with
python code in charge of handling similar shaped independant lines of data.
python code in charge of handling similar shaped independent lines of data.
Bonobo *is not* a statistical or data-science tool. If you're looking for a data-analysis tool in python, use Pandas.
@ -21,13 +21,13 @@ Tutorial
Good documentation is not easy to write. We do our best to make it better and better.
Although all content here should be accurate, you may feel a lack of completeness, for which we plaid guilty and
Although all content here should be accurate, you may feel a lack of completeness, for which we plead guilty and
apologize.
If you're stuck, please come and ask on our `slack channel <https://bonobo-slack.herokuapp.com/>`_, we'll figure
something out.
If you're not stuck but had trouble understanding something, please consider contributing to the docs (via github
If you're not stuck but had trouble understanding something, please consider contributing to the docs (via GitHub
pull requests).
.. toctree::

6
requirements-docker.txt
View File

@ -1,16 +1,16 @@
-e .[docker]
appdirs==1.4.3
bonobo-docker==0.2.11
bonobo-docker==0.2.12
certifi==2017.7.27.1
chardet==3.0.4
colorama==0.3.9
docker-pycreds==0.2.1
docker==2.3.0
fs==2.0.11
fs==2.0.12
idna==2.6
packaging==16.8
pbr==3.1.1
psutil==5.3.1
psutil==5.4.0
pyparsing==2.2.0
pytz==2017.2
requests==2.18.4

2
requirements-jupyter.txt
View File

@ -19,7 +19,7 @@ markupsafe==1.0
mistune==0.7.4
nbconvert==5.3.1
nbformat==4.4.0
notebook==5.1.0
notebook==5.2.0
pandocfilters==1.4.2
parso==0.1.0
pexpect==4.2.1

4
requirements.txt
View File

@ -3,11 +3,11 @@ appdirs==1.4.3
certifi==2017.7.27.1
chardet==3.0.4
colorama==0.3.9
fs==2.0.11
fs==2.0.12
idna==2.6
packaging==16.8
pbr==3.1.1
psutil==5.3.1
psutil==5.4.0
pyparsing==2.2.0
pytz==2017.2
requests==2.18.4

2
setup.py
View File

@ -1,4 +1,4 @@
# This file is autogenerated by edgy.project code generator.
# This file is autogenerated by medikit code generator.
# All changes will be overwritten.
from setuptools import setup, find_packages

25
tests/config/test_methods.py
View File

@ -50,7 +50,10 @@ def test_define_with_decorator():
calls = []
def my_handler(*args, **kwargs):
calls.append((args, kwargs, ))
calls.append((
args,
kwargs,
))
Concrete = MethodBasedConfigurable(my_handler)
@ -74,7 +77,10 @@ def test_late_binding_method_decoration():
@MethodBasedConfigurable(foo='foo')
def Concrete(*args, **kwargs):
calls.append((args, kwargs, ))
calls.append((
args,
kwargs,
))
assert callable(Concrete.handler)
t = Concrete(bar='baz')
@ -89,7 +95,10 @@ def test_define_with_argument():
calls = []
def concrete_handler(*args, **kwargs):
calls.append((args, kwargs, ))
calls.append((
args,
kwargs,
))
t = MethodBasedConfigurable(concrete_handler, 'foo', bar='baz')
assert callable(t.handler)
@ -103,7 +112,10 @@ def test_define_with_inheritance():
class Inheriting(MethodBasedConfigurable):
def handler(self, *args, **kwargs):
calls.append((args, kwargs, ))
calls.append((
args,
kwargs,
))
t = Inheriting('foo', bar='baz')
assert callable(t.handler)
@ -120,7 +132,10 @@ def test_inheritance_then_decorate():
@Inheriting
def Concrete(*args, **kwargs):
calls.append((args, kwargs, ))
calls.append((
args,
kwargs,
))
assert callable(Concrete.handler)
t = Concrete('foo', bar='baz')

5
tests/config/test_methods_partial.py
View File

@ -53,7 +53,10 @@ def test_partial():
assert len(ci.options) == 4
assert len(ci.processors) == 1
assert ci.partial
assert ci.partial[0] == (f1, f2, )
assert ci.partial[0] == (
f1,
f2,
)
assert not len(ci.partial[1])
c = C('foo')

4
tests/config/test_services.py
View File

@ -28,9 +28,7 @@ SERVICES = Container(
class MyServiceDependantConfigurable(Configurable):
printer = Service(
PrinterInterface,
)
printer = Service(PrinterInterface, )
def __call__(self, printer: PrinterInterface, *args):
return printer.print(*args)

9
tests/io/test_csv.py
View File

@ -18,9 +18,12 @@ def test_write_csv_ioformat_arg0(tmpdir):
CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0),
@pytest.mark.parametrize('add_kwargs', ({}, {
'ioformat': settings.IOFORMAT_KWARGS,
}, ))
@pytest.mark.parametrize('add_kwargs', (
{},
{
'ioformat': settings.IOFORMAT_KWARGS,
},
))
def test_write_csv_to_file_kwargs(tmpdir, add_kwargs):
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)

9
tests/io/test_json.py
View File

@ -19,9 +19,12 @@ def test_write_json_ioformat_arg0(tmpdir):
JsonReader(filename, ioformat=settings.IOFORMAT_ARG0),
@pytest.mark.parametrize('add_kwargs', ({}, {
'ioformat': settings.IOFORMAT_KWARGS,
}, ))
@pytest.mark.parametrize('add_kwargs', (
{},
{
'ioformat': settings.IOFORMAT_KWARGS,
},
))
def test_write_json_kwargs(tmpdir, add_kwargs):
fs, filename, services = json_tester.get_services_for_writer(tmpdir)

26
tests/structs/test_bags.py
View File

@ -5,7 +5,10 @@ from bonobo import Bag
from bonobo.constants import INHERIT_INPUT
from bonobo.structs import Token
args = ('foo', 'bar', )
args = (
'foo',
'bar',
)
kwargs = dict(acme='corp')
@ -38,11 +41,17 @@ def test_inherit():
assert bag.kwargs == {'a': 1}
assert bag.flags is ()
assert bag2.args == ('a', 'b', )
assert bag2.args == (
'a',
'b',
)
assert bag2.kwargs == {'a': 1, 'b': 2}
assert INHERIT_INPUT in bag2.flags
assert bag3.args == ('a', 'c', )
assert bag3.args == (
'a',
'c',
)
assert bag3.kwargs == {'a': 1, 'c': 3}
assert bag3.flags is ()
@ -51,12 +60,19 @@ def test_inherit():
assert bag4.flags is ()
bag4.set_parent(bag)
assert bag4.args == ('a', 'd', )
assert bag4.args == (
'a',
'd',
)
assert bag4.kwargs == {'a': 1, 'd': 4}
assert bag4.flags is ()
bag4.set_parent(bag3)
assert bag4.args == ('a', 'c', 'd', )
assert bag4.args == (
'a',
'c',
'd',
)
assert bag4.kwargs == {'a': 1, 'c': 3, 'd': 4}
assert bag4.flags is ()

5
tests/util/test_statistics.py
View File

@ -3,7 +3,10 @@ from bonobo.util.statistics import WithStatistics
class MyThingWithStats(WithStatistics):
def get_statistics(self, *args, **kwargs):
return (('foo', 42), ('bar', 69), )
return (
('foo', 42),
('bar', 69),
)
def test_with_statistics():