diff --git a/Makefile b/Makefile index 472b625..22a3a14 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-06-05 09:21:02.910936 +# Updated at 2017-06-08 21:45:05.840502 PACKAGE ?= bonobo PYTHON ?= $(shell which python) diff --git a/Projectfile b/Projectfile index 70eee1c..6fe6c2b 100644 --- a/Projectfile +++ b/Projectfile @@ -44,8 +44,9 @@ python.add_requirements( 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0', dev=[ - 'pytest-timeout >=1,<2', 'cookiecutter >=1.5,<1.6', + 'pytest-sugar >=0.8,<0.9', + 'pytest-timeout >=1,<2', ], docker=[ 'bonobo-docker', diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index b2e1bcc..c32e394 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -6,6 +6,7 @@ DEFAULT_SERVICES_ATTR = 'get_services' DEFAULT_GRAPH_FILENAME = '__main__.py' DEFAULT_GRAPH_ATTR = 'get_graph' + def get_default_services(filename, services=None): dirname = os.path.dirname(filename) services_filename = os.path.join(dirname, DEFAULT_SERVICES_FILENAME) diff --git a/bonobo/examples/files/csv_handlers.py b/bonobo/examples/files/csv_handlers.py index a15444d..33412c3 100644 --- a/bonobo/examples/files/csv_handlers.py +++ b/bonobo/examples/files/csv_handlers.py @@ -2,7 +2,7 @@ import bonobo from bonobo.commands.run import get_default_services graph = bonobo.Graph( - bonobo.CsvReader('datasets/coffeeshops.txt', headers=('item',)), + bonobo.CsvReader('datasets/coffeeshops.txt', headers=('item', )), bonobo.PrettyPrinter(), ) diff --git a/bonobo/examples/files/pickle_handlers.py b/bonobo/examples/files/pickle_handlers.py index 6863076..71a2b9a 100644 --- a/bonobo/examples/files/pickle_handlers.py +++ b/bonobo/examples/files/pickle_handlers.py @@ -52,12 +52,7 @@ graph = bonobo.Graph( def get_services(): - return { - 'fs': - TarFS( - bonobo.get_examples_path('datasets/spam.tgz') - ) - } + return {'fs': TarFS(bonobo.get_examples_path('datasets/spam.tgz'))} if __name__ == '__main__': diff --git a/bonobo/examples/tutorials/tut01e02.py b/bonobo/examples/tutorials/tut01e02.py index 3784235..78b7f43 100644 --- a/bonobo/examples/tutorials/tut01e02.py +++ b/bonobo/examples/tutorials/tut01e02.py @@ -1,7 +1,11 @@ import bonobo graph = bonobo.Graph( - ['foo', 'bar', 'baz', ], + [ + 'foo', + 'bar', + 'baz', + ], str.upper, print, ) diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py index bf3872d..da70444 100644 --- a/bonobo/nodes/io/csv.py +++ b/bonobo/nodes/io/csv.py @@ -59,7 +59,7 @@ class CsvReader(CsvHandler, FileReader): for row in reader: if len(row) != field_count: - raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count,)) + raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count, )) yield self.get_output(dict(zip(_headers, row))) diff --git a/bonobo/nodes/io/file.py b/bonobo/nodes/io/file.py index 3e2c51d..53ba138 100644 --- a/bonobo/nodes/io/file.py +++ b/bonobo/nodes/io/file.py @@ -21,10 +21,8 @@ class FileHandler(Configurable): eol = Option(str, default='\n') # type: str mode = Option(str) # type: str encoding = Option(str, default='utf-8') # type: str - fs = Service('fs') # type: str - - ioformat = Option(settings.validate_io_format, default=settings.IOFORMAT) + ioformat = Option(default=settings.IOFORMAT.get) @ContextProcessor def file(self, context, fs): diff --git a/bonobo/settings.py b/bonobo/settings.py index 9481bb2..e0e5289 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -1,6 +1,5 @@ -import os - import logging +import os from bonobo.errors import ValidationError @@ -13,6 +12,36 @@ def to_bool(s): return False +class Setting: + def __init__(self, name, default=None, validator=None): + self.name = name + + if default: + self.default = default if callable(default) else lambda: default + else: + self.default = lambda: None + + if validator: + self.validator = validator + else: + self.validator = None + + def __repr__(self): + return ''.format(self.name, self.value) + + def set(self, value): + if self.validator and not self.validator(value): + raise ValidationError('Invalid value {!r} for setting {}.'.format(value, self.name)) + self.value = value + + def get(self): + try: + return self.value + except AttributeError: + self.value = self.default() + return self.value + + # Debug/verbose mode. DEBUG = to_bool(os.environ.get('DEBUG', 'f')) @@ -34,21 +63,9 @@ IOFORMATS = { IOFORMAT_KWARGS, } -IOFORMAT = os.environ.get('IOFORMAT', IOFORMAT_KWARGS) - - -def validate_io_format(v): - if callable(v): - return v - if v in IOFORMATS: - return v - raise ValidationError('Unsupported format {!r}.'.format(v)) +IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__) def check(): if DEBUG and QUIET: raise RuntimeError('I cannot be verbose and quiet at the same time.') - - if IOFORMAT not in IOFORMATS: - raise RuntimeError('Invalid default input/output format.') - diff --git a/bonobo/util/iterators.py b/bonobo/util/iterators.py index ae39a49..82f8518 100644 --- a/bonobo/util/iterators.py +++ b/bonobo/util/iterators.py @@ -21,7 +21,7 @@ def force_iterator(mixed): def ensure_tuple(tuple_or_mixed): if isinstance(tuple_or_mixed, tuple): return tuple_or_mixed - return (tuple_or_mixed,) + return (tuple_or_mixed, ) def tuplize(generator): diff --git a/requirements-dev.txt b/requirements-dev.txt index c499557..a8ae766 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ arrow==0.10.0 babel==2.4.0 binaryornot==0.4.3 certifi==2017.4.17 -chardet==3.0.3 +chardet==3.0.4 click==6.7 cookiecutter==1.5.1 coverage==4.4.1 @@ -19,6 +19,7 @@ poyo==0.4.1 py==1.4.34 pygments==2.2.0 pytest-cov==2.5.1 +pytest-sugar==0.8.0 pytest-timeout==1.2.0 pytest==3.1.1 python-dateutil==2.6.0 @@ -28,5 +29,6 @@ six==1.10.0 snowballstemmer==1.2.1 sphinx==1.6.2 sphinxcontrib-websupport==1.0.1 +termcolor==1.1.0 urllib3==1.21.1 whichcraft==0.4.1 diff --git a/requirements-docker.txt b/requirements-docker.txt index 792a827..19ae90f 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,9 +1,9 @@ -e .[docker] appdirs==1.4.3 -bonobo-docker==0.2.4 +bonobo-docker==0.2.5 bonobo==0.3.1 certifi==2017.4.17 -chardet==3.0.3 +chardet==3.0.4 colorama==0.3.9 docker-pycreds==0.2.1 docker==2.3.0 @@ -17,6 +17,6 @@ pyparsing==2.2.0 pytz==2017.2 requests==2.17.3 six==1.10.0 -stevedore==1.22.0 +stevedore==1.23.0 urllib3==1.21.1 websocket-client==0.40.0 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 921057c..7db4a67 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -2,7 +2,7 @@ appnope==0.1.0 bleach==2.0.0 decorator==4.0.11 -entrypoints==0.2.2 +entrypoints==0.2.3 html5lib==0.999999999 ipykernel==4.6.1 ipython-genutils==0.2.0 diff --git a/requirements.txt b/requirements.txt index 38ba798..ab50600 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -e . appdirs==1.4.3 certifi==2017.4.17 -chardet==3.0.3 +chardet==3.0.4 colorama==0.3.9 enum34==1.1.6 fs==2.0.3 @@ -13,5 +13,5 @@ pyparsing==2.2.0 pytz==2017.2 requests==2.17.3 six==1.10.0 -stevedore==1.22.0 +stevedore==1.23.0 urllib3==1.21.1 diff --git a/setup.py b/setup.py index 540ea1b..89c9ccd 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,8 @@ setup( extras_require={ 'dev': [ 'cookiecutter (>= 1.5, < 1.6)', 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', - 'pytest-cov (>= 2.5, < 3.0)', 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)' + 'pytest-cov (>= 2.5, < 3.0)', 'pytest-sugar (>= 0.8, < 0.9)', 'pytest-timeout (>= 1, < 2)', + 'sphinx (>= 1.6, < 2.0)' ], 'docker': ['bonobo-docker'], 'jupyter': ['ipywidgets (>= 6.0.0, < 7)', 'jupyter (>= 1.0, < 1.1)'] diff --git a/tests/io/test_csv.py b/tests/io/test_csv.py index 3a6fd37..47e641b 100644 --- a/tests/io/test_csv.py +++ b/tests/io/test_csv.py @@ -1,6 +1,6 @@ import pytest -from bonobo import Bag, CsvReader, CsvWriter, open_fs +from bonobo import Bag, CsvReader, CsvWriter, open_fs, settings from bonobo.constants import BEGIN, END from bonobo.execution.node import NodeExecutionContext from bonobo.util.testing import CapturingNodeExecutionContext @@ -9,7 +9,7 @@ from bonobo.util.testing import CapturingNodeExecutionContext def test_write_csv_to_file(tmpdir): fs, filename = open_fs(tmpdir), 'output.csv' - writer = CsvWriter(path=filename) + writer = CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0) context = NodeExecutionContext(writer, services={'fs': fs}) context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END) @@ -19,7 +19,7 @@ def test_write_csv_to_file(tmpdir): context.step() context.stop() - with fs.open(filename)as fp: + with fs.open(filename) as fp: assert fp.read() == 'foo\nbar\nbaz\n' with pytest.raises(AttributeError): @@ -31,7 +31,7 @@ def test_read_csv_from_file(tmpdir): with fs.open(filename, 'w') as fp: fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar') - reader = CsvReader(path=filename, delimiter=',') + reader = CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0) context = CapturingNodeExecutionContext(reader, services={'fs': fs}) @@ -64,7 +64,7 @@ def test_read_csv_kwargs_output_formater(tmpdir): with fs.open(filename, 'w') as fp: fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar') - reader = CsvReader(path=filename, delimiter=',', output_format='kwargs') + reader = CsvReader(path=filename, delimiter=',') context = CapturingNodeExecutionContext(reader, services={'fs': fs}) diff --git a/tests/io/test_json.py b/tests/io/test_json.py index 56f679f..15d9e7e 100644 --- a/tests/io/test_json.py +++ b/tests/io/test_json.py @@ -1,6 +1,6 @@ import pytest -from bonobo import Bag, JsonReader, JsonWriter, open_fs +from bonobo import Bag, JsonReader, JsonWriter, open_fs, settings from bonobo.constants import BEGIN, END from bonobo.execution.node import NodeExecutionContext from bonobo.util.testing import CapturingNodeExecutionContext @@ -9,7 +9,7 @@ from bonobo.util.testing import CapturingNodeExecutionContext def test_write_json_to_file(tmpdir): fs, filename = open_fs(tmpdir), 'output.json' - writer = JsonWriter(path=filename) + writer = JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0) context = NodeExecutionContext(writer, services={'fs': fs}) context.start() @@ -31,7 +31,7 @@ def test_read_json_from_file(tmpdir): fs, filename = open_fs(tmpdir), 'input.json' with fs.open(filename, 'w') as fp: fp.write('[{"x": "foo"},{"x": "bar"}]') - reader = JsonReader(path=filename) + reader = JsonReader(filename, ioformat=settings.IOFORMAT_ARG0) context = CapturingNodeExecutionContext(reader, services={'fs': fs}) diff --git a/tests/io/test_pickle.py b/tests/io/test_pickle.py index 368e526..1709b16 100644 --- a/tests/io/test_pickle.py +++ b/tests/io/test_pickle.py @@ -1,7 +1,8 @@ import pickle + import pytest -from bonobo import Bag, PickleReader, PickleWriter, open_fs +from bonobo import Bag, PickleReader, PickleWriter, open_fs, settings from bonobo.constants import BEGIN, END from bonobo.execution.node import NodeExecutionContext from bonobo.util.testing import CapturingNodeExecutionContext @@ -10,7 +11,7 @@ from bonobo.util.testing import CapturingNodeExecutionContext def test_write_pickled_dict_to_file(tmpdir): fs, filename = open_fs(tmpdir), 'output.pkl' - writer = PickleWriter(path=filename) + writer = PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0) context = NodeExecutionContext(writer, services={'fs': fs}) context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END) @@ -32,7 +33,7 @@ def test_read_pickled_list_from_file(tmpdir): with fs.open(filename, 'wb') as fp: fp.write(pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']])) - reader = PickleReader(path=filename) + reader = PickleReader(filename, ioformat=settings.IOFORMAT_ARG0) context = CapturingNodeExecutionContext(reader, services={'fs': fs})