Refactoring and fixes around ioformats.

Makefile

@@ -1,7 +1,7 @@
 # This file has been auto-generated.
 # All changes will be lost, see Projectfile.
 #
-# Updated at 2017-06-05 09:21:02.910936
+# Updated at 2017-06-08 21:45:05.840502
 
 PACKAGE ?= bonobo
 PYTHON ?= $(shell which python)

Projectfile

@@ -44,8 +44,9 @@ python.add_requirements(
     'requests >=2.0,<3.0',
     'stevedore >=1.21,<2.0',
     dev=[
-        'pytest-timeout >=1,<2',
         'cookiecutter >=1.5,<1.6',
+        'pytest-sugar >=0.8,<0.9',
+        'pytest-timeout >=1,<2',
     ],
     docker=[
         'bonobo-docker',

bonobo/commands/run.py

@@ -6,6 +6,7 @@ DEFAULT_SERVICES_ATTR = 'get_services'
 DEFAULT_GRAPH_FILENAME = '__main__.py'
 DEFAULT_GRAPH_ATTR = 'get_graph'
 
+
 def get_default_services(filename, services=None):
     dirname = os.path.dirname(filename)
     services_filename = os.path.join(dirname, DEFAULT_SERVICES_FILENAME)

bonobo/examples/files/csv_handlers.py

@@ -2,7 +2,7 @@ import bonobo
 from bonobo.commands.run import get_default_services
 
 graph = bonobo.Graph(
-    bonobo.CsvReader('datasets/coffeeshops.txt', headers=('item',)),
+    bonobo.CsvReader('datasets/coffeeshops.txt', headers=('item', )),
     bonobo.PrettyPrinter(),
 )
 

bonobo/examples/files/pickle_handlers.py

@@ -52,12 +52,7 @@ graph = bonobo.Graph(
 
 
 def get_services():
-    return {
+    return {'fs': TarFS(bonobo.get_examples_path('datasets/spam.tgz'))}
-        'fs':
-            TarFS(
-                bonobo.get_examples_path('datasets/spam.tgz')
-            )
-    }
 
 
 if __name__ == '__main__':

bonobo/examples/tutorials/tut01e02.py

@@ -1,7 +1,11 @@
 import bonobo
 
 graph = bonobo.Graph(
-    ['foo', 'bar', 'baz', ],
+    [
+        'foo',
+        'bar',
+        'baz',
+    ],
     str.upper,
     print,
 )

bonobo/nodes/io/csv.py

@@ -59,7 +59,7 @@ class CsvReader(CsvHandler, FileReader):
 
         for row in reader:
             if len(row) != field_count:
-                raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count,))
+                raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count, ))
 
             yield self.get_output(dict(zip(_headers, row)))
 

bonobo/nodes/io/file.py

@@ -21,10 +21,8 @@ class FileHandler(Configurable):
     eol = Option(str, default='\n')  # type: str
     mode = Option(str)  # type: str
     encoding = Option(str, default='utf-8')  # type: str
-
     fs = Service('fs')  # type: str
-
+    ioformat = Option(default=settings.IOFORMAT.get)
-    ioformat = Option(settings.validate_io_format, default=settings.IOFORMAT)
 
     @ContextProcessor
     def file(self, context, fs):

bonobo/settings.py

@@ -1,6 +1,5 @@
-import os
-
 import logging
+import os
 
 from bonobo.errors import ValidationError
 
@@ -13,6 +12,36 @@ def to_bool(s):
     return False
 
 
+class Setting:
+    def __init__(self, name, default=None, validator=None):
+        self.name = name
+
+        if default:
+            self.default = default if callable(default) else lambda: default
+        else:
+            self.default = lambda: None
+
+        if validator:
+            self.validator = validator
+        else:
+            self.validator = None
+
+    def __repr__(self):
+        return '<Setting {}={!r}>'.format(self.name, self.value)
+
+    def set(self, value):
+        if self.validator and not self.validator(value):
+            raise ValidationError('Invalid value {!r} for setting {}.'.format(value, self.name))
+        self.value = value
+
+    def get(self):
+        try:
+            return self.value
+        except AttributeError:
+            self.value = self.default()
+            return self.value
+
+
 # Debug/verbose mode.
 DEBUG = to_bool(os.environ.get('DEBUG', 'f'))
 
@@ -34,21 +63,9 @@ IOFORMATS = {
     IOFORMAT_KWARGS,
 }
 
-IOFORMAT = os.environ.get('IOFORMAT', IOFORMAT_KWARGS)
+IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__)
-
-
-def validate_io_format(v):
-    if callable(v):
-        return v
-    if v in IOFORMATS:
-        return v
-    raise ValidationError('Unsupported format {!r}.'.format(v))
 
 
 def check():
     if DEBUG and QUIET:
         raise RuntimeError('I cannot be verbose and quiet at the same time.')
-
-    if IOFORMAT not in IOFORMATS:
-        raise RuntimeError('Invalid default input/output format.')
-

bonobo/util/iterators.py

@@ -21,7 +21,7 @@ def force_iterator(mixed):
 def ensure_tuple(tuple_or_mixed):
     if isinstance(tuple_or_mixed, tuple):
         return tuple_or_mixed
-    return (tuple_or_mixed,)
+    return (tuple_or_mixed, )
 
 
 def tuplize(generator):

requirements-dev.txt

@@ -4,7 +4,7 @@ arrow==0.10.0
 babel==2.4.0
 binaryornot==0.4.3
 certifi==2017.4.17
-chardet==3.0.3
+chardet==3.0.4
 click==6.7
 cookiecutter==1.5.1
 coverage==4.4.1
@@ -19,6 +19,7 @@ poyo==0.4.1
 py==1.4.34
 pygments==2.2.0
 pytest-cov==2.5.1
+pytest-sugar==0.8.0
 pytest-timeout==1.2.0
 pytest==3.1.1
 python-dateutil==2.6.0
@@ -28,5 +29,6 @@ six==1.10.0
 snowballstemmer==1.2.1
 sphinx==1.6.2
 sphinxcontrib-websupport==1.0.1
+termcolor==1.1.0
 urllib3==1.21.1
 whichcraft==0.4.1

requirements-docker.txt

@@ -1,9 +1,9 @@
 -e .[docker]
 appdirs==1.4.3
-bonobo-docker==0.2.4
+bonobo-docker==0.2.5
 bonobo==0.3.1
 certifi==2017.4.17
-chardet==3.0.3
+chardet==3.0.4
 colorama==0.3.9
 docker-pycreds==0.2.1
 docker==2.3.0
@@ -17,6 +17,6 @@ pyparsing==2.2.0
 pytz==2017.2
 requests==2.17.3
 six==1.10.0
-stevedore==1.22.0
+stevedore==1.23.0
 urllib3==1.21.1
 websocket-client==0.40.0

requirements-jupyter.txt

@@ -2,7 +2,7 @@
 appnope==0.1.0
 bleach==2.0.0
 decorator==4.0.11
-entrypoints==0.2.2
+entrypoints==0.2.3
 html5lib==0.999999999
 ipykernel==4.6.1
 ipython-genutils==0.2.0

requirements.txt

@@ -1,7 +1,7 @@
 -e .
 appdirs==1.4.3
 certifi==2017.4.17
-chardet==3.0.3
+chardet==3.0.4
 colorama==0.3.9
 enum34==1.1.6
 fs==2.0.3
@@ -13,5 +13,5 @@ pyparsing==2.2.0
 pytz==2017.2
 requests==2.17.3
 six==1.10.0
-stevedore==1.22.0
+stevedore==1.23.0
 urllib3==1.21.1

setup.py

@@ -59,7 +59,8 @@ setup(
     extras_require={
         'dev': [
             'cookiecutter (>= 1.5, < 1.6)', 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)',
-            'pytest-cov (>= 2.5, < 3.0)', 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)'
+            'pytest-cov (>= 2.5, < 3.0)', 'pytest-sugar (>= 0.8, < 0.9)', 'pytest-timeout (>= 1, < 2)',
+            'sphinx (>= 1.6, < 2.0)'
         ],
         'docker': ['bonobo-docker'],
         'jupyter': ['ipywidgets (>= 6.0.0, < 7)', 'jupyter (>= 1.0, < 1.1)']

tests/io/test_csv.py

@@ -1,6 +1,6 @@
 import pytest
 
-from bonobo import Bag, CsvReader, CsvWriter, open_fs
+from bonobo import Bag, CsvReader, CsvWriter, open_fs, settings
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
 from bonobo.util.testing import CapturingNodeExecutionContext
@@ -9,7 +9,7 @@ from bonobo.util.testing import CapturingNodeExecutionContext
 def test_write_csv_to_file(tmpdir):
     fs, filename = open_fs(tmpdir), 'output.csv'
 
-    writer = CsvWriter(path=filename)
+    writer = CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0)
     context = NodeExecutionContext(writer, services={'fs': fs})
 
     context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
@@ -19,7 +19,7 @@ def test_write_csv_to_file(tmpdir):
     context.step()
     context.stop()
 
-    with fs.open(filename)as fp:
+    with fs.open(filename) as fp:
         assert fp.read() == 'foo\nbar\nbaz\n'
 
     with pytest.raises(AttributeError):
@@ -31,7 +31,7 @@ def test_read_csv_from_file(tmpdir):
     with fs.open(filename, 'w') as fp:
         fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar')
 
-    reader = CsvReader(path=filename, delimiter=',')
+    reader = CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0)
 
     context = CapturingNodeExecutionContext(reader, services={'fs': fs})
 
@@ -64,7 +64,7 @@ def test_read_csv_kwargs_output_formater(tmpdir):
     with fs.open(filename, 'w') as fp:
         fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar')
 
-    reader = CsvReader(path=filename, delimiter=',', output_format='kwargs')
+    reader = CsvReader(path=filename, delimiter=',')
 
     context = CapturingNodeExecutionContext(reader, services={'fs': fs})
 

tests/io/test_json.py

@@ -1,6 +1,6 @@
 import pytest
 
-from bonobo import Bag, JsonReader, JsonWriter, open_fs
+from bonobo import Bag, JsonReader, JsonWriter, open_fs, settings
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
 from bonobo.util.testing import CapturingNodeExecutionContext
@@ -9,7 +9,7 @@ from bonobo.util.testing import CapturingNodeExecutionContext
 def test_write_json_to_file(tmpdir):
     fs, filename = open_fs(tmpdir), 'output.json'
 
-    writer = JsonWriter(path=filename)
+    writer = JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0)
     context = NodeExecutionContext(writer, services={'fs': fs})
 
     context.start()
@@ -31,7 +31,7 @@ def test_read_json_from_file(tmpdir):
     fs, filename = open_fs(tmpdir), 'input.json'
     with fs.open(filename, 'w') as fp:
         fp.write('[{"x": "foo"},{"x": "bar"}]')
-    reader = JsonReader(path=filename)
+    reader = JsonReader(filename, ioformat=settings.IOFORMAT_ARG0)
 
     context = CapturingNodeExecutionContext(reader, services={'fs': fs})
 

tests/io/test_pickle.py

@@ -1,7 +1,8 @@
 import pickle
+
 import pytest
 
-from bonobo import Bag, PickleReader, PickleWriter, open_fs
+from bonobo import Bag, PickleReader, PickleWriter, open_fs, settings
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
 from bonobo.util.testing import CapturingNodeExecutionContext
@@ -10,7 +11,7 @@ from bonobo.util.testing import CapturingNodeExecutionContext
 def test_write_pickled_dict_to_file(tmpdir):
     fs, filename = open_fs(tmpdir), 'output.pkl'
 
-    writer = PickleWriter(path=filename)
+    writer = PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0)
     context = NodeExecutionContext(writer, services={'fs': fs})
 
     context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
@@ -32,7 +33,7 @@ def test_read_pickled_list_from_file(tmpdir):
     with fs.open(filename, 'wb') as fp:
         fp.write(pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']]))
 
-    reader = PickleReader(path=filename)
+    reader = PickleReader(filename, ioformat=settings.IOFORMAT_ARG0)
 
     context = CapturingNodeExecutionContext(reader, services={'fs': fs})