[core] Refactoring IOFormats so there is one and only obvious way to send it.

This is the commit where I admit that having more than one input/output
format for readers and writers was complicating the code too much for a
very small gain, and that it would be easier to only have one way to do
it.

So such way is now:

- Returning (or yielding) a dict if you have key-value type collections.
- Returning (or yielding) a tuple if you have a list-type collection.
- Returning (or yielding) something else otherwise, which will continue
  to work like the old "arg0" format.

IOFORMAT options has been removed in favour of a RemovedOption, which
will complain if you're still trying to set it to anything else than the
one value allowed.
This commit is contained in:
Romain Dorgueil
2017-10-15 21:37:22 +02:00
parent dc59c88c3d
commit 92cc400fe7
27 changed files with 427 additions and 269 deletions

View File

@ -3,25 +3,19 @@ import pytest
from bonobo import Bag, CsvReader, CsvWriter, settings
from bonobo.constants import BEGIN, END
from bonobo.execution.node import NodeExecutionContext
from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext
csv_tester = FilesystemTester('csv')
csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar'
def test_write_csv_to_file_arg0(tmpdir):
def test_write_csv_ioformat_arg0(tmpdir):
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
with pytest.raises(ValueError):
CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0)
with NodeExecutionContext(CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
context.step()
context.step()
with fs.open(filename) as fp:
assert fp.read() == 'foo\nbar\nbaz\n'
with pytest.raises(AttributeError):
getattr(context, 'file')
with pytest.raises(ValueError):
CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0),
@pytest.mark.parametrize('add_kwargs', ({}, {
@ -30,7 +24,7 @@ def test_write_csv_to_file_arg0(tmpdir):
def test_write_csv_to_file_kwargs(tmpdir, add_kwargs):
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
with NodeExecutionContext(CsvWriter(path=filename, **add_kwargs), services=services) as context:
with NodeExecutionContext(CsvWriter(filename, **add_kwargs), services=services) as context:
context.write(BEGIN, Bag(**{'foo': 'bar'}), Bag(**{'foo': 'baz', 'ignore': 'this'}), END)
context.step()
context.step()
@ -42,61 +36,24 @@ def test_write_csv_to_file_kwargs(tmpdir, add_kwargs):
getattr(context, 'file')
def test_read_csv_from_file_arg0(tmpdir):
fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
with CapturingNodeExecutionContext(
CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0),
services=services,
) as context:
context.write(BEGIN, Bag(), END)
context.step()
assert len(context.send.mock_calls) == 2
args0, kwargs0 = context.send.call_args_list[0]
assert len(args0) == 1 and not len(kwargs0)
args1, kwargs1 = context.send.call_args_list[1]
assert len(args1) == 1 and not len(kwargs1)
assert args0[0].args[0] == {
'a': 'a foo',
'b': 'b foo',
'c': 'c foo',
}
assert args1[0].args[0] == {
'a': 'a bar',
'b': 'b bar',
'c': 'c bar',
}
def test_read_csv_from_file_kwargs(tmpdir):
fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
with CapturingNodeExecutionContext(
with BufferingNodeExecutionContext(
CsvReader(path=filename, delimiter=','),
services=services,
) as context:
context.write(BEGIN, Bag(), END)
context.step()
output = context.get_buffer()
assert len(context.send.mock_calls) == 2
args0, kwargs0 = context.send.call_args_list[0]
assert len(args0) == 1 and not len(kwargs0)
args1, kwargs1 = context.send.call_args_list[1]
assert len(args1) == 1 and not len(kwargs1)
_args, _kwargs = args0[0].get()
assert not len(_args) and _kwargs == {
assert len(output) == 2
assert output[0] == {
'a': 'a foo',
'b': 'b foo',
'c': 'c foo',
}
_args, _kwargs = args1[0].get()
assert not len(_args) and _kwargs == {
assert output[1] == {
'a': 'a bar',
'b': 'b bar',
'c': 'c bar',