paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: new alternate syntax and switch to black + isort (yeah, maybe not the best time, but that is done).

Browse Source
This commit is contained in:
Romain Dorgueil
2018-07-29 18:21:56 +01:00
parent 3094e43f9f
commit 89dda0dca6
123 changed files with 1672 additions and 1640 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
tests/nodes/io/test_csv.py
View File

@ -6,13 +6,14 @@ import pytest
from bonobo import CsvReader, CsvWriter
from bonobo.constants import EMPTY
from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext, WriterTest, ConfigurableNodeTest, \
ReaderTest
from bonobo.util.testing import (
BufferingNodeExecutionContext, ConfigurableNodeTest, FilesystemTester, ReaderTest, WriterTest
)
csv_tester = FilesystemTester('csv')
csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar'
csv_tester = FilesystemTester("csv")
csv_tester.input_data = "a,b,c\na foo,b foo,c foo\na bar,b bar,c bar"
defaults = {'lineterminator': '\n'}
defaults = {"lineterminator": "\n"}
incontext = ConfigurableNodeTest.incontext
@ -23,15 +24,10 @@ def test_read_csv_from_file_kwargs(tmpdir):
with BufferingNodeExecutionContext(CsvReader(filename, **defaults), services=services) as context:
context.write_sync(EMPTY)
assert context.get_buffer_args_as_dicts() == [{
'a': 'a foo',
'b': 'b foo',
'c': 'c foo',
}, {
'a': 'a bar',
'b': 'b bar',
'c': 'c bar',
}]
assert context.get_buffer_args_as_dicts() == [
{"a": "a foo", "b": "b foo", "c": "c foo"},
{"a": "a bar", "b": "b bar", "c": "c bar"},
]
###
@ -40,86 +36,66 @@ def test_read_csv_from_file_kwargs(tmpdir):
class Csv:
extension = 'csv'
extension = "csv"
ReaderNodeType = CsvReader
WriterNodeType = CsvWriter
L1, L2, L3, L4 = ('a', 'hey'), ('b', 'bee'), ('c', 'see'), ('d', 'dee')
LL = ('i', 'have', 'more', 'values')
L1, L2, L3, L4 = ("a", "hey"), ("b", "bee"), ("c", "see"), ("d", "dee")
LL = ("i", "have", "more", "values")
class CsvReaderTest(Csv, ReaderTest, TestCase):
input_data = '\n'.join((
'id,name',
'1,John Doe',
'2,Jane Doe',
',DPR',
'42,Elon Musk',
))
input_data = "\n".join(("id,name", "1,John Doe", "2,Jane Doe", ",DPR", "42,Elon Musk"))
def check_output(self, context, *, prepend=None):
out = context.get_buffer()
assert out == (prepend or list()) + [
('1', 'John Doe'),
('2', 'Jane Doe'),
('', 'DPR'),
('42', 'Elon Musk'),
]
assert out == (prepend or list()) + [("1", "John Doe"), ("2", "Jane Doe"), ("", "DPR"), ("42", "Elon Musk")]
@incontext()
def test_nofields(self, context):
context.write_sync(EMPTY)
context.stop()
self.check_output(context)
assert context.get_output_fields() == ('id', 'name')
assert context.get_output_fields() == ("id", "name")
@incontext(output_type=tuple)
def test_output_type(self, context):
context.write_sync(EMPTY)
context.stop()
self.check_output(context, prepend=[('id', 'name')])
self.check_output(context, prepend=[("id", "name")])
@incontext(
output_fields=(
'x',
'y',
), skip=1
)
@incontext(output_fields=("x", "y"), skip=1)
def test_output_fields(self, context):
context.write_sync(EMPTY)
context.stop()
self.check_output(context)
assert context.get_output_fields() == ('x', 'y')
assert context.get_output_fields() == ("x", "y")
@incontext(quoting=QUOTE_ALL)
def test_quoting(self, context):
context.write_sync(EMPTY)
context.stop()
self.check_output(context)
assert context.get_output_fields() == ('id', 'name')
assert context.get_output_fields() == ("id", "name")
class CsvWriterTest(Csv, WriterTest, TestCase):
@incontext()
def test_fields(self, context):
context.set_input_fields(['foo', 'bar'])
context.write_sync(('a', 'b'), ('c', 'd'))
context.set_input_fields(["foo", "bar"])
context.write_sync(("a", "b"), ("c", "d"))
context.stop()
assert self.readlines() == (
'foo,bar',
'a,b',
'c,d',
)
assert self.readlines() == ("foo,bar", "a,b", "c,d")
@incontext()
def test_fields_from_type(self, context):
context.set_input_type(namedtuple('Point', 'x y'))
context.set_input_type(namedtuple("Point", "x y"))
context.write_sync((1, 2), (3, 4))
context.stop()
assert self.readlines() == ('x,y', '1,2', '3,4')
assert self.readlines() == ("x,y", "1,2", "3,4")
@incontext()
def test_nofields_multiple_args(self, context):
@ -127,30 +103,21 @@ class CsvWriterTest(Csv, WriterTest, TestCase):
context.write_sync((L1, L2), (L3, L4))
context.stop()
assert self.readlines() == (
'a,hey',
'b,bee',
'c,see',
'd,dee',
)
assert self.readlines() == ("a,hey", "b,bee", "c,see", "d,dee")
@incontext()
def test_nofields_multiple_args_length_mismatch(self, context):
# if length of input vary, then we get a TypeError (unrecoverable)
with pytest.raises(TypeError):
context.write_sync((L1, L2), (L3, ))
context.write_sync((L1, L2), (L3,))
@incontext()
def test_nofields_single_arg(self, context):
# single args are just dumped, shapes can vary.
context.write_sync((L1, ), (LL, ), (L3, ))
context.write_sync((L1,), (LL,), (L3,))
context.stop()
assert self.readlines() == (
'a,hey',
'i,have,more,values',
'c,see',
)
assert self.readlines() == ("a,hey", "i,have,more,values", "c,see")
@incontext()
def test_nofields_empty_args(self, context):

19
tests/nodes/io/test_file.py
View File

@ -5,26 +5,23 @@ from bonobo.constants import EMPTY
from bonobo.execution.contexts.node import NodeExecutionContext
from bonobo.util.testing import BufferingNodeExecutionContext, FilesystemTester
txt_tester = FilesystemTester('txt')
txt_tester.input_data = 'Hello\nWorld\n'
txt_tester = FilesystemTester("txt")
txt_tester.input_data = "Hello\nWorld\n"
def test_file_writer_contextless(tmpdir):
fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
with FileWriter(path=filename).open(fs) as fp:
fp.write('Yosh!')
fp.write("Yosh!")
with fs.open(filename) as fp:
assert fp.read() == 'Yosh!'
assert fp.read() == "Yosh!"
@pytest.mark.parametrize(
'lines,output',
[
(('ACME', ), 'ACME'), # one line...
(('Foo', 'Bar', 'Baz'), 'Foo\nBar\nBaz'), # more than one line...
]
"lines,output",
[(("ACME",), "ACME"), (("Foo", "Bar", "Baz"), "Foo\nBar\nBaz")], # one line... # more than one line...
)
def test_file_writer_in_context(tmpdir, lines, output):
fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
@ -44,5 +41,5 @@ def test_file_reader(tmpdir):
output = context.get_buffer()
assert len(output) == 2
assert output[0] == ('Hello', )
assert output[1] == ('World', )
assert output[0] == ("Hello",)
assert output[1] == ("World",)

126
tests/nodes/io/test_json.py
View File

@ -4,14 +4,13 @@ from unittest import TestCase
import pytest
from bonobo import JsonReader, JsonWriter
from bonobo import LdjsonReader, LdjsonWriter
from bonobo import JsonReader, JsonWriter, LdjsonReader, LdjsonWriter
from bonobo.constants import EMPTY
from bonobo.util.testing import WriterTest, ReaderTest, ConfigurableNodeTest
from bonobo.util.testing import ConfigurableNodeTest, ReaderTest, WriterTest
FOOBAR = {'foo': 'bar'}
OD_ABC = OrderedDict((('a', 'A'), ('b', 'B'), ('c', 'C')))
FOOBAZ = {'foo': 'baz'}
FOOBAR = {"foo": "bar"}
OD_ABC = OrderedDict((("a", "A"), ("b", "B"), ("c", "C")))
FOOBAZ = {"foo": "baz"}
incontext = ConfigurableNodeTest.incontext
@ -21,7 +20,7 @@ incontext = ConfigurableNodeTest.incontext
class Json:
extension = 'json'
extension = "json"
ReaderNodeType = JsonReader
WriterNodeType = JsonWriter
@ -34,88 +33,61 @@ class JsonReaderDictsTest(Json, ReaderTest, TestCase):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
({
"foo": "bar"
}, ),
({
"baz": "boz"
}, ),
]
assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)]
class JsonReaderListsTest(Json, ReaderTest, TestCase):
input_data = '[[1,2,3],\n[4,5,6]]'
input_data = "[[1,2,3],\n[4,5,6]]"
@incontext()
def test_nofields(self, context):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
([1, 2, 3], ),
([4, 5, 6], ),
]
assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
@incontext(output_type=tuple)
def test_output_type(self, context):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
([1, 2, 3], ),
([4, 5, 6], ),
]
assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
class JsonReaderStringsTest(Json, ReaderTest, TestCase):
input_data = '[' + ',\n'.join(map(json.dumps, ('foo', 'bar', 'baz'))) + ']'
input_data = "[" + ",\n".join(map(json.dumps, ("foo", "bar", "baz"))) + "]"
@incontext()
def test_nofields(self, context):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
('foo', ),
('bar', ),
('baz', ),
]
assert context.get_buffer() == [("foo",), ("bar",), ("baz",)]
@incontext(output_type=tuple)
def test_output_type(self, context):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
('foo', ),
('bar', ),
('baz', ),
]
assert context.get_buffer() == [("foo",), ("bar",), ("baz",)]
class JsonWriterTest(Json, WriterTest, TestCase):
@incontext()
def test_fields(self, context):
context.set_input_fields(['foo', 'bar'])
context.write_sync(('a', 'b'), ('c', 'd'))
context.set_input_fields(["foo", "bar"])
context.write_sync(("a", "b"), ("c", "d"))
context.stop()
assert self.readlines() == (
'[{"foo": "a", "bar": "b"},',
'{"foo": "c", "bar": "d"}]',
)
assert self.readlines() == ('[{"foo": "a", "bar": "b"},', '{"foo": "c", "bar": "d"}]')
@incontext()
def test_fields_from_type(self, context):
context.set_input_type(namedtuple('Point', 'x y'))
context.set_input_type(namedtuple("Point", "x y"))
context.write_sync((1, 2), (3, 4))
context.stop()
assert self.readlines() == (
'[{"x": 1, "y": 2},',
'{"x": 3, "y": 4}]',
)
assert self.readlines() == ('[{"x": 1, "y": 2},', '{"x": 3, "y": 4}]')
@incontext()
def test_nofields_multiple_args(self, context):
@ -144,11 +116,7 @@ class JsonWriterTest(Json, WriterTest, TestCase):
context.write_sync(FOOBAR, OD_ABC, FOOBAZ)
context.stop()
assert self.readlines() == (
'[{"foo": "bar"},',
'{"a": "A", "b": "B", "c": "C"},',
'{"foo": "baz"}]',
)
assert self.readlines() == ('[{"foo": "bar"},', '{"a": "A", "b": "B", "c": "C"},', '{"foo": "baz"}]')
@incontext()
def test_nofields_empty_args(self, context):
@ -156,7 +124,7 @@ class JsonWriterTest(Json, WriterTest, TestCase):
context.write_sync(EMPTY, EMPTY, EMPTY)
context.stop()
assert self.readlines() == ('[]', )
assert self.readlines() == ("[]",)
###
@ -165,7 +133,7 @@ class JsonWriterTest(Json, WriterTest, TestCase):
class Ldjson:
extension = 'ldjson'
extension = "ldjson"
ReaderNodeType = LdjsonReader
WriterNodeType = LdjsonWriter
@ -178,85 +146,61 @@ class LdjsonReaderDictsTest(Ldjson, ReaderTest, TestCase):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
({
"foo": "bar"
}, ),
({
"baz": "boz"
}, ),
]
assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)]
class LdjsonReaderListsTest(Ldjson, ReaderTest, TestCase):
input_data = '[1,2,3]\n[4,5,6]'
input_data = "[1,2,3]\n[4,5,6]"
@incontext()
def test_nofields(self, context):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
([1, 2, 3], ),
([4, 5, 6], ),
]
assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
@incontext(output_type=tuple)
def test_output_type(self, context):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
([1, 2, 3], ),
([4, 5, 6], ),
]
assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
class LdjsonReaderStringsTest(Ldjson, ReaderTest, TestCase):
input_data = '\n'.join(map(json.dumps, ('foo', 'bar', 'baz')))
input_data = "\n".join(map(json.dumps, ("foo", "bar", "baz")))
@incontext()
def test_nofields(self, context):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
('foo', ),
('bar', ),
('baz', ),
]
assert context.get_buffer() == [("foo",), ("bar",), ("baz",)]
@incontext(output_type=tuple)
def test_output_type(self, context):
context.write_sync(EMPTY)
context.stop()
assert context.get_buffer() == [
('foo', ),
('bar', ),
('baz', ),
]
assert context.get_buffer() == [("foo",), ("bar",), ("baz",)]
class LdjsonWriterTest(Ldjson, WriterTest, TestCase):
@incontext()
def test_fields(self, context):
context.set_input_fields(['foo', 'bar'])
context.write_sync(('a', 'b'), ('c', 'd'))
context.set_input_fields(["foo", "bar"])
context.write_sync(("a", "b"), ("c", "d"))
context.stop()
assert self.readlines() == ('{"foo": "a", "bar": "b"}', '{"foo": "c", "bar": "d"}')
@incontext()
def test_fields_from_type(self, context):
context.set_input_type(namedtuple('Point', 'x y'))
context.set_input_type(namedtuple("Point", "x y"))
context.write_sync((1, 2), (3, 4))
context.stop()
assert self.readlines() == (
'{"x": 1, "y": 2}',
'{"x": 3, "y": 4}',
)
assert self.readlines() == ('{"x": 1, "y": 2}', '{"x": 3, "y": 4}')
@incontext()
def test_nofields_multiple_args(self, context):
@ -285,11 +229,7 @@ class LdjsonWriterTest(Ldjson, WriterTest, TestCase):
context.write_sync(FOOBAR, OD_ABC, FOOBAZ)
context.stop()
assert self.readlines() == (
'{"foo": "bar"}',
'{"a": "A", "b": "B", "c": "C"}',
'{"foo": "baz"}',
)
assert self.readlines() == ('{"foo": "bar"}', '{"a": "A", "b": "B", "c": "C"}', '{"foo": "baz"}')
@incontext()
def test_nofields_empty_args(self, context):

19
tests/nodes/io/test_pickle.py
View File

@ -7,21 +7,21 @@ from bonobo.constants import EMPTY
from bonobo.execution.contexts.node import NodeExecutionContext
from bonobo.util.testing import BufferingNodeExecutionContext, FilesystemTester
pickle_tester = FilesystemTester('pkl', mode='wb')
pickle_tester.input_data = pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']])
pickle_tester = FilesystemTester("pkl", mode="wb")
pickle_tester.input_data = pickle.dumps([["a", "b", "c"], ["a foo", "b foo", "c foo"], ["a bar", "b bar", "c bar"]])
def test_write_pickled_dict_to_file(tmpdir):
fs, filename, services = pickle_tester.get_services_for_writer(tmpdir)
with NodeExecutionContext(PickleWriter(filename), services=services) as context:
context.write_sync({'foo': 'bar'}, {'foo': 'baz', 'ignore': 'this'})
context.write_sync({"foo": "bar"}, {"foo": "baz", "ignore": "this"})
with fs.open(filename, 'rb') as fp:
assert pickle.loads(fp.read()) == {'foo': 'bar'}
with fs.open(filename, "rb") as fp:
assert pickle.loads(fp.read()) == {"foo": "bar"}
with pytest.raises(AttributeError):
getattr(context, 'file')
getattr(context, "file")
def test_read_pickled_list_from_file(tmpdir):
@ -31,8 +31,5 @@ def test_read_pickled_list_from_file(tmpdir):
context.write_sync(EMPTY)
output = context.get_buffer()
assert context.get_output_fields() == ('a', 'b', 'c')
assert output == [
('a foo', 'b foo', 'c foo'),
('a bar', 'b bar', 'c bar'),
]
assert context.get_output_fields() == ("a", "b", "c")
assert output == [("a foo", "b foo", "c foo"), ("a bar", "b bar", "c bar")]

31
tests/nodes/test_basics.py
View File

@ -5,9 +5,9 @@ from unittest.mock import MagicMock
import pytest
import bonobo
from bonobo.constants import NOT_MODIFIED, EMPTY
from bonobo.util import ensure_tuple, ValueHolder
from bonobo.util.testing import BufferingNodeExecutionContext, StaticNodeTest, ConfigurableNodeTest
from bonobo.constants import EMPTY, NOT_MODIFIED
from bonobo.util import ValueHolder, ensure_tuple
from bonobo.util.testing import BufferingNodeExecutionContext, ConfigurableNodeTest, StaticNodeTest
class CountTest(StaticNodeTest, TestCase):
@ -26,7 +26,7 @@ class CountTest(StaticNodeTest, TestCase):
def test_execution(self):
with self.execute() as context:
context.write_sync(*([EMPTY] * 42))
assert context.get_buffer() == [(42, )]
assert context.get_buffer() == [(42,)]
class IdentityTest(StaticNodeTest, TestCase):
@ -81,14 +81,14 @@ def test_tee():
tee = bonobo.Tee(inner)
results = []
for i in range(10):
results.append(tee('foo'))
results.append(tee("foo"))
assert results == [NOT_MODIFIED] * 10
assert len(inner.mock_calls) == 10
def test_noop():
assert bonobo.noop(1, 2, 3, 4, foo='bar') == NOT_MODIFIED
assert bonobo.noop(1, 2, 3, 4, foo="bar") == NOT_MODIFIED
def test_fixedwindow():
@ -98,21 +98,18 @@ def test_fixedwindow():
with BufferingNodeExecutionContext(bonobo.FixedWindow(2)) as context:
context.write_sync(*range(9))
assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), (
8,
None,
)]
assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), (8, None)]
with BufferingNodeExecutionContext(bonobo.FixedWindow(1)) as context:
context.write_sync(*range(3))
assert context.get_buffer() == [(0, ), (1, ), (2, )]
assert context.get_buffer() == [(0,), (1,), (2,)]
def test_methodcaller():
with BufferingNodeExecutionContext(methodcaller('swapcase')) as context:
context.write_sync('aaa', 'bBb', 'CcC')
assert context.get_buffer() == list(map(ensure_tuple, ['AAA', 'BbB', 'cCc']))
with BufferingNodeExecutionContext(methodcaller("swapcase")) as context:
context.write_sync("aaa", "bBb", "CcC")
assert context.get_buffer() == list(map(ensure_tuple, ["AAA", "BbB", "cCc"]))
with BufferingNodeExecutionContext(methodcaller('zfill', 5)) as context:
context.write_sync('a', 'bb', 'ccc')
assert context.get_buffer() == list(map(ensure_tuple, ['0000a', '000bb', '00ccc']))
with BufferingNodeExecutionContext(methodcaller("zfill", 5)) as context:
context.write_sync("a", "bb", "ccc")
assert context.get_buffer() == list(map(ensure_tuple, ["0000a", "000bb", "00ccc"]))

29
tests/nodes/test_casts.py
View File

@ -8,11 +8,11 @@ from bonobo.util.bags import BagType
from bonobo.util.envelopes import Envelope
from bonobo.util.testing import BufferingNodeExecutionContext
MyTuple = namedtuple('MyTuple', ['a', 'b', 'c'])
MyBag = BagType('MyBag', ['a', 'b', 'c'])
MyTuple = namedtuple("MyTuple", ["a", "b", "c"])
MyBag = BagType("MyBag", ["a", "b", "c"])
class MyCustomType():
class MyCustomType:
def __init__(self, *args):
self.args = args
@ -20,16 +20,19 @@ class MyCustomType():
return MyBag(*self.args)
@pytest.mark.parametrize(['factory', 'expected', 'expected_item0'], [
[lambda: (1, 2, 3), tuple, int],
[lambda: Envelope((1, 2, 3)), tuple, int],
[lambda: MyTuple(1, 2, 3), MyTuple, int],
[lambda: Envelope(MyTuple(1, 2, 3)), MyTuple, int],
[lambda: MyBag(1, 2, 3), MyBag, int],
[lambda: Envelope(MyBag(1, 2, 3)), MyBag, int],
[lambda: MyCustomType(1, 2, 3), tuple, MyCustomType],
[lambda: Envelope(MyCustomType(1, 2, 3)), tuple, MyCustomType],
])
@pytest.mark.parametrize(
["factory", "expected", "expected_item0"],
[
[lambda: (1, 2, 3), tuple, int],
[lambda: Envelope((1, 2, 3)), tuple, int],
[lambda: MyTuple(1, 2, 3), MyTuple, int],
[lambda: Envelope(MyTuple(1, 2, 3)), MyTuple, int],
[lambda: MyBag(1, 2, 3), MyBag, int],
[lambda: Envelope(MyBag(1, 2, 3)), MyBag, int],
[lambda: MyCustomType(1, 2, 3), tuple, MyCustomType],
[lambda: Envelope(MyCustomType(1, 2, 3)), tuple, MyCustomType],
],
)
def test_casts_after_output(factory: Callable, expected, expected_item0):
def transform():
yield factory()