Rewritting Bags from scratch using a namedtuple approach, along with other (less major) updates.
New bag implementation improves a lot how bonobo works, even if this is highly backward incompatible (sorry, that's needed, and better sooner than later). * New implementation uses the same approach as python's namedtuple, by dynamically creating the python type's code. This has drawbacks, as it feels like not the right way, but also a lot of benefits that cannot be achieved using a regular approach, especially the constructor parameter order, hardcoded. * Memory usage is now much more efficient. The "keys" memory space will be used only once per "io type", being spent in the underlying type definition instead of in the actual instances. * Transformations now needs to use tuples as output, which will be bound to its "output type". The output type can be infered from the tuple length, or explicitely set by the user using either `context.set_output_type(...)` or `context.set_output_fields(...)` (to build a bag type from a list of field names). Jupyter/Graphviz integration is more tight, allowing to easily display graphs in a notebook, or displaying the live transformation status in an html table instead of a simple <div>. For now, context processors were hacked to stay working as before but the current API is not satisfactory, and should be replaced. This new big change being unreasonable without some time to work on it properly, it is postponed for next versions (0.7, 0.8, ...). Maybe the best idea is to have some kind of "local services", that would use the same dependency injection mechanism as the execution-wide services. Services are now passed by keywoerd arguments only, to avoid confusion with data-arguments.
This commit is contained in:
Romain Dorgueil
@ -1,3 +1,5 @@
|
||||
import pprint
|
||||
|
||||
import pytest
|
||||
|
||||
from bonobo.config.configurables import Configurable
|
||||
|
||||
@ -7,7 +7,7 @@ class MethodBasedConfigurable(Configurable):
|
||||
foo = Option(positional=True)
|
||||
bar = Option()
|
||||
|
||||
def call(self, *args, **kwargs):
|
||||
def __call__(self, *args, **kwargs):
|
||||
self.handler(*args, **kwargs)
|
||||
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ class Bobby(Configurable):
|
||||
def think(self, context):
|
||||
yield 'different'
|
||||
|
||||
def call(self, think, *args, **kwargs):
|
||||
def __call__(self, think, *args, **kwargs):
|
||||
self.handler('1', *args, **kwargs)
|
||||
self.handler2('2', *args, **kwargs)
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
from operator import attrgetter
|
||||
|
||||
from bonobo.config import Configurable
|
||||
from bonobo.config.processors import ContextProcessor, resolve_processors, ContextCurrifier
|
||||
from bonobo.config.processors import ContextProcessor, resolve_processors, ContextCurrifier, use_context_processor
|
||||
|
||||
|
||||
class CP1(Configurable):
|
||||
@ -59,5 +59,16 @@ def test_setup_teardown():
|
||||
o = CP1()
|
||||
stack = ContextCurrifier(o)
|
||||
stack.setup()
|
||||
assert o(*stack.context) == ('this is A', 'THIS IS b')
|
||||
assert o(*stack.args) == ('this is A', 'THIS IS b')
|
||||
stack.teardown()
|
||||
|
||||
|
||||
def test_processors_on_func():
|
||||
def cp(context):
|
||||
yield context
|
||||
|
||||
@use_context_processor(cp)
|
||||
def node(context):
|
||||
pass
|
||||
|
||||
assert get_all_processors_names(node) == ['cp']
|
||||
|
||||
@ -3,9 +3,9 @@ import time
|
||||
|
||||
import pytest
|
||||
|
||||
from bonobo.util import get_name
|
||||
from bonobo.config import Configurable, Container, Exclusive, Service, requires
|
||||
from bonobo.config import Configurable, Container, Exclusive, Service, use
|
||||
from bonobo.config.services import validate_service_name, create_container
|
||||
from bonobo.util import get_name
|
||||
|
||||
|
||||
class PrinterInterface():
|
||||
@ -30,7 +30,7 @@ SERVICES = Container(
|
||||
class MyServiceDependantConfigurable(Configurable):
|
||||
printer = Service(PrinterInterface, )
|
||||
|
||||
def __call__(self, printer: PrinterInterface, *args):
|
||||
def __call__(self, *args, printer: PrinterInterface):
|
||||
return printer.print(*args)
|
||||
|
||||
|
||||
@ -51,15 +51,15 @@ def test_service_name_validator():
|
||||
def test_service_dependency():
|
||||
o = MyServiceDependantConfigurable(printer='printer0')
|
||||
|
||||
assert o(SERVICES.get('printer0'), 'foo', 'bar') == '0;foo;bar'
|
||||
assert o(SERVICES.get('printer1'), 'bar', 'baz') == '1;bar;baz'
|
||||
assert o(*SERVICES.args_for(o), 'foo', 'bar') == '0;foo;bar'
|
||||
assert o('foo', 'bar', printer=SERVICES.get('printer0')) == '0;foo;bar'
|
||||
assert o('bar', 'baz', printer=SERVICES.get('printer1')) == '1;bar;baz'
|
||||
assert o('foo', 'bar', **SERVICES.kwargs_for(o)) == '0;foo;bar'
|
||||
|
||||
|
||||
def test_service_dependency_unavailable():
|
||||
o = MyServiceDependantConfigurable(printer='printer2')
|
||||
with pytest.raises(KeyError):
|
||||
SERVICES.args_for(o)
|
||||
SERVICES.kwargs_for(o)
|
||||
|
||||
|
||||
class VCR:
|
||||
@ -100,13 +100,13 @@ def test_requires():
|
||||
|
||||
services = Container(output=vcr.append)
|
||||
|
||||
@requires('output')
|
||||
@use('output')
|
||||
def append(out, x):
|
||||
out(x)
|
||||
|
||||
svcargs = services.args_for(append)
|
||||
svcargs = services.kwargs_for(append)
|
||||
assert len(svcargs) == 1
|
||||
assert svcargs[0] == vcr.append
|
||||
assert svcargs['output'] == vcr.append
|
||||
|
||||
|
||||
@pytest.mark.parametrize('services', [None, {}])
|
||||
|
||||
@ -2,7 +2,8 @@ from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from bonobo import Bag, Graph
|
||||
from bonobo import Graph
|
||||
from bonobo.constants import EMPTY
|
||||
from bonobo.execution.contexts.node import NodeExecutionContext
|
||||
from bonobo.execution.strategies import NaiveStrategy
|
||||
from bonobo.util.testing import BufferingNodeExecutionContext, BufferingGraphExecutionContext
|
||||
@ -13,23 +14,23 @@ def test_node_string():
|
||||
return 'foo'
|
||||
|
||||
with BufferingNodeExecutionContext(f) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 1
|
||||
assert output[0] == 'foo'
|
||||
assert output[0] == ('foo', )
|
||||
|
||||
def g():
|
||||
yield 'foo'
|
||||
yield 'bar'
|
||||
|
||||
with BufferingNodeExecutionContext(g) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 2
|
||||
assert output[0] == 'foo'
|
||||
assert output[1] == 'bar'
|
||||
assert output[0] == ('foo', )
|
||||
assert output[1] == ('bar', )
|
||||
|
||||
|
||||
def test_node_bytes():
|
||||
@ -37,23 +38,23 @@ def test_node_bytes():
|
||||
return b'foo'
|
||||
|
||||
with BufferingNodeExecutionContext(f) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
|
||||
output = context.get_buffer()
|
||||
assert len(output) == 1
|
||||
assert output[0] == b'foo'
|
||||
assert output[0] == (b'foo', )
|
||||
|
||||
def g():
|
||||
yield b'foo'
|
||||
yield b'bar'
|
||||
|
||||
with BufferingNodeExecutionContext(g) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 2
|
||||
assert output[0] == b'foo'
|
||||
assert output[1] == b'bar'
|
||||
assert output[0] == (b'foo', )
|
||||
assert output[1] == (b'bar', )
|
||||
|
||||
|
||||
def test_node_dict():
|
||||
@ -61,40 +62,38 @@ def test_node_dict():
|
||||
return {'id': 1, 'name': 'foo'}
|
||||
|
||||
with BufferingNodeExecutionContext(f) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 1
|
||||
assert output[0] == {'id': 1, 'name': 'foo'}
|
||||
assert output[0] == ({'id': 1, 'name': 'foo'}, )
|
||||
|
||||
def g():
|
||||
yield {'id': 1, 'name': 'foo'}
|
||||
yield {'id': 2, 'name': 'bar'}
|
||||
|
||||
with BufferingNodeExecutionContext(g) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 2
|
||||
assert output[0] == {'id': 1, 'name': 'foo'}
|
||||
assert output[1] == {'id': 2, 'name': 'bar'}
|
||||
assert output[0] == ({'id': 1, 'name': 'foo'}, )
|
||||
assert output[1] == ({'id': 2, 'name': 'bar'}, )
|
||||
|
||||
|
||||
def test_node_dict_chained():
|
||||
strategy = NaiveStrategy(GraphExecutionContextType=BufferingGraphExecutionContext)
|
||||
|
||||
def uppercase_name(**kwargs):
|
||||
return {**kwargs, 'name': kwargs['name'].upper()}
|
||||
|
||||
def f():
|
||||
return {'id': 1, 'name': 'foo'}
|
||||
|
||||
def uppercase_name(values):
|
||||
return {**values, 'name': values['name'].upper()}
|
||||
|
||||
graph = Graph(f, uppercase_name)
|
||||
context = strategy.execute(graph)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 1
|
||||
assert output[0] == {'id': 1, 'name': 'FOO'}
|
||||
assert output[0] == ({'id': 1, 'name': 'FOO'}, )
|
||||
|
||||
def g():
|
||||
yield {'id': 1, 'name': 'foo'}
|
||||
@ -105,8 +104,8 @@ def test_node_dict_chained():
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 2
|
||||
assert output[0] == {'id': 1, 'name': 'FOO'}
|
||||
assert output[1] == {'id': 2, 'name': 'BAR'}
|
||||
assert output[0] == ({'id': 1, 'name': 'FOO'}, )
|
||||
assert output[1] == ({'id': 2, 'name': 'BAR'}, )
|
||||
|
||||
|
||||
def test_node_tuple():
|
||||
@ -114,7 +113,7 @@ def test_node_tuple():
|
||||
return 'foo', 'bar'
|
||||
|
||||
with BufferingNodeExecutionContext(f) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 1
|
||||
@ -125,7 +124,7 @@ def test_node_tuple():
|
||||
yield 'foo', 'baz'
|
||||
|
||||
with BufferingNodeExecutionContext(g) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 2
|
||||
@ -167,7 +166,7 @@ def test_node_tuple_dict():
|
||||
return 'foo', 'bar', {'id': 1}
|
||||
|
||||
with BufferingNodeExecutionContext(f) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 1
|
||||
@ -178,7 +177,7 @@ def test_node_tuple_dict():
|
||||
yield 'foo', 'baz', {'id': 2}
|
||||
|
||||
with BufferingNodeExecutionContext(g) as context:
|
||||
context.write_sync(Bag())
|
||||
context.write_sync(EMPTY)
|
||||
output = context.get_buffer()
|
||||
|
||||
assert len(output) == 2
|
||||
|
||||
@ -21,19 +21,9 @@ class ResponseMock:
|
||||
|
||||
def test_read_from_opendatasoft_api():
|
||||
extract = OpenDataSoftAPI(dataset='test-a-set')
|
||||
with patch(
|
||||
'requests.get', return_value=ResponseMock([
|
||||
{
|
||||
'fields': {
|
||||
'foo': 'bar'
|
||||
}
|
||||
},
|
||||
{
|
||||
'fields': {
|
||||
'foo': 'zab'
|
||||
}
|
||||
},
|
||||
])
|
||||
):
|
||||
with patch('requests.get', return_value=ResponseMock([
|
||||
{'fields': {'foo': 'bar'}},
|
||||
{'fields': {'foo': 'zab'}},
|
||||
])):
|
||||
for line in extract('http://example.com/', ValueHolder(0)):
|
||||
assert 'foo' in line
|
||||
|
||||
@ -9,13 +9,7 @@ def useless(*args, **kwargs):
|
||||
def test_not_modified():
|
||||
input_messages = [
|
||||
('foo', 'bar'),
|
||||
{
|
||||
'foo': 'bar'
|
||||
},
|
||||
('foo', {
|
||||
'bar': 'baz'
|
||||
}),
|
||||
(),
|
||||
('foo', 'baz'),
|
||||
]
|
||||
|
||||
with BufferingNodeExecutionContext(useless) as context:
|
||||
|
||||
@ -1,62 +1,152 @@
|
||||
from collections import namedtuple
|
||||
from unittest import TestCase
|
||||
|
||||
import pytest
|
||||
|
||||
from bonobo import CsvReader, CsvWriter, settings
|
||||
from bonobo.execution.contexts.node import NodeExecutionContext
|
||||
from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext
|
||||
from bonobo import CsvReader, CsvWriter
|
||||
from bonobo.constants import EMPTY
|
||||
from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext, WriterTest, ConfigurableNodeTest, ReaderTest
|
||||
|
||||
csv_tester = FilesystemTester('csv')
|
||||
csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar'
|
||||
|
||||
defaults = {'lineterminator': '\n'}
|
||||
|
||||
def test_write_csv_ioformat_arg0(tmpdir):
|
||||
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
|
||||
with pytest.raises(ValueError):
|
||||
CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0),
|
||||
|
||||
|
||||
def test_write_csv_to_file_no_headers(tmpdir):
|
||||
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
|
||||
|
||||
with NodeExecutionContext(CsvWriter(filename), services=services) as context:
|
||||
context.write_sync(('bar', ), ('baz', 'boo'))
|
||||
|
||||
with fs.open(filename) as fp:
|
||||
assert fp.read() == 'bar\nbaz;boo\n'
|
||||
|
||||
|
||||
def test_write_csv_to_file_with_headers(tmpdir):
|
||||
fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
|
||||
|
||||
with NodeExecutionContext(CsvWriter(filename, headers='foo'), services=services) as context:
|
||||
context.write_sync(('bar', ), ('baz', 'boo'))
|
||||
|
||||
with fs.open(filename) as fp:
|
||||
assert fp.read() == 'foo\nbar\nbaz\n'
|
||||
|
||||
with pytest.raises(AttributeError):
|
||||
getattr(context, 'file')
|
||||
incontext = ConfigurableNodeTest.incontext
|
||||
|
||||
|
||||
def test_read_csv_from_file_kwargs(tmpdir):
|
||||
fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
|
||||
|
||||
with BufferingNodeExecutionContext(
|
||||
CsvReader(path=filename, delimiter=','),
|
||||
services=services,
|
||||
) as context:
|
||||
context.write_sync(())
|
||||
with BufferingNodeExecutionContext(CsvReader(filename, **defaults), services=services) as context:
|
||||
context.write_sync(EMPTY)
|
||||
|
||||
assert context.get_buffer_args_as_dicts() == [
|
||||
{
|
||||
'a': 'a foo',
|
||||
'b': 'b foo',
|
||||
'c': 'c foo',
|
||||
}, {
|
||||
'a': 'a bar',
|
||||
'b': 'b bar',
|
||||
'c': 'c bar',
|
||||
}
|
||||
]
|
||||
assert context.get_buffer_args_as_dicts() == [{
|
||||
'a': 'a foo',
|
||||
'b': 'b foo',
|
||||
'c': 'c foo',
|
||||
}, {
|
||||
'a': 'a bar',
|
||||
'b': 'b bar',
|
||||
'c': 'c bar',
|
||||
}]
|
||||
|
||||
|
||||
###
|
||||
# CSV Readers / Writers
|
||||
###
|
||||
|
||||
|
||||
class Csv:
|
||||
extension = 'csv'
|
||||
ReaderNodeType = CsvReader
|
||||
WriterNodeType = CsvWriter
|
||||
|
||||
|
||||
L1, L2, L3, L4 = ('a', 'hey'), ('b', 'bee'), ('c', 'see'), ('d', 'dee')
|
||||
LL = ('i', 'have', 'more', 'values')
|
||||
|
||||
|
||||
class CsvReaderTest(Csv, ReaderTest, TestCase):
|
||||
input_data = '\n'.join((
|
||||
'id,name',
|
||||
'1,John Doe',
|
||||
'2,Jane Doe',
|
||||
',DPR',
|
||||
'42,Elon Musk',
|
||||
))
|
||||
|
||||
def check_output(self, context, *, prepend=None):
|
||||
out = context.get_buffer()
|
||||
assert out == (prepend or list()) + [
|
||||
('1', 'John Doe'),
|
||||
('2', 'Jane Doe'),
|
||||
('', 'DPR'),
|
||||
('42', 'Elon Musk'),
|
||||
]
|
||||
|
||||
@incontext()
|
||||
def test_nofields(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
self.check_output(context)
|
||||
assert context.get_output_fields() == ('id', 'name')
|
||||
|
||||
@incontext(output_type=tuple)
|
||||
def test_output_type(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
self.check_output(context, prepend=[('id', 'name')])
|
||||
|
||||
@incontext(
|
||||
output_fields=(
|
||||
'x',
|
||||
'y',
|
||||
), skip=1
|
||||
)
|
||||
def test_output_fields(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
self.check_output(context)
|
||||
assert context.get_output_fields() == ('x', 'y')
|
||||
|
||||
|
||||
class CsvWriterTest(Csv, WriterTest, TestCase):
|
||||
@incontext()
|
||||
def test_fields(self, context):
|
||||
context.set_input_fields(['foo', 'bar'])
|
||||
context.write_sync(('a', 'b'), ('c', 'd'))
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == (
|
||||
'foo,bar',
|
||||
'a,b',
|
||||
'c,d',
|
||||
)
|
||||
|
||||
@incontext()
|
||||
def test_fields_from_type(self, context):
|
||||
context.set_input_type(namedtuple('Point', 'x y'))
|
||||
context.write_sync((1, 2), (3, 4))
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == ('x,y', '1,2', '3,4')
|
||||
|
||||
@incontext()
|
||||
def test_nofields_multiple_args(self, context):
|
||||
# multiple args are iterated onto and flattened in output
|
||||
context.write_sync((L1, L2), (L3, L4))
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == (
|
||||
'a,hey',
|
||||
'b,bee',
|
||||
'c,see',
|
||||
'd,dee',
|
||||
)
|
||||
|
||||
@incontext()
|
||||
def test_nofields_multiple_args_length_mismatch(self, context):
|
||||
# if length of input vary, then we get a TypeError (unrecoverable)
|
||||
with pytest.raises(TypeError):
|
||||
context.write_sync((L1, L2), (L3, ))
|
||||
|
||||
@incontext()
|
||||
def test_nofields_single_arg(self, context):
|
||||
# single args are just dumped, shapes can vary.
|
||||
context.write_sync((L1, ), (LL, ), (L3, ))
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == (
|
||||
'a,hey',
|
||||
'i,have,more,values',
|
||||
'c,see',
|
||||
)
|
||||
|
||||
@incontext()
|
||||
def test_nofields_empty_args(self, context):
|
||||
# empty calls are ignored
|
||||
context.write_sync(EMPTY, EMPTY, EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == ()
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import pytest
|
||||
|
||||
from bonobo import Bag, FileReader, FileWriter
|
||||
from bonobo.constants import BEGIN, END
|
||||
from bonobo import FileReader, FileWriter
|
||||
from bonobo.constants import EMPTY
|
||||
from bonobo.execution.contexts.node import NodeExecutionContext
|
||||
from bonobo.util.testing import BufferingNodeExecutionContext, FilesystemTester
|
||||
|
||||
@ -30,9 +30,7 @@ def test_file_writer_in_context(tmpdir, lines, output):
|
||||
fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
|
||||
|
||||
with NodeExecutionContext(FileWriter(path=filename), services=services) as context:
|
||||
context.write(BEGIN, *map(Bag, lines), END)
|
||||
for _ in range(len(lines)):
|
||||
context.step()
|
||||
context.write_sync(*lines)
|
||||
|
||||
with fs.open(filename) as fp:
|
||||
assert fp.read() == output
|
||||
@ -42,9 +40,9 @@ def test_file_reader(tmpdir):
|
||||
fs, filename, services = txt_tester.get_services_for_reader(tmpdir)
|
||||
|
||||
with BufferingNodeExecutionContext(FileReader(path=filename), services=services) as context:
|
||||
context.write_sync(Bag())
|
||||
output = context.get_buffer()
|
||||
context.write_sync(EMPTY)
|
||||
|
||||
output = context.get_buffer()
|
||||
assert len(output) == 2
|
||||
assert output[0] == 'Hello'
|
||||
assert output[1] == 'World'
|
||||
assert output[0] == ('Hello', )
|
||||
assert output[1] == ('World', )
|
||||
|
||||
@ -1,66 +1,300 @@
|
||||
import json
|
||||
from collections import OrderedDict, namedtuple
|
||||
from unittest import TestCase
|
||||
|
||||
import pytest
|
||||
|
||||
from bonobo import JsonReader, JsonWriter, settings
|
||||
from bonobo import JsonReader, JsonWriter
|
||||
from bonobo import LdjsonReader, LdjsonWriter
|
||||
from bonobo.execution.contexts.node import NodeExecutionContext
|
||||
from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext
|
||||
from bonobo.constants import EMPTY
|
||||
from bonobo.util.testing import WriterTest, ReaderTest, ConfigurableNodeTest
|
||||
|
||||
json_tester = FilesystemTester('json')
|
||||
json_tester.input_data = '''[{"x": "foo"},{"x": "bar"}]'''
|
||||
FOOBAR = {'foo': 'bar'}
|
||||
OD_ABC = OrderedDict((('a', 'A'), ('b', 'B'), ('c', 'C')))
|
||||
FOOBAZ = {'foo': 'baz'}
|
||||
|
||||
incontext = ConfigurableNodeTest.incontext
|
||||
|
||||
###
|
||||
# Standard JSON Readers / Writers
|
||||
###
|
||||
|
||||
|
||||
def test_write_json_ioformat_arg0(tmpdir):
|
||||
fs, filename, services = json_tester.get_services_for_writer(tmpdir)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
JsonReader(filename, ioformat=settings.IOFORMAT_ARG0),
|
||||
class Json:
|
||||
extension = 'json'
|
||||
ReaderNodeType = JsonReader
|
||||
WriterNodeType = JsonWriter
|
||||
|
||||
|
||||
@pytest.mark.parametrize('add_kwargs', (
|
||||
{},
|
||||
{
|
||||
'ioformat': settings.IOFORMAT_KWARGS,
|
||||
},
|
||||
))
|
||||
def test_write_json_kwargs(tmpdir, add_kwargs):
|
||||
fs, filename, services = json_tester.get_services_for_writer(tmpdir)
|
||||
class JsonReaderDictsTest(Json, ReaderTest, TestCase):
|
||||
input_data = '[{"foo": "bar"},\n{"baz": "boz"}]'
|
||||
|
||||
with NodeExecutionContext(JsonWriter(filename, **add_kwargs), services=services) as context:
|
||||
context.write_sync({'foo': 'bar'})
|
||||
@incontext()
|
||||
def test_nofields(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
with fs.open(filename) as fp:
|
||||
assert fp.read() == '[{"foo": "bar"}]'
|
||||
assert context.get_buffer() == [
|
||||
({
|
||||
"foo": "bar"
|
||||
}, ),
|
||||
({
|
||||
"baz": "boz"
|
||||
}, ),
|
||||
]
|
||||
|
||||
|
||||
stream_json_tester = FilesystemTester('json')
|
||||
stream_json_tester.input_data = '''{"foo": "bar"}\n{"baz": "boz"}'''
|
||||
class JsonReaderListsTest(Json, ReaderTest, TestCase):
|
||||
input_data = '[[1,2,3],\n[4,5,6]]'
|
||||
|
||||
@incontext()
|
||||
def test_nofields(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert context.get_buffer() == [
|
||||
([1, 2, 3], ),
|
||||
([4, 5, 6], ),
|
||||
]
|
||||
|
||||
@incontext(output_type=tuple)
|
||||
def test_output_type(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert context.get_buffer() == [
|
||||
([1, 2, 3], ),
|
||||
([4, 5, 6], ),
|
||||
]
|
||||
|
||||
|
||||
def test_read_stream_json(tmpdir):
|
||||
fs, filename, services = stream_json_tester.get_services_for_reader(tmpdir)
|
||||
with BufferingNodeExecutionContext(LdjsonReader(filename), services=services) as context:
|
||||
context.write_sync(tuple())
|
||||
actual = context.get_buffer()
|
||||
class JsonReaderStringsTest(Json, ReaderTest, TestCase):
|
||||
input_data = '[' + ',\n'.join(map(json.dumps, ('foo', 'bar', 'baz'))) + ']'
|
||||
|
||||
expected = [{"foo": "bar"}, {"baz": "boz"}]
|
||||
assert expected == actual
|
||||
@incontext()
|
||||
def test_nofields(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert context.get_buffer() == [
|
||||
('foo', ),
|
||||
('bar', ),
|
||||
('baz', ),
|
||||
]
|
||||
|
||||
@incontext(output_type=tuple)
|
||||
def test_output_type(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert context.get_buffer() == [
|
||||
('foo', ),
|
||||
('bar', ),
|
||||
('baz', ),
|
||||
]
|
||||
|
||||
|
||||
def test_write_stream_json(tmpdir):
|
||||
fs, filename, services = stream_json_tester.get_services_for_reader(tmpdir)
|
||||
class JsonWriterTest(Json, WriterTest, TestCase):
|
||||
@incontext()
|
||||
def test_fields(self, context):
|
||||
context.set_input_fields(['foo', 'bar'])
|
||||
context.write_sync(('a', 'b'), ('c', 'd'))
|
||||
context.stop()
|
||||
|
||||
with BufferingNodeExecutionContext(LdjsonWriter(filename), services=services) as context:
|
||||
context.write_sync(
|
||||
{
|
||||
'foo': 'bar'
|
||||
},
|
||||
{'baz': 'boz'},
|
||||
assert self.readlines() == (
|
||||
'[{"foo": "a", "bar": "b"},',
|
||||
'{"foo": "c", "bar": "d"}]',
|
||||
)
|
||||
|
||||
expected = '''{"foo": "bar"}\n{"baz": "boz"}\n'''
|
||||
with fs.open(filename) as fin:
|
||||
actual = fin.read()
|
||||
assert expected == actual
|
||||
@incontext()
|
||||
def test_fields_from_type(self, context):
|
||||
context.set_input_type(namedtuple('Point', 'x y'))
|
||||
context.write_sync((1, 2), (3, 4))
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == (
|
||||
'[{"x": 1, "y": 2},',
|
||||
'{"x": 3, "y": 4}]',
|
||||
)
|
||||
|
||||
@incontext()
|
||||
def test_nofields_multiple_args(self, context):
|
||||
# multiple args are iterated onto and flattened in output
|
||||
context.write_sync((FOOBAR, FOOBAR), (OD_ABC, FOOBAR), (FOOBAZ, FOOBAR))
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == (
|
||||
'[{"foo": "bar"},',
|
||||
'{"foo": "bar"},',
|
||||
'{"a": "A", "b": "B", "c": "C"},',
|
||||
'{"foo": "bar"},',
|
||||
'{"foo": "baz"},',
|
||||
'{"foo": "bar"}]',
|
||||
)
|
||||
|
||||
@incontext()
|
||||
def test_nofields_multiple_args_length_mismatch(self, context):
|
||||
# if length of input vary, then we get a TypeError (unrecoverable)
|
||||
with pytest.raises(TypeError):
|
||||
context.write_sync((FOOBAR, FOOBAR), (OD_ABC))
|
||||
|
||||
@incontext()
|
||||
def test_nofields_single_arg(self, context):
|
||||
# single args are just dumped, shapes can vary.
|
||||
context.write_sync(FOOBAR, OD_ABC, FOOBAZ)
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == (
|
||||
'[{"foo": "bar"},',
|
||||
'{"a": "A", "b": "B", "c": "C"},',
|
||||
'{"foo": "baz"}]',
|
||||
)
|
||||
|
||||
@incontext()
|
||||
def test_nofields_empty_args(self, context):
|
||||
# empty calls are ignored
|
||||
context.write_sync(EMPTY, EMPTY, EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == ('[]', )
|
||||
|
||||
|
||||
###
|
||||
# Line Delimiter JSON Readers / Writers
|
||||
###
|
||||
|
||||
|
||||
class Ldjson:
|
||||
extension = 'ldjson'
|
||||
ReaderNodeType = LdjsonReader
|
||||
WriterNodeType = LdjsonWriter
|
||||
|
||||
|
||||
class LdjsonReaderDictsTest(Ldjson, ReaderTest, TestCase):
|
||||
input_data = '{"foo": "bar"}\n{"baz": "boz"}'
|
||||
|
||||
@incontext()
|
||||
def test_nofields(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert context.get_buffer() == [
|
||||
({
|
||||
"foo": "bar"
|
||||
}, ),
|
||||
({
|
||||
"baz": "boz"
|
||||
}, ),
|
||||
]
|
||||
|
||||
|
||||
class LdjsonReaderListsTest(Ldjson, ReaderTest, TestCase):
|
||||
input_data = '[1,2,3]\n[4,5,6]'
|
||||
|
||||
@incontext()
|
||||
def test_nofields(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert context.get_buffer() == [
|
||||
([1, 2, 3], ),
|
||||
([4, 5, 6], ),
|
||||
]
|
||||
|
||||
@incontext(output_type=tuple)
|
||||
def test_output_type(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert context.get_buffer() == [
|
||||
([1, 2, 3], ),
|
||||
([4, 5, 6], ),
|
||||
]
|
||||
|
||||
|
||||
class LdjsonReaderStringsTest(Ldjson, ReaderTest, TestCase):
|
||||
input_data = '\n'.join(map(json.dumps, ('foo', 'bar', 'baz')))
|
||||
|
||||
@incontext()
|
||||
def test_nofields(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert context.get_buffer() == [
|
||||
('foo', ),
|
||||
('bar', ),
|
||||
('baz', ),
|
||||
]
|
||||
|
||||
@incontext(output_type=tuple)
|
||||
def test_output_type(self, context):
|
||||
context.write_sync(EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert context.get_buffer() == [
|
||||
('foo', ),
|
||||
('bar', ),
|
||||
('baz', ),
|
||||
]
|
||||
|
||||
|
||||
class LdjsonWriterTest(Ldjson, WriterTest, TestCase):
|
||||
@incontext()
|
||||
def test_fields(self, context):
|
||||
context.set_input_fields(['foo', 'bar'])
|
||||
context.write_sync(('a', 'b'), ('c', 'd'))
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == ('{"foo": "a", "bar": "b"}', '{"foo": "c", "bar": "d"}')
|
||||
|
||||
@incontext()
|
||||
def test_fields_from_type(self, context):
|
||||
context.set_input_type(namedtuple('Point', 'x y'))
|
||||
context.write_sync((1, 2), (3, 4))
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == (
|
||||
'{"x": 1, "y": 2}',
|
||||
'{"x": 3, "y": 4}',
|
||||
)
|
||||
|
||||
@incontext()
|
||||
def test_nofields_multiple_args(self, context):
|
||||
# multiple args are iterated onto and flattened in output
|
||||
context.write_sync((FOOBAR, FOOBAR), (OD_ABC, FOOBAR), (FOOBAZ, FOOBAR))
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == (
|
||||
'{"foo": "bar"}',
|
||||
'{"foo": "bar"}',
|
||||
'{"a": "A", "b": "B", "c": "C"}',
|
||||
'{"foo": "bar"}',
|
||||
'{"foo": "baz"}',
|
||||
'{"foo": "bar"}',
|
||||
)
|
||||
|
||||
@incontext()
|
||||
def test_nofields_multiple_args_length_mismatch(self, context):
|
||||
# if length of input vary, then we get a TypeError (unrecoverable)
|
||||
with pytest.raises(TypeError):
|
||||
context.write_sync((FOOBAR, FOOBAR), (OD_ABC))
|
||||
|
||||
@incontext()
|
||||
def test_nofields_single_arg(self, context):
|
||||
# single args are just dumped, shapes can vary.
|
||||
context.write_sync(FOOBAR, OD_ABC, FOOBAZ)
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == (
|
||||
'{"foo": "bar"}',
|
||||
'{"a": "A", "b": "B", "c": "C"}',
|
||||
'{"foo": "baz"}',
|
||||
)
|
||||
|
||||
@incontext()
|
||||
def test_nofields_empty_args(self, context):
|
||||
# empty calls are ignored
|
||||
context.write_sync(EMPTY, EMPTY, EMPTY)
|
||||
context.stop()
|
||||
|
||||
assert self.readlines() == ()
|
||||
|
||||
@ -2,7 +2,8 @@ import pickle
|
||||
|
||||
import pytest
|
||||
|
||||
from bonobo import Bag, PickleReader, PickleWriter
|
||||
from bonobo import PickleReader, PickleWriter
|
||||
from bonobo.constants import EMPTY
|
||||
from bonobo.execution.contexts.node import NodeExecutionContext
|
||||
from bonobo.util.testing import BufferingNodeExecutionContext, FilesystemTester
|
||||
|
||||
@ -14,7 +15,7 @@ def test_write_pickled_dict_to_file(tmpdir):
|
||||
fs, filename, services = pickle_tester.get_services_for_writer(tmpdir)
|
||||
|
||||
with NodeExecutionContext(PickleWriter(filename), services=services) as context:
|
||||
context.write_sync(Bag(({'foo': 'bar'}, {})), Bag(({'foo': 'baz', 'ignore': 'this'}, {})))
|
||||
context.write_sync({'foo': 'bar'}, {'foo': 'baz', 'ignore': 'this'})
|
||||
|
||||
with fs.open(filename, 'rb') as fp:
|
||||
assert pickle.loads(fp.read()) == {'foo': 'bar'}
|
||||
@ -27,17 +28,11 @@ def test_read_pickled_list_from_file(tmpdir):
|
||||
fs, filename, services = pickle_tester.get_services_for_reader(tmpdir)
|
||||
|
||||
with BufferingNodeExecutionContext(PickleReader(filename), services=services) as context:
|
||||
context.write_sync(())
|
||||
output = context.get_buffer()
|
||||
context.write_sync(EMPTY)
|
||||
|
||||
assert len(output) == 2
|
||||
assert output[0] == {
|
||||
'a': 'a foo',
|
||||
'b': 'b foo',
|
||||
'c': 'c foo',
|
||||
}
|
||||
assert output[1] == {
|
||||
'a': 'a bar',
|
||||
'b': 'b bar',
|
||||
'c': 'c bar',
|
||||
}
|
||||
output = context.get_buffer()
|
||||
assert context.get_output_fields() == ('a', 'b', 'c')
|
||||
assert output == [
|
||||
('a foo', 'b foo', 'c foo'),
|
||||
('a bar', 'b bar', 'c bar'),
|
||||
]
|
||||
|
||||
@ -1,64 +1,79 @@
|
||||
from operator import methodcaller
|
||||
from unittest import TestCase
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import bonobo
|
||||
from bonobo.config.processors import ContextCurrifier
|
||||
from bonobo.constants import NOT_MODIFIED
|
||||
from bonobo.util.testing import BufferingNodeExecutionContext
|
||||
from bonobo.constants import NOT_MODIFIED, EMPTY
|
||||
from bonobo.util import ensure_tuple, ValueHolder
|
||||
from bonobo.util.testing import BufferingNodeExecutionContext, StaticNodeTest, ConfigurableNodeTest
|
||||
|
||||
|
||||
def test_count():
|
||||
with pytest.raises(TypeError):
|
||||
bonobo.count()
|
||||
class CountTest(StaticNodeTest, TestCase):
|
||||
node = bonobo.count
|
||||
|
||||
context = MagicMock()
|
||||
def test_counter_required(self):
|
||||
with pytest.raises(TypeError):
|
||||
self.call()
|
||||
|
||||
with ContextCurrifier(bonobo.count).as_contextmanager(context) as stack:
|
||||
for i in range(42):
|
||||
stack()
|
||||
def test_manual_call(self):
|
||||
counter = ValueHolder(0)
|
||||
for i in range(3):
|
||||
self.call(counter)
|
||||
assert counter == 3
|
||||
|
||||
assert len(context.method_calls) == 1
|
||||
bag = context.send.call_args[0][0]
|
||||
assert isinstance(bag, bonobo.Bag)
|
||||
assert 0 == len(bag.kwargs)
|
||||
assert 1 == len(bag.args)
|
||||
assert bag.args[0] == 42
|
||||
def test_execution(self):
|
||||
with self.execute() as context:
|
||||
context.write_sync(*([EMPTY] * 42))
|
||||
assert context.get_buffer() == [(42, )]
|
||||
|
||||
|
||||
def test_identity():
|
||||
assert bonobo.identity(42) == 42
|
||||
class IdentityTest(StaticNodeTest, TestCase):
|
||||
node = bonobo.identity
|
||||
|
||||
def test_basic_call(self):
|
||||
assert self.call(42) == 42
|
||||
|
||||
def test_execution(self):
|
||||
object_list = [object() for _ in range(42)]
|
||||
with self.execute() as context:
|
||||
context.write_sync(*object_list)
|
||||
assert context.get_buffer() == list(map(ensure_tuple, object_list))
|
||||
|
||||
|
||||
def test_limit():
|
||||
context, results = MagicMock(), []
|
||||
class LimitTest(ConfigurableNodeTest, TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.NodeType = bonobo.Limit
|
||||
|
||||
with ContextCurrifier(bonobo.Limit(2)).as_contextmanager(context) as stack:
|
||||
for i in range(42):
|
||||
results += list(stack())
|
||||
def test_execution_default(self):
|
||||
object_list = [object() for _ in range(42)]
|
||||
with self.execute() as context:
|
||||
context.write_sync(*object_list)
|
||||
|
||||
assert results == [NOT_MODIFIED] * 2
|
||||
assert context.get_buffer() == list(map(ensure_tuple, object_list[:10]))
|
||||
|
||||
def test_execution_custom(self):
|
||||
object_list = [object() for _ in range(42)]
|
||||
with self.execute(21) as context:
|
||||
context.write_sync(*object_list)
|
||||
|
||||
def test_limit_not_there():
|
||||
context, results = MagicMock(), []
|
||||
assert context.get_buffer() == list(map(ensure_tuple, object_list[:21]))
|
||||
|
||||
with ContextCurrifier(bonobo.Limit(42)).as_contextmanager(context) as stack:
|
||||
for i in range(10):
|
||||
results += list(stack())
|
||||
def test_manual(self):
|
||||
limit = self.NodeType(5)
|
||||
buffer = []
|
||||
for x in range(10):
|
||||
buffer += list(limit(x))
|
||||
assert len(buffer) == 5
|
||||
|
||||
assert results == [NOT_MODIFIED] * 10
|
||||
|
||||
|
||||
def test_limit_default():
|
||||
context, results = MagicMock(), []
|
||||
|
||||
with ContextCurrifier(bonobo.Limit()).as_contextmanager(context) as stack:
|
||||
for i in range(20):
|
||||
results += list(stack())
|
||||
|
||||
assert results == [NOT_MODIFIED] * 10
|
||||
def test_underflow(self):
|
||||
limit = self.NodeType(10)
|
||||
buffer = []
|
||||
for x in range(5):
|
||||
buffer += list(limit(x))
|
||||
assert len(buffer) == 5
|
||||
|
||||
|
||||
def test_tee():
|
||||
@ -76,36 +91,28 @@ def test_noop():
|
||||
assert bonobo.noop(1, 2, 3, 4, foo='bar') == NOT_MODIFIED
|
||||
|
||||
|
||||
def test_update():
|
||||
with BufferingNodeExecutionContext(bonobo.Update('a', k=True)) as context:
|
||||
context.write_sync('a', ('a', {'b': 1}), ('b', {'k': False}))
|
||||
assert context.get_buffer() == [
|
||||
bonobo.Bag('a', 'a', k=True),
|
||||
bonobo.Bag('a', 'a', b=1, k=True),
|
||||
bonobo.Bag('b', 'a', k=True),
|
||||
]
|
||||
assert context.name == "Update('a', k=True)"
|
||||
|
||||
|
||||
def test_fixedwindow():
|
||||
with BufferingNodeExecutionContext(bonobo.FixedWindow(2)) as context:
|
||||
context.write_sync(*range(10))
|
||||
assert context.get_buffer() == [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]]
|
||||
assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)]
|
||||
|
||||
with BufferingNodeExecutionContext(bonobo.FixedWindow(2)) as context:
|
||||
context.write_sync(*range(9))
|
||||
assert context.get_buffer() == [[0, 1], [2, 3], [4, 5], [6, 7], [8]]
|
||||
assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), (
|
||||
8,
|
||||
None,
|
||||
)]
|
||||
|
||||
with BufferingNodeExecutionContext(bonobo.FixedWindow(1)) as context:
|
||||
context.write_sync(*range(3))
|
||||
assert context.get_buffer() == [[0], [1], [2]]
|
||||
assert context.get_buffer() == [(0, ), (1, ), (2, )]
|
||||
|
||||
|
||||
def test_methodcaller():
|
||||
with BufferingNodeExecutionContext(methodcaller('swapcase')) as context:
|
||||
context.write_sync('aaa', 'bBb', 'CcC')
|
||||
assert context.get_buffer() == ['AAA', 'BbB', 'cCc']
|
||||
assert context.get_buffer() == list(map(ensure_tuple, ['AAA', 'BbB', 'cCc']))
|
||||
|
||||
with BufferingNodeExecutionContext(methodcaller('zfill', 5)) as context:
|
||||
context.write_sync('a', 'bb', 'ccc')
|
||||
assert context.get_buffer() == ['0000a', '000bb', '00ccc']
|
||||
assert context.get_buffer() == list(map(ensure_tuple, ['0000a', '000bb', '00ccc']))
|
||||
|
||||
@ -1,170 +0,0 @@
|
||||
import pickle
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from bonobo import Bag
|
||||
from bonobo.constants import INHERIT_INPUT, BEGIN
|
||||
from bonobo.structs import Token
|
||||
|
||||
args = (
|
||||
'foo',
|
||||
'bar',
|
||||
)
|
||||
kwargs = dict(acme='corp')
|
||||
|
||||
|
||||
def test_basic():
|
||||
my_callable1 = Mock()
|
||||
my_callable2 = Mock()
|
||||
bag = Bag(*args, **kwargs)
|
||||
|
||||
assert not my_callable1.called
|
||||
result1 = bag.apply(my_callable1)
|
||||
assert my_callable1.called and result1 is my_callable1.return_value
|
||||
|
||||
assert not my_callable2.called
|
||||
result2 = bag.apply(my_callable2)
|
||||
assert my_callable2.called and result2 is my_callable2.return_value
|
||||
|
||||
assert result1 is not result2
|
||||
|
||||
my_callable1.assert_called_once_with(*args, **kwargs)
|
||||
my_callable2.assert_called_once_with(*args, **kwargs)
|
||||
|
||||
|
||||
def test_constructor_empty():
|
||||
a, b = Bag(), Bag()
|
||||
assert a == b
|
||||
assert a.args is ()
|
||||
assert a.kwargs == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(('arg_in', 'arg_out'), (
|
||||
((), ()),
|
||||
({}, ()),
|
||||
(('a', 'b', 'c'), None),
|
||||
))
|
||||
def test_constructor_shorthand(arg_in, arg_out):
|
||||
if arg_out is None:
|
||||
arg_out = arg_in
|
||||
assert Bag(arg_in) == arg_out
|
||||
|
||||
|
||||
def test_constructor_kwargs_only():
|
||||
assert Bag(foo='bar') == {'foo': 'bar'}
|
||||
|
||||
|
||||
def test_constructor_identity():
|
||||
assert Bag(BEGIN) is BEGIN
|
||||
|
||||
|
||||
def test_inherit():
|
||||
bag = Bag('a', a=1)
|
||||
bag2 = Bag.inherit('b', b=2, _parent=bag)
|
||||
bag3 = bag.extend('c', c=3)
|
||||
bag4 = Bag('d', d=4)
|
||||
|
||||
assert bag.args == ('a', )
|
||||
assert bag.kwargs == {'a': 1}
|
||||
assert bag.flags is ()
|
||||
|
||||
assert bag2.args == (
|
||||
'a',
|
||||
'b',
|
||||
)
|
||||
assert bag2.kwargs == {'a': 1, 'b': 2}
|
||||
assert INHERIT_INPUT in bag2.flags
|
||||
|
||||
assert bag3.args == (
|
||||
'a',
|
||||
'c',
|
||||
)
|
||||
assert bag3.kwargs == {'a': 1, 'c': 3}
|
||||
assert bag3.flags is ()
|
||||
|
||||
assert bag4.args == ('d', )
|
||||
assert bag4.kwargs == {'d': 4}
|
||||
assert bag4.flags is ()
|
||||
|
||||
bag4.set_parent(bag)
|
||||
assert bag4.args == (
|
||||
'a',
|
||||
'd',
|
||||
)
|
||||
assert bag4.kwargs == {'a': 1, 'd': 4}
|
||||
assert bag4.flags is ()
|
||||
|
||||
bag4.set_parent(bag3)
|
||||
assert bag4.args == (
|
||||
'a',
|
||||
'c',
|
||||
'd',
|
||||
)
|
||||
assert bag4.kwargs == {'a': 1, 'c': 3, 'd': 4}
|
||||
assert bag4.flags is ()
|
||||
|
||||
|
||||
def test_pickle():
|
||||
bag1 = Bag('a', a=1)
|
||||
bag2 = Bag.inherit('b', b=2, _parent=bag1)
|
||||
bag3 = bag1.extend('c', c=3)
|
||||
bag4 = Bag('d', d=4)
|
||||
|
||||
# XXX todo this probably won't work with inheriting bags if parent is not there anymore? maybe that's not true
|
||||
# because the parent may be in the serialization output but we need to verify this assertion.
|
||||
|
||||
for bag in bag1, bag2, bag3, bag4:
|
||||
pickled = pickle.dumps(bag)
|
||||
unpickled = pickle.loads(pickled)
|
||||
assert unpickled == bag
|
||||
|
||||
|
||||
def test_eq_operator_bag():
|
||||
assert Bag('foo') == Bag('foo')
|
||||
assert Bag('foo') != Bag('bar')
|
||||
assert Bag('foo') is not Bag('foo')
|
||||
assert Bag('foo') != Token('foo')
|
||||
assert Token('foo') != Bag('foo')
|
||||
|
||||
|
||||
def test_eq_operator_tuple_mixed():
|
||||
assert Bag('foo', bar='baz') == ('foo', {'bar': 'baz'})
|
||||
assert Bag('foo') == ('foo', {})
|
||||
assert Bag() == ({}, )
|
||||
|
||||
|
||||
def test_eq_operator_tuple_not_mixed():
|
||||
assert Bag('foo', 'bar') == ('foo', 'bar')
|
||||
assert Bag('foo') == ('foo', )
|
||||
assert Bag() == ()
|
||||
|
||||
|
||||
def test_eq_operator_dict():
|
||||
assert Bag(foo='bar') == {'foo': 'bar'}
|
||||
assert Bag(
|
||||
foo='bar', corp='acme'
|
||||
) == {
|
||||
'foo': 'bar',
|
||||
'corp': 'acme',
|
||||
}
|
||||
assert Bag(
|
||||
foo='bar', corp='acme'
|
||||
) == {
|
||||
'corp': 'acme',
|
||||
'foo': 'bar',
|
||||
}
|
||||
assert Bag() == {}
|
||||
|
||||
|
||||
def test_repr():
|
||||
bag = Bag('a', a=1)
|
||||
assert repr(bag) == "Bag('a', a=1)"
|
||||
|
||||
|
||||
def test_iterator():
|
||||
bag = Bag()
|
||||
assert list(bag.apply([1, 2, 3])) == [1, 2, 3]
|
||||
assert list(bag.apply((1, 2, 3))) == [1, 2, 3]
|
||||
assert list(bag.apply(range(5))) == [0, 1, 2, 3, 4]
|
||||
assert list(bag.apply('azerty')) == ['a', 'z', 'e', 'r', 't', 'y']
|
||||
@ -1,4 +1,4 @@
|
||||
from bonobo.structs import Token
|
||||
from bonobo.constants import Token
|
||||
|
||||
|
||||
def test_token_repr():
|
||||
|
||||
@ -1,29 +1,28 @@
|
||||
from bonobo.config.processors import ContextProcessor
|
||||
from bonobo.config.processors import use_context_processor
|
||||
from bonobo.constants import BEGIN, END
|
||||
from bonobo.execution.contexts.graph import GraphExecutionContext
|
||||
from bonobo.execution.strategies import NaiveStrategy
|
||||
from bonobo.structs import Bag, Graph
|
||||
from bonobo.structs import Graph
|
||||
|
||||
|
||||
def generate_integers():
|
||||
yield from range(10)
|
||||
|
||||
|
||||
def square(i: int) -> int:
|
||||
def square(i):
|
||||
return i**2
|
||||
|
||||
|
||||
def push_result(results, i: int):
|
||||
results.append(i)
|
||||
|
||||
|
||||
@ContextProcessor.decorate(push_result)
|
||||
def results(f, context):
|
||||
results = []
|
||||
yield results
|
||||
results = yield list()
|
||||
context.parent.results = results
|
||||
|
||||
|
||||
@use_context_processor(results)
|
||||
def push_result(results, i):
|
||||
results.append(i)
|
||||
|
||||
|
||||
chain = (generate_integers, square, push_result)
|
||||
|
||||
|
||||
@ -62,7 +61,7 @@ def test_simple_execution_context():
|
||||
assert not context.started
|
||||
assert not context.stopped
|
||||
|
||||
context.write(BEGIN, Bag(), END)
|
||||
context.write(BEGIN, (), END)
|
||||
|
||||
assert not context.alive
|
||||
assert not context.started
|
||||
|
||||
278
tests/util/test_bags.py
Normal file
278
tests/util/test_bags.py
Normal file
@ -0,0 +1,278 @@
|
||||
"""Those tests are mostly a copy paste of cpython unit tests for namedtuple, with a few differences to reflect the
|
||||
implementation details that differs. It ensures that we caught the same edge cases as they did."""
|
||||
|
||||
import collections
|
||||
import copy
|
||||
import pickle
|
||||
import string
|
||||
import sys
|
||||
import unittest
|
||||
from collections import OrderedDict
|
||||
from random import choice
|
||||
|
||||
from bonobo.util.bags import BagType
|
||||
|
||||
################################################################################
|
||||
### Named Tuples
|
||||
################################################################################
|
||||
|
||||
TBag = BagType('TBag', ('x', 'y', 'z')) # type used for pickle tests
|
||||
|
||||
|
||||
class TestBagType(unittest.TestCase):
|
||||
def _create(self, *fields, typename='abc'):
|
||||
bt = BagType(typename, fields)
|
||||
assert bt._fields == fields
|
||||
assert len(bt._fields) == len(bt._attrs)
|
||||
return bt
|
||||
|
||||
def test_factory(self):
|
||||
Point = BagType('Point', ('x', 'y'))
|
||||
self.assertEqual(Point.__name__, 'Point')
|
||||
self.assertEqual(Point.__slots__, ())
|
||||
self.assertEqual(Point.__module__, __name__)
|
||||
self.assertEqual(Point.__getitem__, tuple.__getitem__)
|
||||
assert Point._fields == ('x', 'y')
|
||||
assert Point._attrs == ('x', 'y')
|
||||
|
||||
self.assertRaises(ValueError, BagType, 'abc%', ('efg', 'ghi')) # type has non-alpha char
|
||||
self.assertRaises(ValueError, BagType, 'class', ('efg', 'ghi')) # type has keyword
|
||||
self.assertRaises(ValueError, BagType, '9abc', ('efg', 'ghi')) # type starts with digit
|
||||
|
||||
assert self._create('efg', 'g%hi')._attrs == ('efg', 'g_hi')
|
||||
assert self._create('abc', 'class')._attrs == ('abc', '_class')
|
||||
assert self._create('8efg', '9ghi')._attrs == ('_8efg', '_9ghi')
|
||||
assert self._create('_efg', 'ghi')._attrs == ('_efg', 'ghi')
|
||||
|
||||
self.assertRaises(ValueError, BagType, 'abc', ('efg', 'efg', 'ghi')) # duplicate field
|
||||
|
||||
self._create('x1', 'y2', typename='Point0') # Verify that numbers are allowed in names
|
||||
self._create('a', 'b', 'c', typename='_') # Test leading underscores in a typename
|
||||
|
||||
bt = self._create('a!', 'a?')
|
||||
assert bt._attrs == ('a0', 'a1')
|
||||
x = bt('foo', 'bar')
|
||||
assert x.get('a!') == 'foo'
|
||||
assert x.a0 == 'foo'
|
||||
assert x.get('a?') == 'bar'
|
||||
assert x.a1 == 'bar'
|
||||
|
||||
# check unicode output
|
||||
bt = self._create('the', 'quick', 'brown', 'fox')
|
||||
assert "u'" not in repr(bt._fields)
|
||||
|
||||
self.assertRaises(TypeError, Point._make, [11]) # catch too few args
|
||||
self.assertRaises(TypeError, Point._make, [11, 22, 33]) # catch too many args
|
||||
|
||||
@unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above")
|
||||
def test_factory_doc_attr(self):
|
||||
Point = BagType('Point', ('x', 'y'))
|
||||
self.assertEqual(Point.__doc__, 'Point(x, y)')
|
||||
|
||||
@unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above")
|
||||
def test_doc_writable(self):
|
||||
Point = BagType('Point', ('x', 'y'))
|
||||
self.assertEqual(Point.x.__doc__, "Alias for 'x'")
|
||||
Point.x.__doc__ = 'docstring for Point.x'
|
||||
self.assertEqual(Point.x.__doc__, 'docstring for Point.x')
|
||||
|
||||
def test_name_fixer(self):
|
||||
for spec, renamed in [
|
||||
[('efg', 'g%hi'), ('efg', 'g_hi')], # field with non-alpha char
|
||||
[('abc', 'class'), ('abc', '_class')], # field has keyword
|
||||
[('8efg', '9ghi'), ('_8efg', '_9ghi')], # field starts with digit
|
||||
[('abc', '_efg'), ('abc', '_efg')], # field with leading underscore
|
||||
[('abc', '', 'x'), ('abc', '_0', 'x')], # fieldname is a space
|
||||
[('&', '¨', '*'), ('_0', '_1', '_2')], # Duplicate attrs, in theory
|
||||
]:
|
||||
assert self._create(*spec)._attrs == renamed
|
||||
|
||||
def test_module_parameter(self):
|
||||
NT = BagType('NT', ['x', 'y'], module=collections)
|
||||
self.assertEqual(NT.__module__, collections)
|
||||
|
||||
def test_instance(self):
|
||||
Point = self._create('x', 'y', typename='Point')
|
||||
p = Point(11, 22)
|
||||
self.assertEqual(p, Point(x=11, y=22))
|
||||
self.assertEqual(p, Point(11, y=22))
|
||||
self.assertEqual(p, Point(y=22, x=11))
|
||||
self.assertEqual(p, Point(*(11, 22)))
|
||||
self.assertEqual(p, Point(**dict(x=11, y=22)))
|
||||
self.assertRaises(TypeError, Point, 1) # too few args
|
||||
self.assertRaises(TypeError, Point, 1, 2, 3) # too many args
|
||||
self.assertRaises(TypeError, eval, 'Point(XXX=1, y=2)', locals()) # wrong keyword argument
|
||||
self.assertRaises(TypeError, eval, 'Point(x=1)', locals()) # missing keyword argument
|
||||
self.assertEqual(repr(p), 'Point(x=11, y=22)')
|
||||
self.assertNotIn('__weakref__', dir(p))
|
||||
self.assertEqual(p, Point._make([11, 22])) # test _make classmethod
|
||||
self.assertEqual(p._fields, ('x', 'y')) # test _fields attribute
|
||||
self.assertEqual(p._replace(x=1), (1, 22)) # test _replace method
|
||||
self.assertEqual(p._asdict(), dict(x=11, y=22)) # test _asdict method
|
||||
|
||||
try:
|
||||
p._replace(x=1, error=2)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
self._fail('Did not detect an incorrect fieldname')
|
||||
|
||||
p = Point(x=11, y=22)
|
||||
self.assertEqual(repr(p), 'Point(x=11, y=22)')
|
||||
|
||||
def test_tupleness(self):
|
||||
Point = BagType('Point', ('x', 'y'))
|
||||
p = Point(11, 22)
|
||||
|
||||
self.assertIsInstance(p, tuple)
|
||||
self.assertEqual(p, (11, 22)) # matches a real tuple
|
||||
self.assertEqual(tuple(p), (11, 22)) # coercable to a real tuple
|
||||
self.assertEqual(list(p), [11, 22]) # coercable to a list
|
||||
self.assertEqual(max(p), 22) # iterable
|
||||
self.assertEqual(max(*p), 22) # star-able
|
||||
x, y = p
|
||||
self.assertEqual(p, (x, y)) # unpacks like a tuple
|
||||
self.assertEqual((p[0], p[1]), (11, 22)) # indexable like a tuple
|
||||
self.assertRaises(IndexError, p.__getitem__, 3)
|
||||
|
||||
self.assertEqual(p.x, x)
|
||||
self.assertEqual(p.y, y)
|
||||
self.assertRaises(AttributeError, eval, 'p.z', locals())
|
||||
|
||||
def test_odd_sizes(self):
|
||||
Zero = BagType('Zero', ())
|
||||
self.assertEqual(Zero(), ())
|
||||
self.assertEqual(Zero._make([]), ())
|
||||
self.assertEqual(repr(Zero()), 'Zero()')
|
||||
self.assertEqual(Zero()._asdict(), {})
|
||||
self.assertEqual(Zero()._fields, ())
|
||||
|
||||
Dot = BagType('Dot', ('d', ))
|
||||
self.assertEqual(Dot(1), (1, ))
|
||||
self.assertEqual(Dot._make([1]), (1, ))
|
||||
self.assertEqual(Dot(1).d, 1)
|
||||
self.assertEqual(repr(Dot(1)), 'Dot(d=1)')
|
||||
self.assertEqual(Dot(1)._asdict(), {'d': 1})
|
||||
self.assertEqual(Dot(1)._replace(d=999), (999, ))
|
||||
self.assertEqual(Dot(1)._fields, ('d', ))
|
||||
|
||||
n = 5000 if sys.version_info >= (3, 7) else 254
|
||||
names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n)))
|
||||
n = len(names)
|
||||
Big = BagType('Big', names)
|
||||
b = Big(*range(n))
|
||||
self.assertEqual(b, tuple(range(n)))
|
||||
self.assertEqual(Big._make(range(n)), tuple(range(n)))
|
||||
for pos, name in enumerate(names):
|
||||
self.assertEqual(getattr(b, name), pos)
|
||||
repr(b) # make sure repr() doesn't blow-up
|
||||
d = b._asdict()
|
||||
d_expected = dict(zip(names, range(n)))
|
||||
self.assertEqual(d, d_expected)
|
||||
b2 = b._replace(**dict([(names[1], 999), (names[-5], 42)]))
|
||||
b2_expected = list(range(n))
|
||||
b2_expected[1] = 999
|
||||
b2_expected[-5] = 42
|
||||
self.assertEqual(b2, tuple(b2_expected))
|
||||
self.assertEqual(b._fields, tuple(names))
|
||||
|
||||
def test_pickle(self):
|
||||
p = TBag(x=10, y=20, z=30)
|
||||
for module in (pickle, ):
|
||||
loads = getattr(module, 'loads')
|
||||
dumps = getattr(module, 'dumps')
|
||||
for protocol in range(-1, module.HIGHEST_PROTOCOL + 1):
|
||||
q = loads(dumps(p, protocol))
|
||||
self.assertEqual(p, q)
|
||||
self.assertEqual(p._fields, q._fields)
|
||||
self.assertNotIn(b'OrderedDict', dumps(p, protocol))
|
||||
|
||||
def test_copy(self):
|
||||
p = TBag(x=10, y=20, z=30)
|
||||
for copier in copy.copy, copy.deepcopy:
|
||||
q = copier(p)
|
||||
self.assertEqual(p, q)
|
||||
self.assertEqual(p._fields, q._fields)
|
||||
|
||||
def test_name_conflicts(self):
|
||||
# Some names like "self", "cls", "tuple", "itemgetter", and "property"
|
||||
# failed when used as field names. Test to make sure these now work.
|
||||
T = BagType('T', ('itemgetter', 'property', 'self', 'cls', 'tuple'))
|
||||
t = T(1, 2, 3, 4, 5)
|
||||
self.assertEqual(t, (1, 2, 3, 4, 5))
|
||||
newt = t._replace(itemgetter=10, property=20, self=30, cls=40, tuple=50)
|
||||
self.assertEqual(newt, (10, 20, 30, 40, 50))
|
||||
|
||||
# Broader test of all interesting names taken from the code, old
|
||||
# template, and an example
|
||||
words = {
|
||||
'Alias', 'At', 'AttributeError', 'Build', 'Bypass', 'Create', 'Encountered', 'Expected', 'Field', 'For',
|
||||
'Got', 'Helper', 'IronPython', 'Jython', 'KeyError', 'Make', 'Modify', 'Note', 'OrderedDict', 'Point',
|
||||
'Return', 'Returns', 'Type', 'TypeError', 'Used', 'Validate', 'ValueError', 'Variables', 'a', 'accessible',
|
||||
'add', 'added', 'all', 'also', 'an', 'arg_list', 'args', 'arguments', 'automatically', 'be', 'build',
|
||||
'builtins', 'but', 'by', 'cannot', 'class_namespace', 'classmethod', 'cls', 'collections', 'convert',
|
||||
'copy', 'created', 'creation', 'd', 'debugging', 'defined', 'dict', 'dictionary', 'doc', 'docstring',
|
||||
'docstrings', 'duplicate', 'effect', 'either', 'enumerate', 'environments', 'error', 'example', 'exec', 'f',
|
||||
'f_globals', 'field', 'field_names', 'fields', 'formatted', 'frame', 'function', 'functions', 'generate',
|
||||
'getter', 'got', 'greater', 'has', 'help', 'identifiers', 'indexable', 'instance', 'instantiate',
|
||||
'interning', 'introspection', 'isidentifier', 'isinstance', 'itemgetter', 'iterable', 'join', 'keyword',
|
||||
'keywords', 'kwds', 'len', 'like', 'list', 'map', 'maps', 'message', 'metadata', 'method', 'methods',
|
||||
'module', 'module_name', 'must', 'name', 'named', 'namedtuple', 'namedtuple_', 'names', 'namespace',
|
||||
'needs', 'new', 'nicely', 'num_fields', 'number', 'object', 'of', 'operator', 'option', 'p', 'particular',
|
||||
'pickle', 'pickling', 'plain', 'pop', 'positional', 'property', 'r', 'regular', 'rename', 'replace',
|
||||
'replacing', 'repr', 'repr_fmt', 'representation', 'result', 'reuse_itemgetter', 's', 'seen', 'sequence',
|
||||
'set', 'side', 'specified', 'split', 'start', 'startswith', 'step', 'str', 'string', 'strings', 'subclass',
|
||||
'sys', 'targets', 'than', 'the', 'their', 'this', 'to', 'tuple_new', 'type', 'typename', 'underscore',
|
||||
'unexpected', 'unpack', 'up', 'use', 'used', 'user', 'valid', 'values', 'variable', 'verbose', 'where',
|
||||
'which', 'work', 'x', 'y', 'z', 'zip'
|
||||
}
|
||||
sorted_words = tuple(sorted(words))
|
||||
T = BagType('T', sorted_words)
|
||||
# test __new__
|
||||
values = tuple(range(len(words)))
|
||||
t = T(*values)
|
||||
self.assertEqual(t, values)
|
||||
t = T(**dict(zip(T._attrs, values)))
|
||||
self.assertEqual(t, values)
|
||||
# test _make
|
||||
t = T._make(values)
|
||||
self.assertEqual(t, values)
|
||||
# exercise __repr__
|
||||
repr(t)
|
||||
# test _asdict
|
||||
self.assertEqual(t._asdict(), dict(zip(T._fields, values)))
|
||||
# test _replace
|
||||
t = T._make(values)
|
||||
newvalues = tuple(v * 10 for v in values)
|
||||
newt = t._replace(**dict(zip(T._fields, newvalues)))
|
||||
self.assertEqual(newt, newvalues)
|
||||
# test _fields
|
||||
self.assertEqual(T._attrs, sorted_words)
|
||||
# test __getnewargs__
|
||||
self.assertEqual(t.__getnewargs__(), values)
|
||||
|
||||
def test_repr(self):
|
||||
A = BagType('A', ('x', ))
|
||||
self.assertEqual(repr(A(1)), 'A(x=1)')
|
||||
|
||||
# repr should show the name of the subclass
|
||||
class B(A):
|
||||
pass
|
||||
|
||||
self.assertEqual(repr(B(1)), 'B(x=1)')
|
||||
|
||||
def test_namedtuple_subclass_issue_24931(self):
|
||||
class Point(BagType('_Point', ['x', 'y'])):
|
||||
pass
|
||||
|
||||
a = Point(3, 4)
|
||||
self.assertEqual(a._asdict(), OrderedDict([('x', 3), ('y', 4)]))
|
||||
|
||||
a.w = 5
|
||||
self.assertEqual(a.__dict__, {'w': 5})
|
||||
|
||||
def test_annoying_attribute_names(self):
|
||||
self._create(
|
||||
'__slots__', '__getattr__', '_attrs', '_fields', '__new__', '__getnewargs__', '__repr__', '_make', 'get',
|
||||
'_replace', '_asdict', '_cls', 'self', 'tuple'
|
||||
)
|
||||
Reference in New Issue
Block a user