241 lines
6.9 KiB
Python
241 lines
6.9 KiB
Python
import json
|
|
from collections import OrderedDict, namedtuple
|
|
from unittest import TestCase
|
|
|
|
import pytest
|
|
|
|
from bonobo import JsonReader, JsonWriter, LdjsonReader, LdjsonWriter
|
|
from bonobo.constants import EMPTY
|
|
from bonobo.util.testing import ConfigurableNodeTest, ReaderTest, WriterTest
|
|
|
|
FOOBAR = {'foo': 'bar'}
|
|
OD_ABC = OrderedDict((('a', 'A'), ('b', 'B'), ('c', 'C')))
|
|
FOOBAZ = {'foo': 'baz'}
|
|
|
|
incontext = ConfigurableNodeTest.incontext
|
|
|
|
###
|
|
# Standard JSON Readers / Writers
|
|
###
|
|
|
|
|
|
class Json:
|
|
extension = 'json'
|
|
ReaderNodeType = JsonReader
|
|
WriterNodeType = JsonWriter
|
|
|
|
|
|
class JsonReaderDictsTest(Json, ReaderTest, TestCase):
|
|
input_data = '[{"foo": "bar"},\n{"baz": "boz"}]'
|
|
|
|
@incontext()
|
|
def test_nofields(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)]
|
|
|
|
|
|
class JsonReaderListsTest(Json, ReaderTest, TestCase):
|
|
input_data = '[[1,2,3],\n[4,5,6]]'
|
|
|
|
@incontext()
|
|
def test_nofields(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
|
|
|
|
@incontext(output_type=tuple)
|
|
def test_output_type(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
|
|
|
|
|
|
class JsonReaderStringsTest(Json, ReaderTest, TestCase):
|
|
input_data = '[' + ',\n'.join(map(json.dumps, ('foo', 'bar', 'baz'))) + ']'
|
|
|
|
@incontext()
|
|
def test_nofields(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [('foo',), ('bar',), ('baz',)]
|
|
|
|
@incontext(output_type=tuple)
|
|
def test_output_type(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [('foo',), ('bar',), ('baz',)]
|
|
|
|
|
|
class JsonWriterTest(Json, WriterTest, TestCase):
|
|
@incontext()
|
|
def test_fields(self, context):
|
|
context.set_input_fields(['foo', 'bar'])
|
|
context.write_sync(('a', 'b'), ('c', 'd'))
|
|
context.stop()
|
|
|
|
assert self.readlines() == ('[{"foo": "a", "bar": "b"},', '{"foo": "c", "bar": "d"}]')
|
|
|
|
@incontext()
|
|
def test_fields_from_type(self, context):
|
|
context.set_input_type(namedtuple('Point', 'x y'))
|
|
context.write_sync((1, 2), (3, 4))
|
|
context.stop()
|
|
|
|
assert self.readlines() == ('[{"x": 1, "y": 2},', '{"x": 3, "y": 4}]')
|
|
|
|
@incontext()
|
|
def test_nofields_multiple_args(self, context):
|
|
# multiple args are iterated onto and flattened in output
|
|
context.write_sync((FOOBAR, FOOBAR), (OD_ABC, FOOBAR), (FOOBAZ, FOOBAR))
|
|
context.stop()
|
|
|
|
assert self.readlines() == (
|
|
'[{"foo": "bar"},',
|
|
'{"foo": "bar"},',
|
|
'{"a": "A", "b": "B", "c": "C"},',
|
|
'{"foo": "bar"},',
|
|
'{"foo": "baz"},',
|
|
'{"foo": "bar"}]',
|
|
)
|
|
|
|
@incontext()
|
|
def test_nofields_multiple_args_length_mismatch(self, context):
|
|
# if length of input vary, then we get a TypeError (unrecoverable)
|
|
with pytest.raises(TypeError):
|
|
context.write_sync((FOOBAR, FOOBAR), (OD_ABC))
|
|
|
|
@incontext()
|
|
def test_nofields_single_arg(self, context):
|
|
# single args are just dumped, shapes can vary.
|
|
context.write_sync(FOOBAR, OD_ABC, FOOBAZ)
|
|
context.stop()
|
|
|
|
assert self.readlines() == ('[{"foo": "bar"},', '{"a": "A", "b": "B", "c": "C"},', '{"foo": "baz"}]')
|
|
|
|
@incontext()
|
|
def test_nofields_empty_args(self, context):
|
|
# empty calls are ignored
|
|
context.write_sync(EMPTY, EMPTY, EMPTY)
|
|
context.stop()
|
|
|
|
assert self.readlines() == ('[]',)
|
|
|
|
|
|
###
|
|
# Line Delimiter JSON Readers / Writers
|
|
###
|
|
|
|
|
|
class Ldjson:
|
|
extension = 'ldjson'
|
|
ReaderNodeType = LdjsonReader
|
|
WriterNodeType = LdjsonWriter
|
|
|
|
|
|
class LdjsonReaderDictsTest(Ldjson, ReaderTest, TestCase):
|
|
input_data = '{"foo": "bar"}\n{"baz": "boz"}'
|
|
|
|
@incontext()
|
|
def test_nofields(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)]
|
|
|
|
|
|
class LdjsonReaderListsTest(Ldjson, ReaderTest, TestCase):
|
|
input_data = '[1,2,3]\n[4,5,6]'
|
|
|
|
@incontext()
|
|
def test_nofields(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
|
|
|
|
@incontext(output_type=tuple)
|
|
def test_output_type(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)]
|
|
|
|
|
|
class LdjsonReaderStringsTest(Ldjson, ReaderTest, TestCase):
|
|
input_data = '\n'.join(map(json.dumps, ('foo', 'bar', 'baz')))
|
|
|
|
@incontext()
|
|
def test_nofields(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [('foo',), ('bar',), ('baz',)]
|
|
|
|
@incontext(output_type=tuple)
|
|
def test_output_type(self, context):
|
|
context.write_sync(EMPTY)
|
|
context.stop()
|
|
|
|
assert context.get_buffer() == [('foo',), ('bar',), ('baz',)]
|
|
|
|
|
|
class LdjsonWriterTest(Ldjson, WriterTest, TestCase):
|
|
@incontext()
|
|
def test_fields(self, context):
|
|
context.set_input_fields(['foo', 'bar'])
|
|
context.write_sync(('a', 'b'), ('c', 'd'))
|
|
context.stop()
|
|
|
|
assert self.readlines() == ('{"foo": "a", "bar": "b"}', '{"foo": "c", "bar": "d"}')
|
|
|
|
@incontext()
|
|
def test_fields_from_type(self, context):
|
|
context.set_input_type(namedtuple('Point', 'x y'))
|
|
context.write_sync((1, 2), (3, 4))
|
|
context.stop()
|
|
|
|
assert self.readlines() == ('{"x": 1, "y": 2}', '{"x": 3, "y": 4}')
|
|
|
|
@incontext()
|
|
def test_nofields_multiple_args(self, context):
|
|
# multiple args are iterated onto and flattened in output
|
|
context.write_sync((FOOBAR, FOOBAR), (OD_ABC, FOOBAR), (FOOBAZ, FOOBAR))
|
|
context.stop()
|
|
|
|
assert self.readlines() == (
|
|
'{"foo": "bar"}',
|
|
'{"foo": "bar"}',
|
|
'{"a": "A", "b": "B", "c": "C"}',
|
|
'{"foo": "bar"}',
|
|
'{"foo": "baz"}',
|
|
'{"foo": "bar"}',
|
|
)
|
|
|
|
@incontext()
|
|
def test_nofields_multiple_args_length_mismatch(self, context):
|
|
# if length of input vary, then we get a TypeError (unrecoverable)
|
|
with pytest.raises(TypeError):
|
|
context.write_sync((FOOBAR, FOOBAR), (OD_ABC))
|
|
|
|
@incontext()
|
|
def test_nofields_single_arg(self, context):
|
|
# single args are just dumped, shapes can vary.
|
|
context.write_sync(FOOBAR, OD_ABC, FOOBAZ)
|
|
context.stop()
|
|
|
|
assert self.readlines() == ('{"foo": "bar"}', '{"a": "A", "b": "B", "c": "C"}', '{"foo": "baz"}')
|
|
|
|
@incontext()
|
|
def test_nofields_empty_args(self, context):
|
|
# empty calls are ignored
|
|
context.write_sync(EMPTY, EMPTY, EMPTY)
|
|
context.stop()
|
|
|
|
assert self.readlines() == ()
|