[stdlib] Fix I/O related nodes (especially json), there were bad bugs with ioformat.

tests/io/test_csv.py

@@ -1,23 +1,21 @@
 import pytest
 
-from bonobo import Bag, CsvReader, CsvWriter, open_fs, settings
+from bonobo import Bag, CsvReader, CsvWriter, settings
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
-from bonobo.util.testing import CapturingNodeExecutionContext
+from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
+
+csv_tester = FilesystemTester('csv')
+csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar'
 
 
-def test_write_csv_to_file(tmpdir):
-    fs, filename = open_fs(tmpdir), 'output.csv'
+def test_write_csv_to_file_arg0(tmpdir):
+    fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
 
-    writer = CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0)
-    context = NodeExecutionContext(writer, services={'fs': fs})
-
-    context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
-
-    context.start()
-    context.step()
-    context.step()
-    context.stop()
+    with NodeExecutionContext(CsvWriter(path=filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
+        context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
+        context.step()
+        context.step()
 
     with fs.open(filename) as fp:
         assert fp.read() == 'foo\nbar\nbaz\n'
@@ -26,19 +24,33 @@ def test_write_csv_to_file(tmpdir):
         getattr(context, 'file')
 
 
-def test_read_csv_from_file(tmpdir):
-    fs, filename = open_fs(tmpdir), 'input.csv'
-    with fs.open(filename, 'w') as fp:
-        fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar')
+@pytest.mark.parametrize('add_kwargs', ({}, {
+    'ioformat': settings.IOFORMAT_KWARGS,
+},))
+def test_write_csv_to_file_kwargs(tmpdir, add_kwargs):
+    fs, filename, services = csv_tester.get_services_for_writer(tmpdir)
 
-    reader = CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0)
+    with NodeExecutionContext(CsvWriter(path=filename, **add_kwargs), services=services) as context:
+        context.write(BEGIN, Bag(**{'foo': 'bar'}), Bag(**{'foo': 'baz', 'ignore': 'this'}), END)
+        context.step()
+        context.step()
 
-    context = CapturingNodeExecutionContext(reader, services={'fs': fs})
+    with fs.open(filename) as fp:
+        assert fp.read() == 'foo\nbar\nbaz\n'
 
-    context.start()
-    context.write(BEGIN, Bag(), END)
-    context.step()
-    context.stop()
+    with pytest.raises(AttributeError):
+        getattr(context, 'file')
+
+
+def test_read_csv_from_file_arg0(tmpdir):
+    fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
+
+    with CapturingNodeExecutionContext(
+            CsvReader(path=filename, delimiter=',', ioformat=settings.IOFORMAT_ARG0),
+            services=services,
+    ) as context:
+        context.write(BEGIN, Bag(), END)
+        context.step()
 
     assert len(context.send.mock_calls) == 2
 
@@ -59,19 +71,15 @@ def test_read_csv_from_file(tmpdir):
     }
 
 
-def test_read_csv_kwargs_output_formater(tmpdir):
-    fs, filename = open_fs(tmpdir), 'input.csv'
-    with fs.open(filename, 'w') as fp:
-        fp.write('a,b,c\na foo,b foo,c foo\na bar,b bar,c bar')
+def test_read_csv_from_file_kwargs(tmpdir):
+    fs, filename, services = csv_tester.get_services_for_reader(tmpdir)
 
-    reader = CsvReader(path=filename, delimiter=',')
-
-    context = CapturingNodeExecutionContext(reader, services={'fs': fs})
-
-    context.start()
-    context.write(BEGIN, Bag(), END)
-    context.step()
-    context.stop()
+    with CapturingNodeExecutionContext(
+            CsvReader(path=filename, delimiter=','),
+            services=services,
+    ) as context:
+        context.write(BEGIN, Bag(), END)
+        context.step()
 
     assert len(context.send.mock_calls) == 2
 

tests/io/test_file.py

@@ -1,9 +1,22 @@
 import pytest
 
-from bonobo import Bag, FileReader, FileWriter, open_fs
+from bonobo import Bag, FileReader, FileWriter
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
-from bonobo.util.testing import CapturingNodeExecutionContext
+from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
+
+txt_tester = FilesystemTester('txt')
+txt_tester.input_data = 'Hello\nWorld\n'
+
+
+def test_file_writer_contextless(tmpdir):
+    fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
+
+    with FileWriter(path=filename).open(fs) as fp:
+        fp.write('Yosh!')
+
+    with fs.open(filename) as fp:
+        assert fp.read() == 'Yosh!'
 
 
 @pytest.mark.parametrize(
@@ -14,46 +27,23 @@ from bonobo.util.testing import CapturingNodeExecutionContext
     ]
 )
 def test_file_writer_in_context(tmpdir, lines, output):
-    fs, filename = open_fs(tmpdir), 'output.txt'
+    fs, filename, services = txt_tester.get_services_for_writer(tmpdir)
 
-    writer = FileWriter(path=filename)
-    context = NodeExecutionContext(writer, services={'fs': fs})
-
-    context.start()
-    context.write(BEGIN, *map(Bag, lines), END)
-    for _ in range(len(lines)):
-        context.step()
-    context.stop()
+    with NodeExecutionContext(FileWriter(path=filename), services=services) as context:
+        context.write(BEGIN, *map(Bag, lines), END)
+        for _ in range(len(lines)):
+            context.step()
 
     with fs.open(filename) as fp:
         assert fp.read() == output
 
 
-def test_file_writer_out_of_context(tmpdir):
-    fs, filename = open_fs(tmpdir), 'output.txt'
+def test_file_reader(tmpdir):
+    fs, filename, services = txt_tester.get_services_for_reader(tmpdir)
 
-    writer = FileWriter(path=filename)
-
-    with writer.open(fs) as fp:
-        fp.write('Yosh!')
-
-    with fs.open(filename) as fp:
-        assert fp.read() == 'Yosh!'
-
-
-def test_file_reader_in_context(tmpdir):
-    fs, filename = open_fs(tmpdir), 'input.txt'
-
-    with fs.open(filename, 'w') as fp:
-        fp.write('Hello\nWorld\n')
-
-    reader = FileReader(path=filename)
-    context = CapturingNodeExecutionContext(reader, services={'fs': fs})
-
-    context.start()
-    context.write(BEGIN, Bag(), END)
-    context.step()
-    context.stop()
+    with CapturingNodeExecutionContext(FileReader(path=filename), services=services) as context:
+        context.write(BEGIN, Bag(), END)
+        context.step()
 
     assert len(context.send.mock_calls) == 2
 

tests/io/test_json.py

@@ -1,44 +1,48 @@
 import pytest
 
-from bonobo import Bag, JsonReader, JsonWriter, open_fs, settings
+from bonobo import Bag, JsonReader, JsonWriter, settings
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
-from bonobo.util.testing import CapturingNodeExecutionContext
+from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
+
+json_tester = FilesystemTester('json')
+json_tester.input_data = '''[{"x": "foo"},{"x": "bar"}]'''
 
 
-def test_write_json_to_file(tmpdir):
-    fs, filename = open_fs(tmpdir), 'output.json'
+def test_write_json_arg0(tmpdir):
+    fs, filename, services = json_tester.get_services_for_writer(tmpdir)
 
-    writer = JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0)
-    context = NodeExecutionContext(writer, services={'fs': fs})
-
-    context.start()
-    context.write(BEGIN, Bag({'foo': 'bar'}), END)
-    context.step()
-    context.stop()
+    with NodeExecutionContext(JsonWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
+        context.write(BEGIN, Bag({'foo': 'bar'}), END)
+        context.step()
 
     with fs.open(filename) as fp:
         assert fp.read() == '[{"foo": "bar"}]'
 
-    with pytest.raises(AttributeError):
-        getattr(context, 'file')
 
-    with pytest.raises(AttributeError):
-        getattr(context, 'first')
+@pytest.mark.parametrize('add_kwargs', ({}, {
+    'ioformat': settings.IOFORMAT_KWARGS,
+}, ))
+def test_write_json_kwargs(tmpdir, add_kwargs):
+    fs, filename, services = json_tester.get_services_for_writer(tmpdir)
+
+    with NodeExecutionContext(JsonWriter(filename, **add_kwargs), services=services) as context:
+        context.write(BEGIN, Bag(**{'foo': 'bar'}), END)
+        context.step()
+
+    with fs.open(filename) as fp:
+        assert fp.read() == '[{"foo": "bar"}]'
 
 
-def test_read_json_from_file(tmpdir):
-    fs, filename = open_fs(tmpdir), 'input.json'
-    with fs.open(filename, 'w') as fp:
-        fp.write('[{"x": "foo"},{"x": "bar"}]')
-    reader = JsonReader(filename, ioformat=settings.IOFORMAT_ARG0)
+def test_read_json_arg0(tmpdir):
+    fs, filename, services = json_tester.get_services_for_reader(tmpdir)
 
-    context = CapturingNodeExecutionContext(reader, services={'fs': fs})
-
-    context.start()
-    context.write(BEGIN, Bag(), END)
-    context.step()
-    context.stop()
+    with CapturingNodeExecutionContext(
+        JsonReader(filename, ioformat=settings.IOFORMAT_ARG0),
+        services=services,
+    ) as context:
+        context.write(BEGIN, Bag(), END)
+        context.step()
 
     assert len(context.send.mock_calls) == 2
 

tests/io/test_pickle.py

@@ -2,24 +2,22 @@ import pickle
 
 import pytest
 
-from bonobo import Bag, PickleReader, PickleWriter, open_fs, settings
+from bonobo import Bag, PickleReader, PickleWriter, settings
 from bonobo.constants import BEGIN, END
 from bonobo.execution.node import NodeExecutionContext
-from bonobo.util.testing import CapturingNodeExecutionContext
+from bonobo.util.testing import CapturingNodeExecutionContext, FilesystemTester
+
+pickle_tester = FilesystemTester('pkl', mode='wb')
+pickle_tester.input_data = pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']])
 
 
 def test_write_pickled_dict_to_file(tmpdir):
-    fs, filename = open_fs(tmpdir), 'output.pkl'
+    fs, filename, services = pickle_tester.get_services_for_writer(tmpdir)
 
-    writer = PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0)
-    context = NodeExecutionContext(writer, services={'fs': fs})
-
-    context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
-
-    context.start()
-    context.step()
-    context.step()
-    context.stop()
+    with NodeExecutionContext(PickleWriter(filename, ioformat=settings.IOFORMAT_ARG0), services=services) as context:
+        context.write(BEGIN, Bag({'foo': 'bar'}), Bag({'foo': 'baz', 'ignore': 'this'}), END)
+        context.step()
+        context.step()
 
     with fs.open(filename, 'rb') as fp:
         assert pickle.loads(fp.read()) == {'foo': 'bar'}
@@ -29,18 +27,13 @@ def test_write_pickled_dict_to_file(tmpdir):
 
 
 def test_read_pickled_list_from_file(tmpdir):
-    fs, filename = open_fs(tmpdir), 'input.pkl'
-    with fs.open(filename, 'wb') as fp:
-        fp.write(pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']]))
+    fs, filename, services = pickle_tester.get_services_for_reader(tmpdir)
 
-    reader = PickleReader(filename, ioformat=settings.IOFORMAT_ARG0)
-
-    context = CapturingNodeExecutionContext(reader, services={'fs': fs})
-
-    context.start()
-    context.write(BEGIN, Bag(), END)
-    context.step()
-    context.stop()
+    with CapturingNodeExecutionContext(
+        PickleReader(filename, ioformat=settings.IOFORMAT_ARG0), services=services
+    ) as context:
+        context.write(BEGIN, Bag(), END)
+        context.step()
 
     assert len(context.send.mock_calls) == 2