Refactoring the runner to go more towards standard python, also adds the ability to use bonobo argument parser from standard python execution.

Projectfile

@@ -66,8 +66,6 @@ python.add_requirements(
 
 # Following requirements are not enforced, because some dependencies enforce them so we don't want to break
 # the packaging in case it changes in dep.
-python.add_requirements(
+python.add_requirements('colorama >=0.3', )
-    'colorama >=0.3',
-)
 
 # vim: ft=python:

bonobo/_api.py

@@ -1,13 +1,10 @@
-import argparse
-from contextlib import contextmanager
-
-import os
 from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
     PickleReader, PickleWriter, PrettyPrinter, RateLimited, Tee, arg0_to_kwargs, count, identity, kwargs_to_arg0, noop
 from bonobo.nodes import LdjsonReader, LdjsonWriter
 from bonobo.strategies import create_strategy
 from bonobo.structs import Bag, ErrorBag, Graph, Token
 from bonobo.util import get_name
+from bonobo.util.environ import parse_args, get_argument_parser
 
 __all__ = []
 
@@ -22,59 +19,6 @@ def register_api_group(*args):
         register_api(attr)
 
 
-@register_api
-def get_argument_parser(parser=None):
-    if parser is None:
-        import argparse
-        parser = argparse.ArgumentParser()
-
-    parser.add_argument('--default-env-file', action='append')
-    parser.add_argument('--default-env', action='append')
-    parser.add_argument('--env-file', action='append')
-    parser.add_argument('--env', '-e', action='append')
-
-    return parser
-
-
-@register_api
-@contextmanager
-def parse_args(parser, *, args=None, namespace=None):
-    options = parser.parse_args(args=args, namespace=namespace)
-
-    with patch_environ(options) as options:
-        yield options
-
-
-@register_api
-@contextmanager
-def patch_environ(options):
-    from dotenv import load_dotenv
-    from bonobo.commands import set_env_var
-
-    options = options if isinstance(options, dict) else options.__dict__
-
-    default_env_file = options.pop('default_env_file', [])
-    default_env = options.pop('default_env', [])
-    env_file = options.pop('env_file', [])
-    env = options.pop('env', [])
-
-    if default_env_file:
-        for f in default_env_file:
-            load_dotenv(os.path.join(os.getcwd(), f))
-    if default_env:
-        for e in default_env:
-            set_env_var(e)
-    if env_file:
-        for f in env_file:
-            load_dotenv(os.path.join(os.getcwd(), f), override=True)
-    if env:
-        for e in env:
-            set_env_var(e, override=True)
-
-    yield options
-    ## TODO XXX put it back !!!
-
-
 @register_api
 def run(graph, *, plugins=None, services=None, strategy=None):
     """
@@ -126,6 +70,24 @@ def run(graph, *, plugins=None, services=None, strategy=None):
     return strategy.execute(graph, plugins=plugins, services=services)
 
 
+def _inspect_as_graph(graph):
+    return graph._repr_dot_()
+
+
+_inspect_formats = {'graph': _inspect_as_graph}
+
+
+@register_api
+def inspect(graph, *, format):
+    if not format in _inspect_formats:
+        raise NotImplementedError(
+            'Output format {} not implemented. Choices are: {}.'.format(
+                format, ', '.join(sorted(_inspect_formats.keys()))
+            )
+        )
+    print(_inspect_formats[format](graph))
+
+
 # bonobo.structs
 register_api_group(Bag, ErrorBag, Graph, Token)
 
@@ -205,3 +167,6 @@ def get_examples_path(*pathsegments):
 @register_api
 def open_examples_fs(*pathsegments):
     return open_fs(get_examples_path(*pathsegments))
+
+
+register_api_group(get_argument_parser, parse_args)

bonobo/commands/__init__.py

@@ -1,91 +1,19 @@
 import argparse
-import codecs
-import os
-import os.path
-import runpy
-import sys
-from contextlib import contextmanager
 
-from bonobo import settings, logging, get_argument_parser, patch_environ
+from bonobo import settings, logging
-from bonobo.constants import DEFAULT_SERVICES_FILENAME, DEFAULT_SERVICES_ATTR
+from bonobo.commands.base import BaseCommand, BaseGraphCommand
-from bonobo.util import get_name
 
 logger = logging.get_logger()
 
 
-class BaseCommand:
-    @property
-    def logger(self):
-        try:
-            return self._logger
-        except AttributeError:
-            self._logger = logging.get_logger(get_name(self))
-            return self._logger
-
-    def add_arguments(self, parser):
-        """
-        Entry point for subclassed commands to add custom arguments.
-        """
-        pass
-
-    def handle(self, *args, **options):
-        """
-        The actual logic of the command. Subclasses must implement this method.
-        """
-        raise NotImplementedError('Subclasses of BaseCommand must provide a handle() method')
-
-
-class BaseGraphCommand(BaseCommand):
-    required = True
-
-    def add_arguments(self, parser):
-        # target arguments (cannot provide both).
-        source_group = parser.add_mutually_exclusive_group(required=self.required)
-        source_group.add_argument('file', nargs='?', type=str)
-        source_group.add_argument('-m', dest='mod', type=str)
-
-        # add arguments to enforce system environment.
-        parser = get_argument_parser(parser)
-
-        return parser
-
-    def _run_path(self, file):
-        return runpy.run_path(file, run_name='__main__')
-
-    def _run_module(self, mod):
-        return runpy.run_module(mod, run_name='__main__')
-
-    def read(self, *, file, mod, args=None, **options):
-        _graph, _options = None, None
-
-        def _record(graph, **options):
-            nonlocal _graph, _options
-            _graph, _options = graph, options
-
-        with _override_runner(_record), patch_environ(options):
-            _argv = sys.argv
-            try:
-                if file:
-                    sys.argv = [file] + list(args) if args else [file]
-                    self._run_path(file)
-                elif mod:
-                    sys.argv = [mod, *(args or ())]
-                    self._run_module(mod)
-                else:
-                    raise RuntimeError('No target provided.')
-            finally:
-                sys.argv = _argv
-
-        if _graph is None:
-            raise RuntimeError('Could not find graph.')
-
-        return _graph, _options
-
-    def handle(self, *args, **options):
-        pass
-
-
 def entrypoint(args=None):
+    """
+    Main callable for "bonobo" entrypoint.
+
+    Will load commands from "bonobo.commands" entrypoints, using stevedore.
+
+    """
+
     parser = argparse.ArgumentParser()
     parser.add_argument('--debug', '-D', action='store_true')
 
@@ -113,8 +41,7 @@ def entrypoint(args=None):
     mgr = ExtensionManager(namespace='bonobo.commands')
     mgr.map(register_extension)
 
-    parsed_args, remaining = parser.parse_known_args(args)
+    parsed_args = parser.parse_args(args).__dict__
-    parsed_args = parsed_args.__dict__
 
     if parsed_args.pop('debug', False):
         settings.DEBUG.set(True)
@@ -123,45 +50,6 @@ def entrypoint(args=None):
 
     logger.debug('Command: ' + parsed_args['command'] + ' Arguments: ' + repr(parsed_args))
 
-    # Get command handler
+    # Get command handler, execute, rince.
     command = commands[parsed_args.pop('command')]
-
+    command(**parsed_args)
-    if len(remaining):
-        command(_remaining_args=remaining, **parsed_args)
-    else:
-        command(**parsed_args)
-
-
-@contextmanager
-def _override_runner(runner):
-    import bonobo
-    _get_argument_parser = bonobo.get_argument_parser
-    _run = bonobo.run
-    try:
-        def get_argument_parser(parser=None):
-            return parser or argparse.ArgumentParser()
-
-        bonobo.get_argument_parser = get_argument_parser
-        bonobo.run = runner
-
-        yield runner
-    finally:
-        bonobo.get_argument_parser = _get_argument_parser
-        bonobo.run = _run
-
-
-def set_env_var(e, override=False):
-    __escape_decoder = codecs.getdecoder('unicode_escape')
-    ename, evalue = e.split('=', 1)
-
-    def decode_escaped(escaped):
-        return __escape_decoder(escaped)[0]
-
-    if len(evalue) > 0:
-        if evalue[0] == evalue[len(evalue) - 1] in ['"', "'"]:
-            evalue = decode_escaped(evalue[1:-1])
-
-    if override:
-        os.environ[ename] = evalue
-    else:
-        os.environ.setdefault(ename, evalue)

bonobo/commands/base.py

@@ -0,0 +1,129 @@
+import argparse
+import runpy
+import sys
+from contextlib import contextmanager
+
+import bonobo.util.environ
+from bonobo import logging
+from bonobo.util.environ import get_argument_parser, parse_args
+from bonobo.util import get_name
+
+
+class BaseCommand:
+    """
+    Base class for CLI commands.
+
+    """
+
+    @property
+    def logger(self):
+        try:
+            return self._logger
+        except AttributeError:
+            self._logger = logging.get_logger(get_name(self))
+            return self._logger
+
+    def add_arguments(self, parser):
+        """
+        Entry point for subclassed commands to add custom arguments.
+        """
+        pass
+
+    def handle(self, *args, **options):
+        """
+        The actual logic of the command. Subclasses must implement this method.
+        """
+        raise NotImplementedError('Subclasses of BaseCommand must provide a handle() method')
+
+
+class BaseGraphCommand(BaseCommand):
+    """
+    Base class for CLI commands that depends on a graph definition, either from a file or from a module.
+
+    """
+    required = True
+    handler = None
+
+    def add_arguments(self, parser):
+        # target arguments (cannot provide both).
+        source_group = parser.add_mutually_exclusive_group(required=self.required)
+        source_group.add_argument('file', nargs='?', type=str)
+        source_group.add_argument('-m', dest='mod', type=str)
+
+        # add arguments to enforce system environment.
+        parser = get_argument_parser(parser)
+
+        return parser
+
+    def parse_options(self, **options):
+        return options
+
+    def handle(self, file, mod, **options):
+        options = self.parse_options(**options)
+        with self.read(file, mod, **options) as (graph, graph_execution_options, options):
+            return self.do_handle(graph, **graph_execution_options, **options)
+
+    def do_handle(self, graph, **options):
+        if not self.handler:
+            raise RuntimeError('{} has no handler defined.'.format(get_name(self)))
+        return self.handler(graph, **options)
+
+    @contextmanager
+    def read(self, file, mod, **options):
+        _graph, _graph_execution_options = None, None
+
+        def _record(graph, **graph_execution_options):
+            nonlocal _graph, _graph_execution_options
+            _graph, _graph_execution_options = graph, graph_execution_options
+
+        with _override_runner(_record), parse_args(options) as options:
+            _argv = sys.argv
+            try:
+                if file:
+                    sys.argv = [file]
+                    self._run_path(file)
+                elif mod:
+                    sys.argv = [mod]
+                    self._run_module(mod)
+                else:
+                    raise RuntimeError('No target provided.')
+            finally:
+                sys.argv = _argv
+
+            if _graph is None:
+                raise RuntimeError('Could not find graph.')
+
+            yield _graph, _graph_execution_options, options
+
+    def _run_path(self, file):
+        return runpy.run_path(file, run_name='__main__')
+
+    def _run_module(self, mod):
+        return runpy.run_module(mod, run_name='__main__')
+
+
+@contextmanager
+def _override_runner(runner):
+    """
+    Context manager that monkey patches `bonobo.run` function with our current command logic.
+
+    :param runner: the callable that will handle the `run()` logic.
+    """
+    import bonobo
+
+    _get_argument_parser = bonobo.util.environ.get_argument_parser
+    _run = bonobo.run
+    try:
+        # Original get_argument_parser would create or update an argument parser with environment options, but here we
+        # already had them parsed so let's patch with something that creates an empty one instead.
+        def get_argument_parser(parser=None):
+            return parser or argparse.ArgumentParser()
+
+        bonobo.util.environ.get_argument_parser = get_argument_parser
+        bonobo.run = runner
+
+        yield runner
+    finally:
+        # Restore our saved values.
+        bonobo.util.environ.get_argument_parser = _get_argument_parser
+        bonobo.run = _run

bonobo/commands/convert.py

@@ -48,8 +48,17 @@ class ConvertCommand(BaseCommand):
             help='Add a named option to the writer factory.',
         )
 
-    def handle(self, input_filename, output_filename, reader=None, reader_option=None, writer=None, writer_option=None,
+    def handle(
-               option=None, transformation=None):
+        self,
+        input_filename,
+        output_filename,
+        reader=None,
+        reader_option=None,
+        writer=None,
+        writer_option=None,
+        option=None,
+        transformation=None
+    ):
         reader_factory = default_registry.get_reader_factory_for(input_filename, format=reader)
         reader_options = _resolve_options((option or []) + (reader_option or []))
 

bonobo/commands/inspect.py

@@ -1,21 +1,15 @@
+import bonobo
 from bonobo.commands import BaseGraphCommand
 
-OUTPUT_GRAPH = 'graphviz'
-
 
 class InspectCommand(BaseGraphCommand):
+    handler = staticmethod(bonobo.inspect)
+
     def add_arguments(self, parser):
         super(InspectCommand, self).add_arguments(parser)
-        parser.add_argument('--graph', '-g', dest='output', action='store_const', const=OUTPUT_GRAPH)
+        parser.add_argument('--graph', '-g', dest='format', action='store_const', const='graph')
-
-    def handle(self, output=None, **options):
-        if output is None:
-            raise ValueError('Output type must be provided (try --graph/-g).')
-
-        graph, params = self.read(**options)
-
-        if output == OUTPUT_GRAPH:
-            print(graph._repr_dot_())
-        else:
-            raise NotImplementedError('Output type not implemented.')
 
+    def parse_options(self, **options):
+        if not options.get('format'):
+            raise RuntimeError('You must provide a format (try --graph).')
+        return options

bonobo/commands/run.py

@@ -6,6 +6,7 @@ from bonobo.commands import BaseGraphCommand
 
 class RunCommand(BaseGraphCommand):
     install = False
+    handler = staticmethod(bonobo.run)
 
     def add_arguments(self, parser):
         super(RunCommand, self).add_arguments(parser)
@@ -16,7 +17,15 @@ class RunCommand(BaseGraphCommand):
 
         parser.add_argument('--install', '-I', action='store_true')
 
+    def parse_options(self, *, quiet=False, verbose=False, install=False, **options):
+        from bonobo import settings
+        settings.QUIET.set_if_true(quiet)
+        settings.DEBUG.set_if_true(verbose)
+        self.install = install
+        return options
+
     def _run_path(self, file):
+        # add install logic
         if self.install:
             if os.path.isdir(file):
                 requirements = os.path.join(file, 'requirements.txt')
@@ -27,24 +36,12 @@ class RunCommand(BaseGraphCommand):
         return super()._run_path(file)
 
     def _run_module(self, mod):
+        # install not implemented for a module, not sure it even make sense.
         if self.install:
             raise RuntimeError('--install behaviour when running a module is not defined.')
 
         return super()._run_module(mod)
 
-    def handle(self, quiet=False, verbose=False, install=False, _remaining_args=None, **options):
-        from bonobo import settings
-
-        settings.QUIET.set_if_true(quiet)
-        settings.DEBUG.set_if_true(verbose)
-        self.install = install
-
-        graph, params = self.read(args=_remaining_args, **options)
-
-        params['plugins'] = set(params.pop('plugins', ())).union(set(options.pop('plugins', ())))
-
-        return bonobo.run(graph, **params)
-
 
 def register_generic_run_arguments(parser, required=True):
     """

bonobo/examples/environ.py

@@ -0,0 +1,28 @@
+import os
+
+import bonobo
+
+
+def extract_environ():
+    yield from sorted(os.environ.items())
+
+
+def get_graph():
+    """
+    This function builds the graph that needs to be executed.
+
+    :return: bonobo.Graph
+
+    """
+    graph = bonobo.Graph()
+    graph.add_chain(extract_environ, print)
+
+    return graph
+
+
+# The __main__ block actually execute the graph.
+if __name__ == '__main__':
+    parser = bonobo.get_argument_parser()
+    parser.add_argument('-v', action='append', dest='vars')
+    with bonobo.parse_args(parser):
+        bonobo.run(get_graph())

bonobo/examples/tutorials/tut02e02_write.py

@@ -8,9 +8,7 @@ def split_one(line):
 graph = bonobo.Graph(
     bonobo.FileReader('coffeeshops.txt'),
     split_one,
-    bonobo.JsonWriter(
+    bonobo.JsonWriter('coffeeshops.json', fs='fs.output'),
-        'coffeeshops.json', fs='fs.output'
-    ),
 )
 
 

bonobo/examples/types/__main__.py

@@ -1,3 +1,7 @@
-from bonobo.util.python import require
+import bonobo
+from bonobo.examples.types.strings import get_graph
 
-graph = require('strings').graph
+if __name__ == '__main__':
+    parser = bonobo.get_argument_parser()
+    with bonobo.parse_args(parser):
+        bonobo.run(get_graph())

bonobo/examples/types/strings.py

@@ -14,7 +14,7 @@ Example on how to use symple python strings to communicate between transformatio
 """
 from random import randint
 
-from bonobo import Graph
+import bonobo
 
 
 def extract():
@@ -31,9 +31,11 @@ def load(s: str):
     print(s)
 
 
-graph = Graph(extract, transform, load)
+def get_graph():
+    return bonobo.Graph(extract, transform, load)
+
 
 if __name__ == '__main__':
-    from bonobo import run
+    parser = bonobo.get_argument_parser()
-
+    with bonobo.parse_args(parser):
-    run(graph)
+        bonobo.run(get_graph())

bonobo/execution/node.py

@@ -138,7 +138,7 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
 
 
 def isflag(param):
-    return isinstance(param, Token) and param in (NOT_MODIFIED,)
+    return isinstance(param, Token) and param in (NOT_MODIFIED, )
 
 
 def split_tokens(output):
@@ -150,11 +150,11 @@ def split_tokens(output):
     """
     if isinstance(output, Token):
         # just a flag
-        return (output,), ()
+        return (output, ), ()
 
     if not istuple(output):
         # no flag
-        return (), (output,)
+        return (), (output, )
 
     i = 0
     while isflag(output[i]):

bonobo/nodes/io/json.py

@@ -58,6 +58,7 @@ class LdjsonReader(FileReader):
 
 class LdjsonWriter(FileWriter):
     """Write a stream of JSON objects, one object per line."""
+
     def write(self, fs, file, lineno, **row):
         lineno += 1  # class-level variable
         file.write(json.dumps(row) + '\n')

bonobo/util/environ.py

@@ -0,0 +1,154 @@
+import argparse
+import codecs
+import os
+import re
+import warnings
+from contextlib import contextmanager
+
+__escape_decoder = codecs.getdecoder('unicode_escape')
+__posix_variable = re.compile('\$\{[^\}]*\}')
+
+
+def parse_var(var):
+    name, value = var.split('=', 1)
+
+    def decode_escaped(escaped):
+        return __escape_decoder(escaped)[0]
+
+    if len(value) > 1:
+        c = value[0]
+
+        if c in ['"', "'"] and value[-1] == c:
+            value = decode_escaped(value[1:-1])
+
+    return name, value
+
+
+def load_env_from_file(filename):
+    """
+    Read an env file into a collection of (name, value) tuples.
+    """
+    if not os.path.exists(filename):
+        raise FileNotFoundError('Environment file {} does not exist.'.format(filename))
+
+    with open(filename) as f:
+        for lineno, line in enumerate(f):
+            line = line.strip()
+            if not line or line.startswith('#'):
+                continue
+            if '=' not in line:
+                raise SyntaxError('Invalid environment file syntax in {} at line {}.'.format(filename, lineno + 1))
+
+            name, value = parse_var(line)
+
+            yield name, value
+
+
+_parser = None
+
+
+def get_argument_parser(parser=None):
+    """
+    Creates an argument parser with arguments to override the system environment.
+
+    :api: bonobo.get_argument_parser
+
+    :param _parser:
+    :return:
+    """
+    if parser is None:
+        import argparse
+        parser = argparse.ArgumentParser()
+
+    # Store globally to be able to warn the user about the fact he's probably wrong not to pass a parser to
+    # parse_args(), later.
+    global _parser
+    _parser = parser
+
+    _parser.add_argument('--default-env-file', '-E', action='append')
+    _parser.add_argument('--default-env', action='append')
+    _parser.add_argument('--env-file', action='append')
+    _parser.add_argument('--env', '-e', action='append')
+
+    return _parser
+
+
+@contextmanager
+def parse_args(mixed=None):
+    """
+    Context manager to extract and apply environment related options from the provided argparser result.
+
+    A dictionnary with unknown options will be yielded, so the remaining options can be used by the caller.
+
+    :api: bonobo.patch_environ
+
+    :param mixed: ArgumentParser instance, Namespace, or dict.
+    :return:
+    """
+
+    if mixed is None:
+        global _parser
+        if _parser is not None:
+            warnings.warn(
+                'You are calling bonobo.parse_args() without a parser argument, but it looks like you created a parser before. You probably want to pass your parser to this call, or if creating a new parser here is really what you want to do, please create a new one explicitely to silence this warning.'
+            )
+        # use the api from bonobo namespace, in case a command patched it.
+        import bonobo
+        mixed = bonobo.get_argument_parser()
+
+    if isinstance(mixed, argparse.ArgumentParser):
+        options = mixed.parse_args()
+    else:
+        options = mixed
+
+    if not isinstance(options, dict):
+        options = options.__dict__
+
+    # make a copy so we don't polute our parent variables.
+    options = dict(options)
+
+    # storage for values before patch.
+    _backup = {}
+
+    # Priority order: --env > --env-file > system > --default-env > --default-env-file
+    #
+    # * The code below is reading default-env before default-env-file as if the first sets something, default-env-file
+    #   won't override it.
+    # * Then, env-file is read from before env, as the behaviour will be the oposite (env will override a var even if
+    #   env-file sets something.)
+    try:
+        # Set default environment
+        for name, value in map(parse_var, options.pop('default_env', []) or []):
+            if not name in os.environ:
+                if not name in _backup:
+                    _backup[name] = os.environ.get(name, None)
+                os.environ[name] = value
+
+        # Read and set default environment from file(s)
+        for filename in options.pop('default_env_file', []) or []:
+            for name, value in load_env_from_file(filename):
+                if not name in os.environ:
+                    if not name in _backup:
+                        _backup[name] = os.environ.get(name, None)
+                    os.environ[name] = value
+
+        # Read and set environment from file(s)
+        for filename in options.pop('env_file', []) or []:
+            for name, value in load_env_from_file(filename):
+                if not name in _backup:
+                    _backup[name] = os.environ.get(name, None)
+                os.environ[name] = value
+
+        # Set environment
+        for name, value in map(parse_var, options.pop('env', []) or []):
+            if not name in _backup:
+                _backup[name] = os.environ.get(name, None)
+            os.environ[name] = value
+
+        yield options
+    finally:
+        for name, value in _backup.items():
+            if value is None:
+                del os.environ[name]
+            else:
+                os.environ[name] = value

tests/features/test_not_modified.py

@@ -9,8 +9,12 @@ def useless(*args, **kwargs):
 def test_not_modified():
     input_messages = [
         ('foo', 'bar'),
-        {'foo': 'bar'},
+        {
-        ('foo', {'bar': 'baz'}),
+            'foo': 'bar'
+        },
+        ('foo', {
+            'bar': 'baz'
+        }),
         (),
     ]
 
@@ -18,7 +22,3 @@ def test_not_modified():
         context.write_sync(*input_messages)
 
     assert context.get_buffer() == input_messages
-
-
-
-

tests/io/test_csv.py

@@ -51,11 +51,9 @@ def test_read_csv_from_file_kwargs(tmpdir):
             'a': 'a foo',
             'b': 'b foo',
             'c': 'c foo',
-        },
+        }, {
-        {
             'a': 'a bar',
             'b': 'b bar',
             'c': 'c bar',
         }
     ]
-

tests/io/test_json.py

@@ -20,10 +20,10 @@ def test_write_json_ioformat_arg0(tmpdir):
 
 
 @pytest.mark.parametrize('add_kwargs', (
-        {},
+    {},
-        {
+    {
-            'ioformat': settings.IOFORMAT_KWARGS,
+        'ioformat': settings.IOFORMAT_KWARGS,
-        },
+    },
 ))
 def test_write_json_kwargs(tmpdir, add_kwargs):
     fs, filename, services = json_tester.get_services_for_writer(tmpdir)
@@ -41,8 +41,7 @@ stream_json_tester.input_data = '''{"foo": "bar"}\n{"baz": "boz"}'''
 
 def test_read_stream_json(tmpdir):
     fs, filename, services = stream_json_tester.get_services_for_reader(tmpdir)
-    with BufferingNodeExecutionContext(LdjsonReader(filename),
+    with BufferingNodeExecutionContext(LdjsonReader(filename), services=services) as context:
-                                       services=services) as context:
         context.write_sync(tuple())
         actual = context.get_buffer()
 
@@ -53,10 +52,11 @@ def test_read_stream_json(tmpdir):
 def test_write_stream_json(tmpdir):
     fs, filename, services = stream_json_tester.get_services_for_reader(tmpdir)
 
-    with BufferingNodeExecutionContext(LdjsonWriter(filename),
+    with BufferingNodeExecutionContext(LdjsonWriter(filename), services=services) as context:
-                                       services=services) as context:
         context.write_sync(
-            {'foo': 'bar'},
+            {
+                'foo': 'bar'
+            },
             {'baz': 'boz'},
         )
 

tests/test_commands.py

@@ -45,7 +45,6 @@ def runner_module(args):
 
 
 all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module])
-single_runner = pytest.mark.parametrize('runner', [runner_module])
 
 
 def test_entrypoint():
@@ -158,238 +157,141 @@ def test_download_fails_non_example(runner):
         runner('download', '/something/entirely/different.txt')
 
 
-@all_runners
+@pytest.fixture
-class TestDefaultEnvFile(object):
+def env1(tmpdir):
-    def test_run_file_with_default_env_file(self, runner):
+    env_file = tmpdir.join('.env_one')
-        out, err = runner(
+    env_file.write('\n'.join((
-            'run', '--quiet', '--default-env-file', '.env_one',
+        'SECRET=unknown',
-            get_examples_path('environment/env_files/get_passed_env_file.py')
+        'PASSWORD=sweet',
-        )
+        'PATH=first',
-        out = out.split('\n')
+    )))
-        assert out[0] == '321'
+    return str(env_file)
-        assert out[1] == 'sweetpassword'
-        assert out[2] != 'marzo'
 
-    def test_run_file_with_multiple_default_env_files(self, runner):
-        out, err = runner(
-            'run', '--quiet', '--default-env-file', '.env_one', '--default-env-file', '.env_two',
-            get_examples_path('environment/env_files/get_passed_env_file.py')
-        )
-        out = out.split('\n')
-        assert out[0] == '321'
-        assert out[1] == 'sweetpassword'
-        assert out[2] != 'marzo'
 
-    def test_run_module_with_default_env_file(self, runner):
+@pytest.fixture
-        out, err = runner(
+def env2(tmpdir):
-            'run', '--quiet', '-m', 'bonobo.examples.environment.env_files.get_passed_env_file', '--default-env-file',
+    env_file = tmpdir.join('.env_two')
-            '.env_one'
+    env_file.write('\n'.join((
-        )
+        'PASSWORD=bitter',
-        out = out.split('\n')
+        "PATH='second'",
-        assert out[0] == '321'
+    )))
-        assert out[1] == 'sweetpassword'
+    return str(env_file)
-        assert out[2] != 'marzo'
 
-    def test_run_module_with_multiple_default_env_files(self, runner):
+
-        out, err = runner(
+all_environ_targets = pytest.mark.parametrize(
-            'run',
+    'target', [
-            '--quiet',
+        (get_examples_path('environ.py'), ),
+        (
             '-m',
-            'bonobo.examples.environment.env_files.get_passed_env_file',
+            'bonobo.examples.environ',
-            '--default-env-file',
+        ),
-            '.env_one',
+    ]
-            '--default-env-file',
+)
-            '.env_two',
-        )
-        out = out.split('\n')
-        assert out[0] == '321'
-        assert out[1] == 'sweetpassword'
-        assert out[2] != 'marzo'
 
 
 @all_runners
-class TestEnvFile(object):
+@all_environ_targets
-    def test_run_file_with_file(self, runner):
+class EnvironmentTestCase():
-        out, err = runner(
+    def run_quiet(self, runner, *args):
-            'run',
+        return runner('run', '--quiet', *args)
-            '--quiet',
+
-            get_examples_path('environment/env_files/get_passed_env_file.py'),
+    def run_environ(self, runner, *args, environ=None):
-            '--env-file',
+        _environ = {'PATH': '/usr/bin'}
-            '.env_one',
+        if environ:
+            _environ.update(environ)
+
+        with patch.dict('os.environ', _environ, clear=True):
+            out, err = self.run_quiet(runner, *args)
+            assert 'SECRET' not in os.environ
+            assert 'PASSWORD' not in os.environ
+            if 'PATH' in _environ:
+                assert 'PATH' in os.environ
+                assert os.environ['PATH'] == _environ['PATH']
+
+        assert err == ''
+        return dict(map(lambda line: line.split(' ', 1), filter(None, out.split('\n'))))
+
+
+class TestDefaultEnvFile(EnvironmentTestCase):
+    def test_run_with_default_env_file(self, runner, target, env1):
+        env = self.run_environ(runner, *target, '--default-env-file', env1)
+        assert env.get('SECRET') == 'unknown'
+        assert env.get('PASSWORD') == 'sweet'
+        assert env.get('PATH') == '/usr/bin'
+
+    def test_run_with_multiple_default_env_files(self, runner, target, env1, env2):
+        env = self.run_environ(runner, *target, '--default-env-file', env1, '--default-env-file', env2)
+        assert env.get('SECRET') == 'unknown'
+        assert env.get('PASSWORD') == 'sweet'
+        assert env.get('PATH') == '/usr/bin'
+
+        env = self.run_environ(runner, *target, '--default-env-file', env2, '--default-env-file', env1)
+        assert env.get('SECRET') == 'unknown'
+        assert env.get('PASSWORD') == 'bitter'
+        assert env.get('PATH') == '/usr/bin'
+
+
+class TestEnvFile(EnvironmentTestCase):
+    def test_run_with_file(self, runner, target, env1):
+        env = self.run_environ(runner, *target, '--env-file', env1)
+        assert env.get('SECRET') == 'unknown'
+        assert env.get('PASSWORD') == 'sweet'
+        assert env.get('PATH') == 'first'
+
+    def test_run_with_multiple_files(self, runner, target, env1, env2):
+        env = self.run_environ(runner, *target, '--env-file', env1, '--env-file', env2)
+        assert env.get('SECRET') == 'unknown'
+        assert env.get('PASSWORD') == 'bitter'
+        assert env.get('PATH') == 'second'
+
+        env = self.run_environ(runner, *target, '--env-file', env2, '--env-file', env1)
+        assert env.get('SECRET') == 'unknown'
+        assert env.get('PASSWORD') == 'sweet'
+        assert env.get('PATH') == 'first'
+
+
+class TestEnvFileCombinations(EnvironmentTestCase):
+    def test_run_with_both_env_files(self, runner, target, env1, env2):
+        env = self.run_environ(runner, *target, '--default-env-file', env1, '--env-file', env2)
+        assert env.get('SECRET') == 'unknown'
+        assert env.get('PASSWORD') == 'bitter'
+        assert env.get('PATH') == 'second'
+
+    def test_run_with_both_env_files_then_overrides(self, runner, target, env1, env2):
+        env = self.run_environ(
+            runner, *target, '--default-env-file', env1, '--env-file', env2, '--env', 'PASSWORD=mine', '--env',
+            'SECRET=s3cr3t'
         )
-        out = out.split('\n')
+        assert env.get('SECRET') == 's3cr3t'
-        assert out[0] == '321'
+        assert env.get('PASSWORD') == 'mine'
-        assert out[1] == 'sweetpassword'
+        assert env.get('PATH') == 'second'
-        assert out[2] == 'marzo'
 
-    def test_run_file_with_multiple_files(self, runner):
+
-        out, err = runner(
+class TestEnvVars(EnvironmentTestCase):
-            'run',
+    def test_run_no_env(self, runner, target):
-            '--quiet',
+        env = self.run_environ(runner, *target, environ={'USER': 'romain'})
-            get_examples_path('environment/env_files/get_passed_env_file.py'),
+        assert env.get('USER') == 'romain'
-            '--env-file',
+
-            '.env_one',
+    def test_run_env(self, runner, target):
-            '--env-file',
+        env = self.run_environ(runner, *target, '--env', 'USER=serious', environ={'USER': 'romain'})
-            '.env_two',
+        assert env.get('USER') == 'serious'
+
+    def test_run_env_mixed(self, runner, target):
+        env = self.run_environ(runner, *target, '--env', 'ONE=1', '--env', 'TWO="2"', environ={'USER': 'romain'})
+        assert env.get('USER') == 'romain'
+        assert env.get('ONE') == '1'
+        assert env.get('TWO') == '2'
+
+    def test_run_default_env(self, runner, target):
+        env = self.run_environ(runner, *target, '--default-env', 'USER=clown')
+        assert env.get('USER') == 'clown'
+
+        env = self.run_environ(runner, *target, '--default-env', 'USER=clown', environ={'USER': 'romain'})
+        assert env.get('USER') == 'romain'
+
+        env = self.run_environ(
+            runner, *target, '--env', 'USER=serious', '--default-env', 'USER=clown', environ={
+                'USER': 'romain'
+            }
         )
-        out = out.split('\n')
+        assert env.get('USER') == 'serious'
-        assert out[0] == '321'
-        assert out[1] == 'not_sweet_password'
-        assert out[2] == 'abril'
-
-    def test_run_module_with_file(self, runner):
-        out, err = runner(
-            'run',
-            '--quiet',
-            '-m',
-            'bonobo.examples.environment.env_files.get_passed_env_file',
-            '--env-file',
-            '.env_one',
-        )
-        out = out.split('\n')
-        assert out[0] == '321'
-        assert out[1] == 'sweetpassword'
-        assert out[2] == 'marzo'
-
-    def test_run_module_with_multiple_files(self, runner):
-        out, err = runner(
-            'run',
-            '--quiet',
-            '-m',
-            'bonobo.examples.environment.env_files.get_passed_env_file',
-            '--env-file',
-            '.env_one',
-            '--env-file',
-            '.env_two',
-        )
-        out = out.split('\n')
-        assert out[0] == '321'
-        assert out[1] == 'not_sweet_password'
-        assert out[2] == 'abril'
-
-
-@all_runners
-class TestEnvFileCombinations:
-    def test_run_file_with_default_env_file_and_env_file(self, runner):
-        out, err = runner(
-            'run',
-            '--quiet',
-            get_examples_path('environment/env_files/get_passed_env_file.py'),
-            '--default-env-file',
-            '.env_one',
-            '--env-file',
-            '.env_two',
-        )
-        out = out.split('\n')
-        assert out[0] == '321'
-        assert out[1] == 'not_sweet_password'
-        assert out[2] == 'abril'
-
-    def test_run_file_with_default_env_file_and_env_file_and_env_vars(self, runner):
-        out, err = runner(
-            'run',
-            '--quiet',
-            get_examples_path('environment/env_files/get_passed_env_file.py'),
-            '--default-env-file',
-            '.env_one',
-            '--env-file',
-            '.env_two',
-            '--env',
-            'TEST_USER_PASSWORD=SWEETpassWORD',
-            '--env',
-            'MY_SECRET=444',
-        )
-        out = out.split('\n')
-        assert out[0] == '444'
-        assert out[1] == 'SWEETpassWORD'
-        assert out[2] == 'abril'
-
-
-@all_runners
-class TestDefaultEnvVars:
-    def test_run_file_with_default_env_var(self, runner):
-        out, err = runner(
-            'run', '--quiet',
-            get_examples_path('environment/env_vars/get_passed_env.py'), '--default-env', 'USER=clowncity', '--env',
-            'USER=ted'
-        )
-        out = out.split('\n')
-        assert out[0] == 'user'
-        assert out[1] == 'number'
-        assert out[2] == 'string'
-        assert out[3] != 'clowncity'
-
-    def test_run_file_with_default_env_vars(self, runner):
-        out, err = runner(
-            'run', '--quiet',
-            get_examples_path('environment/env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env',
-            'ENV_TEST_USER=cwandrews', '--default-env', "ENV_TEST_STRING='my_test_string'"
-        )
-        out = out.split('\n')
-        assert out[0] == 'cwandrews'
-        assert out[1] == '123'
-        assert out[2] == 'my_test_string'
-
-    def test_run_module_with_default_env_var(self, runner):
-        out, err = runner(
-            'run', '--quiet', '-m', 'bonobo.examples.environment.env_vars.get_passed_env', '--env',
-            'ENV_TEST_NUMBER=123', '--default-env', 'ENV_TEST_STRING=string'
-        )
-        out = out.split('\n')
-        assert out[0] == 'cwandrews'
-        assert out[1] == '123'
-        assert out[2] != 'string'
-
-    def test_run_module_with_default_env_vars(self, runner):
-        out, err = runner(
-            'run', '--quiet', '-m', 'bonobo.examples.environment.env_vars.get_passed_env', '--env',
-            'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', '--default-env', "ENV_TEST_STRING='string'"
-        )
-        out = out.split('\n')
-        assert out[0] == 'cwandrews'
-        assert out[1] == '123'
-        assert out[2] != 'string'
-
-
-@all_runners
-class TestEnvVars:
-    def test_run_file_with_env_var(self, runner):
-        out, err = runner(
-            'run', '--quiet',
-            get_examples_path('environment/env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123'
-        )
-        out = out.split('\n')
-        assert out[0] != 'test_user'
-        assert out[1] == '123'
-        assert out[2] == 'my_test_string'
-
-    def test_run_file_with_env_vars(self, runner):
-        out, err = runner(
-            'run', '--quiet',
-            get_examples_path('environment/env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env',
-            'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
-        )
-        out = out.split('\n')
-        assert out[0] == 'cwandrews'
-        assert out[1] == '123'
-        assert out[2] == 'my_test_string'
-
-    def test_run_module_with_env_var(self, runner):
-        out, err = runner(
-            'run', '--quiet', '-m', 'bonobo.examples.environment.env_vars.get_passed_env', '--env',
-            'ENV_TEST_NUMBER=123'
-        )
-        out = out.split('\n')
-        assert out[0] == 'cwandrews'
-        assert out[1] == '123'
-        assert out[2] == 'my_test_string'
-
-    def test_run_module_with_env_vars(self, runner):
-        out, err = runner(
-            'run', '--quiet', '-m', 'bonobo.examples.environment.env_vars.get_passed_env', '--env',
-            'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
-        )
-        out = out.split('\n')
-        assert out[0] == 'cwandrews'
-        assert out[1] == '123'
-        assert out[2] == 'my_test_string'