[core] still refactoring env-related stuff towards using __main__ blocks (but with argparser, if needed).

Projectfile

@@ -54,12 +54,15 @@ python.add_requirements(
         'pytest-timeout >=1,<2',
     ],
     docker=[
-        'bonobo-docker',
+        'bonobo-docker >=0.5.0',
     ],
     jupyter=[
         'jupyter >=1.0,<1.1',
         'ipywidgets >=6.0.0,<7',
-    ]
+    ],
+    sqlalchemy=[
+        'bonobo-sqlalchemy >=0.5.1',
+    ],
 )
 
 # vim: ft=python:

bonobo/_api.py

@@ -1,3 +1,7 @@
+import argparse
+from contextlib import contextmanager
+
+import os
 from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
     PickleReader, PickleWriter, PrettyPrinter, RateLimited, Tee, arg0_to_kwargs, count, identity, kwargs_to_arg0, noop
 from bonobo.nodes import LdjsonReader, LdjsonWriter
@@ -19,7 +23,60 @@ def register_api_group(*args):
 
 
 @register_api
-def run(graph, *, plugins=None, services=None, **options):
+def get_argument_parser(parser=None):
+    if parser is None:
+        import argparse
+        parser = argparse.ArgumentParser()
+
+    parser.add_argument('--default-env-file', action='append')
+    parser.add_argument('--default-env', action='append')
+    parser.add_argument('--env-file', action='append')
+    parser.add_argument('--env', '-e', action='append')
+
+    return parser
+
+
+@register_api
+@contextmanager
+def parse_args(parser, *, args=None, namespace=None):
+    options = parser.parse_args(args=args, namespace=namespace)
+
+    with patch_environ(options) as options:
+        yield options
+
+
+@register_api
+@contextmanager
+def patch_environ(options):
+    from dotenv import load_dotenv
+    from bonobo.commands import set_env_var
+
+    options = options if isinstance(options, dict) else options.__dict__
+
+    default_env_file = options.pop('default_env_file', [])
+    default_env = options.pop('default_env', [])
+    env_file = options.pop('env_file', [])
+    env = options.pop('env', [])
+
+    if default_env_file:
+        for f in default_env_file:
+            load_dotenv(os.path.join(os.getcwd(), f))
+    if default_env:
+        for e in default_env:
+            set_env_var(e)
+    if env_file:
+        for f in env_file:
+            load_dotenv(os.path.join(os.getcwd(), f), override=True)
+    if env:
+        for e in env:
+            set_env_var(e, override=True)
+
+    yield options
+    ## TODO XXX put it back !!!
+
+
+@register_api
+def run(graph, *, plugins=None, services=None, strategy=None):
     """
     Main entry point of bonobo. It takes a graph and creates all the necessary plumbery around to execute it.
 
@@ -39,7 +96,7 @@ def run(graph, *, plugins=None, services=None, **options):
     :param dict services: The implementations of services this graph will use.
     :return bonobo.execution.graph.GraphExecutionContext:
     """
-    strategy = create_strategy(options.pop('strategy', None))
+    strategy = create_strategy(strategy)
 
     plugins = plugins or []
 

bonobo/_version.py

@@ -1 +1 @@
-__version__ = '0.5.1'
+__version__ = '0.6-dev'

bonobo/commands/__init__.py

@@ -3,10 +3,10 @@ import codecs
 import os
 import os.path
 import runpy
+import sys
 from contextlib import contextmanager
-from functools import partial
 
-from bonobo import settings, logging
+from bonobo import settings, logging, get_argument_parser, patch_environ
 from bonobo.constants import DEFAULT_SERVICES_FILENAME, DEFAULT_SERVICES_ATTR
 from bonobo.util import get_name
 
@@ -44,11 +44,8 @@ class BaseGraphCommand(BaseCommand):
         source_group.add_argument('file', nargs='?', type=str)
         source_group.add_argument('-m', dest='mod', type=str)
 
-        # arguments to enforce system environment.
+        # add arguments to enforce system environment.
-        parser.add_argument('--default-env-file', action='append')
+        parser = get_argument_parser(parser)
-        parser.add_argument('--default-env', action='append')
-        parser.add_argument('--env-file', action='append')
-        parser.add_argument('--env', '-e', action='append')
 
         return parser
 
@@ -58,34 +55,30 @@ class BaseGraphCommand(BaseCommand):
     def _run_module(self, mod):
         return runpy.run_module(mod, run_name='__main__')
 
-    def read(self, *, file, mod, **options):
+    def read(self, *, file, mod, args=None, **options):
-
-        """
-
-        get_default_services(
-            filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
-        )
-        
-        """
-
         _graph, _options = None, None
 
         def _record(graph, **options):
             nonlocal _graph, _options
             _graph, _options = graph, options
 
-        with _override_runner(_record), _override_environment():
+        with _override_runner(_record), patch_environ(options):
-            if file:
+            _argv = sys.argv
-                self._run_path(file)
+            try:
-            elif mod:
+                if file:
-                self._run_module(mod)
+                    sys.argv = [file] + list(args) if args else [file]
-            else:
+                    self._run_path(file)
-                raise RuntimeError('No target provided.')
+                elif mod:
+                    sys.argv = [mod, *(args or ())]
+                    self._run_module(mod)
+                else:
+                    raise RuntimeError('No target provided.')
+            finally:
+                sys.argv = _argv
 
         if _graph is None:
             raise RuntimeError('Could not find graph.')
 
-
         return _graph, _options
 
     def handle(self, *args, **options):
@@ -120,45 +113,41 @@ def entrypoint(args=None):
     mgr = ExtensionManager(namespace='bonobo.commands')
     mgr.map(register_extension)
 
-    args = parser.parse_args(args).__dict__
+    parsed_args, remaining = parser.parse_known_args(args)
-    if args.pop('debug', False):
+    parsed_args = parsed_args.__dict__
+
+    if parsed_args.pop('debug', False):
         settings.DEBUG.set(True)
         settings.LOGGING_LEVEL.set(logging.DEBUG)
         logging.set_level(settings.LOGGING_LEVEL.get())
 
-    logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args))
+    logger.debug('Command: ' + parsed_args['command'] + ' Arguments: ' + repr(parsed_args))
-    commands[args.pop('command')](**args)
+
+    # Get command handler
+    command = commands[parsed_args.pop('command')]
+
+    if len(remaining):
+        command(_remaining_args=remaining, **parsed_args)
+    else:
+        command(**parsed_args)
 
 
 @contextmanager
 def _override_runner(runner):
     import bonobo
-    _runner_backup = bonobo.run
+    _get_argument_parser = bonobo.get_argument_parser
+    _run = bonobo.run
     try:
+        def get_argument_parser(parser=None):
+            return parser or argparse.ArgumentParser()
+
+        bonobo.get_argument_parser = get_argument_parser
         bonobo.run = runner
+
         yield runner
     finally:
-        bonobo.run = _runner_backup
+        bonobo.get_argument_parser = _get_argument_parser
-
+        bonobo.run = _run
-
-@contextmanager
-def _override_environment(root_dir=None, **options):
-    yield
-    return
-    if default_env_file:
-        for f in default_env_file:
-            env_file_path = str(env_dir.joinpath(f))
-            load_dotenv(env_file_path)
-    if default_env:
-        for e in default_env:
-            set_env_var(e)
-    if env_file:
-        for f in env_file:
-            env_file_path = str(env_dir.joinpath(f))
-            load_dotenv(env_file_path, override=True)
-    if env:
-        for e in env:
-            set_env_var(e, override=True)
 
 
 def get_default_services(filename, services=None):
@@ -194,4 +183,4 @@ def set_env_var(e, override=False):
     if override:
         os.environ[ename] = evalue
     else:
-        os.environ.setdefault(ename, evalue)
+        os.environ.setdefault(ename, evalue)

bonobo/commands/init.py

@@ -6,13 +6,13 @@ from bonobo.commands import BaseCommand
 
 
 class InitCommand(BaseCommand):
-    TEMPLATES = {'job'}
+    TEMPLATES = {'default'}
     TEMPLATES_PATH = os.path.join(os.path.dirname(__file__), 'templates')
 
     def add_arguments(self, parser):
-        parser.add_argument('template', choices=self.TEMPLATES)
         parser.add_argument('filename')
         parser.add_argument('--force', '-f', default=False, action='store_true')
+        parser.add_argument('--template', '-t', choices=self.TEMPLATES, default='default')
 
     def handle(self, *, template, filename, force=False):
         template_name = template

bonobo/commands/run.py

@@ -32,14 +32,14 @@ class RunCommand(BaseGraphCommand):
 
         return super()._run_module(mod)
 
-    def handle(self, *args, quiet=False, verbose=False, install=False, **options):
+    def handle(self, quiet=False, verbose=False, install=False, _remaining_args=None, **options):
         from bonobo import settings
 
         settings.QUIET.set_if_true(quiet)
         settings.DEBUG.set_if_true(verbose)
         self.install = install
 
-        graph, params = self.read(**options)
+        graph, params = self.read(args=_remaining_args, **options)
 
         params['plugins'] = set(params.pop('plugins', ())).union(set(options.pop('plugins', ())))
 

tests/test_commands.py

@@ -12,7 +12,6 @@ from cookiecutter.exceptions import OutputDirExistsException
 
 from bonobo import __main__, __version__, get_examples_path
 from bonobo.commands import entrypoint
-from bonobo.commands.run import DEFAULT_GRAPH_FILENAMES
 from bonobo.commands.download import EXAMPLES_BASE_URL
 
 
@@ -72,36 +71,6 @@ def test_no_command(runner):
     assert 'error: the following arguments are required: command' in err
 
 
-@all_runners
-def test_init(runner, tmpdir):
-    name = 'project'
-    tmpdir.chdir()
-    runner('init', name)
-    assert os.path.isdir(name)
-    assert set(os.listdir(name)) & set(DEFAULT_GRAPH_FILENAMES)
-
-@single_runner
-def test_init_in_empty_then_nonempty_directory(runner, tmpdir):
-    name = 'project'
-    tmpdir.chdir()
-    os.mkdir(name)
-
-    # run in empty dir
-    runner('init', name)
-    assert set(os.listdir(name)) & set(DEFAULT_GRAPH_FILENAMES)
-
-    # run in non empty dir
-    with pytest.raises(OutputDirExistsException):
-        runner('init', name)
-
-
-@single_runner
-def test_init_within_empty_directory(runner, tmpdir):
-    tmpdir.chdir()
-    runner('init', '.')
-    assert set(os.listdir()) & set(DEFAULT_GRAPH_FILENAMES)
-
-
 @all_runners
 def test_run(runner):
     out, err = runner('run', '--quiet', get_examples_path('types/strings.py'))