diff --git a/Projectfile b/Projectfile index d730328..eb230d9 100644 --- a/Projectfile +++ b/Projectfile @@ -54,12 +54,15 @@ python.add_requirements( 'pytest-timeout >=1,<2', ], docker=[ - 'bonobo-docker', + 'bonobo-docker >=0.5.0', ], jupyter=[ 'jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0,<7', - ] + ], + sqlalchemy=[ + 'bonobo-sqlalchemy >=0.5.1', + ], ) # vim: ft=python: diff --git a/bonobo/_api.py b/bonobo/_api.py index fb1ef78..43e9a29 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -1,3 +1,7 @@ +import argparse +from contextlib import contextmanager + +import os from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \ PickleReader, PickleWriter, PrettyPrinter, RateLimited, Tee, arg0_to_kwargs, count, identity, kwargs_to_arg0, noop from bonobo.nodes import LdjsonReader, LdjsonWriter @@ -19,7 +23,60 @@ def register_api_group(*args): @register_api -def run(graph, *, plugins=None, services=None, **options): +def get_argument_parser(parser=None): + if parser is None: + import argparse + parser = argparse.ArgumentParser() + + parser.add_argument('--default-env-file', action='append') + parser.add_argument('--default-env', action='append') + parser.add_argument('--env-file', action='append') + parser.add_argument('--env', '-e', action='append') + + return parser + + +@register_api +@contextmanager +def parse_args(parser, *, args=None, namespace=None): + options = parser.parse_args(args=args, namespace=namespace) + + with patch_environ(options) as options: + yield options + + +@register_api +@contextmanager +def patch_environ(options): + from dotenv import load_dotenv + from bonobo.commands import set_env_var + + options = options if isinstance(options, dict) else options.__dict__ + + default_env_file = options.pop('default_env_file', []) + default_env = options.pop('default_env', []) + env_file = options.pop('env_file', []) + env = options.pop('env', []) + + if default_env_file: + for f in default_env_file: + load_dotenv(os.path.join(os.getcwd(), f)) + if default_env: + for e in default_env: + set_env_var(e) + if env_file: + for f in env_file: + load_dotenv(os.path.join(os.getcwd(), f), override=True) + if env: + for e in env: + set_env_var(e, override=True) + + yield options + ## TODO XXX put it back !!! + + +@register_api +def run(graph, *, plugins=None, services=None, strategy=None): """ Main entry point of bonobo. It takes a graph and creates all the necessary plumbery around to execute it. @@ -39,7 +96,7 @@ def run(graph, *, plugins=None, services=None, **options): :param dict services: The implementations of services this graph will use. :return bonobo.execution.graph.GraphExecutionContext: """ - strategy = create_strategy(options.pop('strategy', None)) + strategy = create_strategy(strategy) plugins = plugins or [] diff --git a/bonobo/_version.py b/bonobo/_version.py index 93b60a1..ebc2ff2 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = '0.5.1' +__version__ = '0.6-dev' diff --git a/bonobo/commands/__init__.py b/bonobo/commands/__init__.py index be877d7..c5d4908 100644 --- a/bonobo/commands/__init__.py +++ b/bonobo/commands/__init__.py @@ -3,10 +3,10 @@ import codecs import os import os.path import runpy +import sys from contextlib import contextmanager -from functools import partial -from bonobo import settings, logging +from bonobo import settings, logging, get_argument_parser, patch_environ from bonobo.constants import DEFAULT_SERVICES_FILENAME, DEFAULT_SERVICES_ATTR from bonobo.util import get_name @@ -44,11 +44,8 @@ class BaseGraphCommand(BaseCommand): source_group.add_argument('file', nargs='?', type=str) source_group.add_argument('-m', dest='mod', type=str) - # arguments to enforce system environment. - parser.add_argument('--default-env-file', action='append') - parser.add_argument('--default-env', action='append') - parser.add_argument('--env-file', action='append') - parser.add_argument('--env', '-e', action='append') + # add arguments to enforce system environment. + parser = get_argument_parser(parser) return parser @@ -58,34 +55,30 @@ class BaseGraphCommand(BaseCommand): def _run_module(self, mod): return runpy.run_module(mod, run_name='__main__') - def read(self, *, file, mod, **options): - - """ - - get_default_services( - filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None - ) - - """ - + def read(self, *, file, mod, args=None, **options): _graph, _options = None, None def _record(graph, **options): nonlocal _graph, _options _graph, _options = graph, options - with _override_runner(_record), _override_environment(): - if file: - self._run_path(file) - elif mod: - self._run_module(mod) - else: - raise RuntimeError('No target provided.') + with _override_runner(_record), patch_environ(options): + _argv = sys.argv + try: + if file: + sys.argv = [file] + list(args) if args else [file] + self._run_path(file) + elif mod: + sys.argv = [mod, *(args or ())] + self._run_module(mod) + else: + raise RuntimeError('No target provided.') + finally: + sys.argv = _argv if _graph is None: raise RuntimeError('Could not find graph.') - return _graph, _options def handle(self, *args, **options): @@ -120,45 +113,41 @@ def entrypoint(args=None): mgr = ExtensionManager(namespace='bonobo.commands') mgr.map(register_extension) - args = parser.parse_args(args).__dict__ - if args.pop('debug', False): + parsed_args, remaining = parser.parse_known_args(args) + parsed_args = parsed_args.__dict__ + + if parsed_args.pop('debug', False): settings.DEBUG.set(True) settings.LOGGING_LEVEL.set(logging.DEBUG) logging.set_level(settings.LOGGING_LEVEL.get()) - logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args)) - commands[args.pop('command')](**args) + logger.debug('Command: ' + parsed_args['command'] + ' Arguments: ' + repr(parsed_args)) + + # Get command handler + command = commands[parsed_args.pop('command')] + + if len(remaining): + command(_remaining_args=remaining, **parsed_args) + else: + command(**parsed_args) @contextmanager def _override_runner(runner): import bonobo - _runner_backup = bonobo.run + _get_argument_parser = bonobo.get_argument_parser + _run = bonobo.run try: + def get_argument_parser(parser=None): + return parser or argparse.ArgumentParser() + + bonobo.get_argument_parser = get_argument_parser bonobo.run = runner + yield runner finally: - bonobo.run = _runner_backup - - -@contextmanager -def _override_environment(root_dir=None, **options): - yield - return - if default_env_file: - for f in default_env_file: - env_file_path = str(env_dir.joinpath(f)) - load_dotenv(env_file_path) - if default_env: - for e in default_env: - set_env_var(e) - if env_file: - for f in env_file: - env_file_path = str(env_dir.joinpath(f)) - load_dotenv(env_file_path, override=True) - if env: - for e in env: - set_env_var(e, override=True) + bonobo.get_argument_parser = _get_argument_parser + bonobo.run = _run def get_default_services(filename, services=None): @@ -194,4 +183,4 @@ def set_env_var(e, override=False): if override: os.environ[ename] = evalue else: - os.environ.setdefault(ename, evalue) \ No newline at end of file + os.environ.setdefault(ename, evalue) diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index c0c50f1..6c6c2ff 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -6,13 +6,13 @@ from bonobo.commands import BaseCommand class InitCommand(BaseCommand): - TEMPLATES = {'job'} + TEMPLATES = {'default'} TEMPLATES_PATH = os.path.join(os.path.dirname(__file__), 'templates') def add_arguments(self, parser): - parser.add_argument('template', choices=self.TEMPLATES) parser.add_argument('filename') parser.add_argument('--force', '-f', default=False, action='store_true') + parser.add_argument('--template', '-t', choices=self.TEMPLATES, default='default') def handle(self, *, template, filename, force=False): template_name = template diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 799816f..514bb5d 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -32,14 +32,14 @@ class RunCommand(BaseGraphCommand): return super()._run_module(mod) - def handle(self, *args, quiet=False, verbose=False, install=False, **options): + def handle(self, quiet=False, verbose=False, install=False, _remaining_args=None, **options): from bonobo import settings settings.QUIET.set_if_true(quiet) settings.DEBUG.set_if_true(verbose) self.install = install - graph, params = self.read(**options) + graph, params = self.read(args=_remaining_args, **options) params['plugins'] = set(params.pop('plugins', ())).union(set(options.pop('plugins', ()))) diff --git a/tests/test_commands.py b/tests/test_commands.py index c78fa5f..fe55f87 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -12,7 +12,6 @@ from cookiecutter.exceptions import OutputDirExistsException from bonobo import __main__, __version__, get_examples_path from bonobo.commands import entrypoint -from bonobo.commands.run import DEFAULT_GRAPH_FILENAMES from bonobo.commands.download import EXAMPLES_BASE_URL @@ -72,36 +71,6 @@ def test_no_command(runner): assert 'error: the following arguments are required: command' in err -@all_runners -def test_init(runner, tmpdir): - name = 'project' - tmpdir.chdir() - runner('init', name) - assert os.path.isdir(name) - assert set(os.listdir(name)) & set(DEFAULT_GRAPH_FILENAMES) - -@single_runner -def test_init_in_empty_then_nonempty_directory(runner, tmpdir): - name = 'project' - tmpdir.chdir() - os.mkdir(name) - - # run in empty dir - runner('init', name) - assert set(os.listdir(name)) & set(DEFAULT_GRAPH_FILENAMES) - - # run in non empty dir - with pytest.raises(OutputDirExistsException): - runner('init', name) - - -@single_runner -def test_init_within_empty_directory(runner, tmpdir): - tmpdir.chdir() - runner('init', '.') - assert set(os.listdir()) & set(DEFAULT_GRAPH_FILENAMES) - - @all_runners def test_run(runner): out, err = runner('run', '--quiet', get_examples_path('types/strings.py'))