paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[core] still refactoring env-related stuff towards using __main__ blocks (but with argparser, if needed).

Browse Source
This commit is contained in:
Romain Dorgueil
2017-10-29 23:46:39 +01:00
parent 8351897e3a
commit c770287466
7 changed files with 110 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Projectfile
View File

@ -54,12 +54,15 @@ python.add_requirements(
'pytest-timeout >=1,<2',
],
docker=[
'bonobo-docker',
'bonobo-docker >=0.5.0',
],
jupyter=[
'jupyter >=1.0,<1.1',
'ipywidgets >=6.0.0,<7',
]
],
sqlalchemy=[
'bonobo-sqlalchemy >=0.5.1',
],
)
# vim: ft=python:

61
bonobo/_api.py
View File

@ -1,3 +1,7 @@
import argparse
from contextlib import contextmanager
import os
from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
PickleReader, PickleWriter, PrettyPrinter, RateLimited, Tee, arg0_to_kwargs, count, identity, kwargs_to_arg0, noop
from bonobo.nodes import LdjsonReader, LdjsonWriter
@ -19,7 +23,60 @@ def register_api_group(*args):
@register_api
def run(graph, *, plugins=None, services=None, **options):
def get_argument_parser(parser=None):
if parser is None:
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--default-env-file', action='append')
parser.add_argument('--default-env', action='append')
parser.add_argument('--env-file', action='append')
parser.add_argument('--env', '-e', action='append')
return parser
@register_api
@contextmanager
def parse_args(parser, *, args=None, namespace=None):
options = parser.parse_args(args=args, namespace=namespace)
with patch_environ(options) as options:
yield options
@register_api
@contextmanager
def patch_environ(options):
from dotenv import load_dotenv
from bonobo.commands import set_env_var
options = options if isinstance(options, dict) else options.__dict__
default_env_file = options.pop('default_env_file', [])
default_env = options.pop('default_env', [])
env_file = options.pop('env_file', [])
env = options.pop('env', [])
if default_env_file:
for f in default_env_file:
load_dotenv(os.path.join(os.getcwd(), f))
if default_env:
for e in default_env:
set_env_var(e)
if env_file:
for f in env_file:
load_dotenv(os.path.join(os.getcwd(), f), override=True)
if env:
for e in env:
set_env_var(e, override=True)
yield options
## TODO XXX put it back !!!
@register_api
def run(graph, *, plugins=None, services=None, strategy=None):
"""
Main entry point of bonobo. It takes a graph and creates all the necessary plumbery around to execute it.
@ -39,7 +96,7 @@ def run(graph, *, plugins=None, services=None, **options):
:param dict services: The implementations of services this graph will use.
:return bonobo.execution.graph.GraphExecutionContext:
"""
strategy = create_strategy(options.pop('strategy', None))
strategy = create_strategy(strategy)
plugins = plugins or []

2
bonobo/_version.py
View File

@ -1 +1 @@
__version__ = '0.5.1'
__version__ = '0.6-dev'

93
bonobo/commands/__init__.py
View File

@ -3,10 +3,10 @@ import codecs
import os
import os.path
import runpy
import sys
from contextlib import contextmanager
from functools import partial
from bonobo import settings, logging
from bonobo import settings, logging, get_argument_parser, patch_environ
from bonobo.constants import DEFAULT_SERVICES_FILENAME, DEFAULT_SERVICES_ATTR
from bonobo.util import get_name
@ -44,11 +44,8 @@ class BaseGraphCommand(BaseCommand):
source_group.add_argument('file', nargs='?', type=str)
source_group.add_argument('-m', dest='mod', type=str)
# arguments to enforce system environment.
parser.add_argument('--default-env-file', action='append')
parser.add_argument('--default-env', action='append')
parser.add_argument('--env-file', action='append')
parser.add_argument('--env', '-e', action='append')
# add arguments to enforce system environment.
parser = get_argument_parser(parser)
return parser
@ -58,34 +55,30 @@ class BaseGraphCommand(BaseCommand):
def _run_module(self, mod):
return runpy.run_module(mod, run_name='__main__')
def read(self, *, file, mod, **options):
"""
get_default_services(
filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
)
"""
def read(self, *, file, mod, args=None, **options):
_graph, _options = None, None
def _record(graph, **options):
nonlocal _graph, _options
_graph, _options = graph, options
with _override_runner(_record), _override_environment():
if file:
self._run_path(file)
elif mod:
self._run_module(mod)
else:
raise RuntimeError('No target provided.')
with _override_runner(_record), patch_environ(options):
_argv = sys.argv
try:
if file:
sys.argv = [file] + list(args) if args else [file]
self._run_path(file)
elif mod:
sys.argv = [mod, *(args or ())]
self._run_module(mod)
else:
raise RuntimeError('No target provided.')
finally:
sys.argv = _argv
if _graph is None:
raise RuntimeError('Could not find graph.')
return _graph, _options
def handle(self, *args, **options):
@ -120,45 +113,41 @@ def entrypoint(args=None):
mgr = ExtensionManager(namespace='bonobo.commands')
mgr.map(register_extension)
args = parser.parse_args(args).__dict__
if args.pop('debug', False):
parsed_args, remaining = parser.parse_known_args(args)
parsed_args = parsed_args.__dict__
if parsed_args.pop('debug', False):
settings.DEBUG.set(True)
settings.LOGGING_LEVEL.set(logging.DEBUG)
logging.set_level(settings.LOGGING_LEVEL.get())
logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args))
commands[args.pop('command')](**args)
logger.debug('Command: ' + parsed_args['command'] + ' Arguments: ' + repr(parsed_args))
# Get command handler
command = commands[parsed_args.pop('command')]
if len(remaining):
command(_remaining_args=remaining, **parsed_args)
else:
command(**parsed_args)
@contextmanager
def _override_runner(runner):
import bonobo
_runner_backup = bonobo.run
_get_argument_parser = bonobo.get_argument_parser
_run = bonobo.run
try:
def get_argument_parser(parser=None):
return parser or argparse.ArgumentParser()
bonobo.get_argument_parser = get_argument_parser
bonobo.run = runner
yield runner
finally:
bonobo.run = _runner_backup
@contextmanager
def _override_environment(root_dir=None, **options):
yield
return
if default_env_file:
for f in default_env_file:
env_file_path = str(env_dir.joinpath(f))
load_dotenv(env_file_path)
if default_env:
for e in default_env:
set_env_var(e)
if env_file:
for f in env_file:
env_file_path = str(env_dir.joinpath(f))
load_dotenv(env_file_path, override=True)
if env:
for e in env:
set_env_var(e, override=True)
bonobo.get_argument_parser = _get_argument_parser
bonobo.run = _run
def get_default_services(filename, services=None):
@ -194,4 +183,4 @@ def set_env_var(e, override=False):
if override:
os.environ[ename] = evalue
else:
os.environ.setdefault(ename, evalue)
os.environ.setdefault(ename, evalue)

4
bonobo/commands/init.py
View File

@ -6,13 +6,13 @@ from bonobo.commands import BaseCommand
class InitCommand(BaseCommand):
TEMPLATES = {'job'}
TEMPLATES = {'default'}
TEMPLATES_PATH = os.path.join(os.path.dirname(__file__), 'templates')
def add_arguments(self, parser):
parser.add_argument('template', choices=self.TEMPLATES)
parser.add_argument('filename')
parser.add_argument('--force', '-f', default=False, action='store_true')
parser.add_argument('--template', '-t', choices=self.TEMPLATES, default='default')
def handle(self, *, template, filename, force=False):
template_name = template

4
bonobo/commands/run.py
View File

@ -32,14 +32,14 @@ class RunCommand(BaseGraphCommand):
return super()._run_module(mod)
def handle(self, *args, quiet=False, verbose=False, install=False, **options):
def handle(self, quiet=False, verbose=False, install=False, _remaining_args=None, **options):
from bonobo import settings
settings.QUIET.set_if_true(quiet)
settings.DEBUG.set_if_true(verbose)
self.install = install
graph, params = self.read(**options)
graph, params = self.read(args=_remaining_args, **options)
params['plugins'] = set(params.pop('plugins', ())).union(set(options.pop('plugins', ())))

31
tests/test_commands.py
View File

@ -12,7 +12,6 @@ from cookiecutter.exceptions import OutputDirExistsException
from bonobo import __main__, __version__, get_examples_path
from bonobo.commands import entrypoint
from bonobo.commands.run import DEFAULT_GRAPH_FILENAMES
from bonobo.commands.download import EXAMPLES_BASE_URL
@ -72,36 +71,6 @@ def test_no_command(runner):
assert 'error: the following arguments are required: command' in err
@all_runners
def test_init(runner, tmpdir):
name = 'project'
tmpdir.chdir()
runner('init', name)
assert os.path.isdir(name)
assert set(os.listdir(name)) & set(DEFAULT_GRAPH_FILENAMES)
@single_runner
def test_init_in_empty_then_nonempty_directory(runner, tmpdir):
name = 'project'
tmpdir.chdir()
os.mkdir(name)
# run in empty dir
runner('init', name)
assert set(os.listdir(name)) & set(DEFAULT_GRAPH_FILENAMES)
# run in non empty dir
with pytest.raises(OutputDirExistsException):
runner('init', name)
@single_runner
def test_init_within_empty_directory(runner, tmpdir):
tmpdir.chdir()
runner('init', '.')
assert set(os.listdir()) & set(DEFAULT_GRAPH_FILENAMES)
@all_runners
def test_run(runner):
out, err = runner('run', '--quiet', get_examples_path('types/strings.py'))