paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactoring the runner to go more towards standard python, also adds the ability to use bonobo argument parser from standard python execution.

Browse Source
This commit is contained in:
Romain Dorgueil
2017-11-01 18:46:45 +01:00
parent e6596cf3f3
commit e06b616251
18 changed files with 537 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Projectfile
View File

@ -66,8 +66,6 @@ python.add_requirements(
# Following requirements are not enforced, because some dependencies enforce them so we don't want to break
# the packaging in case it changes in dep.
python.add_requirements(
'colorama >=0.3',
)
python.add_requirements('colorama >=0.3', )
# vim: ft=python:

79
bonobo/_api.py
View File

@ -1,13 +1,10 @@
import argparse
from contextlib import contextmanager
import os
from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
PickleReader, PickleWriter, PrettyPrinter, RateLimited, Tee, arg0_to_kwargs, count, identity, kwargs_to_arg0, noop
from bonobo.nodes import LdjsonReader, LdjsonWriter
from bonobo.strategies import create_strategy
from bonobo.structs import Bag, ErrorBag, Graph, Token
from bonobo.util import get_name
from bonobo.util.environ import parse_args, get_argument_parser
__all__ = []
@ -22,59 +19,6 @@ def register_api_group(*args):
register_api(attr)
@register_api
def get_argument_parser(parser=None):
if parser is None:
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--default-env-file', action='append')
parser.add_argument('--default-env', action='append')
parser.add_argument('--env-file', action='append')
parser.add_argument('--env', '-e', action='append')
return parser
@register_api
@contextmanager
def parse_args(parser, *, args=None, namespace=None):
options = parser.parse_args(args=args, namespace=namespace)
with patch_environ(options) as options:
yield options
@register_api
@contextmanager
def patch_environ(options):
from dotenv import load_dotenv
from bonobo.commands import set_env_var
options = options if isinstance(options, dict) else options.__dict__
default_env_file = options.pop('default_env_file', [])
default_env = options.pop('default_env', [])
env_file = options.pop('env_file', [])
env = options.pop('env', [])
if default_env_file:
for f in default_env_file:
load_dotenv(os.path.join(os.getcwd(), f))
if default_env:
for e in default_env:
set_env_var(e)
if env_file:
for f in env_file:
load_dotenv(os.path.join(os.getcwd(), f), override=True)
if env:
for e in env:
set_env_var(e, override=True)
yield options
## TODO XXX put it back !!!
@register_api
def run(graph, *, plugins=None, services=None, strategy=None):
"""
@ -126,6 +70,24 @@ def run(graph, *, plugins=None, services=None, strategy=None):
return strategy.execute(graph, plugins=plugins, services=services)
def _inspect_as_graph(graph):
return graph._repr_dot_()
_inspect_formats = {'graph': _inspect_as_graph}
@register_api
def inspect(graph, *, format):
if not format in _inspect_formats:
raise NotImplementedError(
'Output format {} not implemented. Choices are: {}.'.format(
format, ', '.join(sorted(_inspect_formats.keys()))
)
)
print(_inspect_formats[format](graph))
# bonobo.structs
register_api_group(Bag, ErrorBag, Graph, Token)
@ -205,3 +167,6 @@ def get_examples_path(*pathsegments):
@register_api
def open_examples_fs(*pathsegments):
return open_fs(get_examples_path(*pathsegments))
register_api_group(get_argument_parser, parse_args)

136
bonobo/commands/__init__.py
View File

@ -1,91 +1,19 @@
import argparse
import codecs
import os
import os.path
import runpy
import sys
from contextlib import contextmanager
from bonobo import settings, logging, get_argument_parser, patch_environ
from bonobo.constants import DEFAULT_SERVICES_FILENAME, DEFAULT_SERVICES_ATTR
from bonobo.util import get_name
from bonobo import settings, logging
from bonobo.commands.base import BaseCommand, BaseGraphCommand
logger = logging.get_logger()
class BaseCommand:
@property
def logger(self):
try:
return self._logger
except AttributeError:
self._logger = logging.get_logger(get_name(self))
return self._logger
def add_arguments(self, parser):
"""
Entry point for subclassed commands to add custom arguments.
"""
pass
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement this method.
"""
raise NotImplementedError('Subclasses of BaseCommand must provide a handle() method')
class BaseGraphCommand(BaseCommand):
required = True
def add_arguments(self, parser):
# target arguments (cannot provide both).
source_group = parser.add_mutually_exclusive_group(required=self.required)
source_group.add_argument('file', nargs='?', type=str)
source_group.add_argument('-m', dest='mod', type=str)
# add arguments to enforce system environment.
parser = get_argument_parser(parser)
return parser
def _run_path(self, file):
return runpy.run_path(file, run_name='__main__')
def _run_module(self, mod):
return runpy.run_module(mod, run_name='__main__')
def read(self, *, file, mod, args=None, **options):
_graph, _options = None, None
def _record(graph, **options):
nonlocal _graph, _options
_graph, _options = graph, options
with _override_runner(_record), patch_environ(options):
_argv = sys.argv
try:
if file:
sys.argv = [file] + list(args) if args else [file]
self._run_path(file)
elif mod:
sys.argv = [mod, *(args or ())]
self._run_module(mod)
else:
raise RuntimeError('No target provided.')
finally:
sys.argv = _argv
if _graph is None:
raise RuntimeError('Could not find graph.')
return _graph, _options
def handle(self, *args, **options):
pass
def entrypoint(args=None):
"""
Main callable for "bonobo" entrypoint.
Will load commands from "bonobo.commands" entrypoints, using stevedore.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--debug', '-D', action='store_true')
@ -113,8 +41,7 @@ def entrypoint(args=None):
mgr = ExtensionManager(namespace='bonobo.commands')
mgr.map(register_extension)
parsed_args, remaining = parser.parse_known_args(args)
parsed_args = parsed_args.__dict__
parsed_args = parser.parse_args(args).__dict__
if parsed_args.pop('debug', False):
settings.DEBUG.set(True)
@ -123,45 +50,6 @@ def entrypoint(args=None):
logger.debug('Command: ' + parsed_args['command'] + ' Arguments: ' + repr(parsed_args))
# Get command handler
# Get command handler, execute, rince.
command = commands[parsed_args.pop('command')]
if len(remaining):
command(_remaining_args=remaining, **parsed_args)
else:
command(**parsed_args)
@contextmanager
def _override_runner(runner):
import bonobo
_get_argument_parser = bonobo.get_argument_parser
_run = bonobo.run
try:
def get_argument_parser(parser=None):
return parser or argparse.ArgumentParser()
bonobo.get_argument_parser = get_argument_parser
bonobo.run = runner
yield runner
finally:
bonobo.get_argument_parser = _get_argument_parser
bonobo.run = _run
def set_env_var(e, override=False):
__escape_decoder = codecs.getdecoder('unicode_escape')
ename, evalue = e.split('=', 1)
def decode_escaped(escaped):
return __escape_decoder(escaped)[0]
if len(evalue) > 0:
if evalue[0] == evalue[len(evalue) - 1] in ['"', "'"]:
evalue = decode_escaped(evalue[1:-1])
if override:
os.environ[ename] = evalue
else:
os.environ.setdefault(ename, evalue)
command(**parsed_args)

129
bonobo/commands/base.py Normal file
View File

@ -0,0 +1,129 @@
import argparse
import runpy
import sys
from contextlib import contextmanager
import bonobo.util.environ
from bonobo import logging
from bonobo.util.environ import get_argument_parser, parse_args
from bonobo.util import get_name
class BaseCommand:
"""
Base class for CLI commands.
"""
@property
def logger(self):
try:
return self._logger
except AttributeError:
self._logger = logging.get_logger(get_name(self))
return self._logger
def add_arguments(self, parser):
"""
Entry point for subclassed commands to add custom arguments.
"""
pass
def handle(self, *args, **options):
"""
The actual logic of the command. Subclasses must implement this method.
"""
raise NotImplementedError('Subclasses of BaseCommand must provide a handle() method')
class BaseGraphCommand(BaseCommand):
"""
Base class for CLI commands that depends on a graph definition, either from a file or from a module.
"""
required = True
handler = None
def add_arguments(self, parser):
# target arguments (cannot provide both).
source_group = parser.add_mutually_exclusive_group(required=self.required)
source_group.add_argument('file', nargs='?', type=str)
source_group.add_argument('-m', dest='mod', type=str)
# add arguments to enforce system environment.
parser = get_argument_parser(parser)
return parser
def parse_options(self, **options):
return options
def handle(self, file, mod, **options):
options = self.parse_options(**options)
with self.read(file, mod, **options) as (graph, graph_execution_options, options):
return self.do_handle(graph, **graph_execution_options, **options)
def do_handle(self, graph, **options):
if not self.handler:
raise RuntimeError('{} has no handler defined.'.format(get_name(self)))
return self.handler(graph, **options)
@contextmanager
def read(self, file, mod, **options):
_graph, _graph_execution_options = None, None
def _record(graph, **graph_execution_options):
nonlocal _graph, _graph_execution_options
_graph, _graph_execution_options = graph, graph_execution_options
with _override_runner(_record), parse_args(options) as options:
_argv = sys.argv
try:
if file:
sys.argv = [file]
self._run_path(file)
elif mod:
sys.argv = [mod]
self._run_module(mod)
else:
raise RuntimeError('No target provided.')
finally:
sys.argv = _argv
if _graph is None:
raise RuntimeError('Could not find graph.')
yield _graph, _graph_execution_options, options
def _run_path(self, file):
return runpy.run_path(file, run_name='__main__')
def _run_module(self, mod):
return runpy.run_module(mod, run_name='__main__')
@contextmanager
def _override_runner(runner):
"""
Context manager that monkey patches `bonobo.run` function with our current command logic.
:param runner: the callable that will handle the `run()` logic.
"""
import bonobo
_get_argument_parser = bonobo.util.environ.get_argument_parser
_run = bonobo.run
try:
# Original get_argument_parser would create or update an argument parser with environment options, but here we
# already had them parsed so let's patch with something that creates an empty one instead.
def get_argument_parser(parser=None):
return parser or argparse.ArgumentParser()
bonobo.util.environ.get_argument_parser = get_argument_parser
bonobo.run = runner
yield runner
finally:
# Restore our saved values.
bonobo.util.environ.get_argument_parser = _get_argument_parser
bonobo.run = _run

13
bonobo/commands/convert.py
View File

@ -48,8 +48,17 @@ class ConvertCommand(BaseCommand):
help='Add a named option to the writer factory.',
)
def handle(self, input_filename, output_filename, reader=None, reader_option=None, writer=None, writer_option=None,
option=None, transformation=None):
def handle(
self,
input_filename,
output_filename,
reader=None,
reader_option=None,
writer=None,
writer_option=None,
option=None,
transformation=None
):
reader_factory = default_registry.get_reader_factory_for(input_filename, format=reader)
reader_options = _resolve_options((option or []) + (reader_option or []))

22
bonobo/commands/inspect.py
View File

@ -1,21 +1,15 @@
import bonobo
from bonobo.commands import BaseGraphCommand
OUTPUT_GRAPH = 'graphviz'
class InspectCommand(BaseGraphCommand):
handler = staticmethod(bonobo.inspect)
def add_arguments(self, parser):
super(InspectCommand, self).add_arguments(parser)
parser.add_argument('--graph', '-g', dest='output', action='store_const', const=OUTPUT_GRAPH)
def handle(self, output=None, **options):
if output is None:
raise ValueError('Output type must be provided (try --graph/-g).')
graph, params = self.read(**options)
if output == OUTPUT_GRAPH:
print(graph._repr_dot_())
else:
raise NotImplementedError('Output type not implemented.')
parser.add_argument('--graph', '-g', dest='format', action='store_const', const='graph')
def parse_options(self, **options):
if not options.get('format'):
raise RuntimeError('You must provide a format (try --graph).')
return options

23
bonobo/commands/run.py
View File

@ -6,6 +6,7 @@ from bonobo.commands import BaseGraphCommand
class RunCommand(BaseGraphCommand):
install = False
handler = staticmethod(bonobo.run)
def add_arguments(self, parser):
super(RunCommand, self).add_arguments(parser)
@ -16,7 +17,15 @@ class RunCommand(BaseGraphCommand):
parser.add_argument('--install', '-I', action='store_true')
def parse_options(self, *, quiet=False, verbose=False, install=False, **options):
from bonobo import settings
settings.QUIET.set_if_true(quiet)
settings.DEBUG.set_if_true(verbose)
self.install = install
return options
def _run_path(self, file):
# add install logic
if self.install:
if os.path.isdir(file):
requirements = os.path.join(file, 'requirements.txt')
@ -27,24 +36,12 @@ class RunCommand(BaseGraphCommand):
return super()._run_path(file)
def _run_module(self, mod):
# install not implemented for a module, not sure it even make sense.
if self.install:
raise RuntimeError('--install behaviour when running a module is not defined.')
return super()._run_module(mod)
def handle(self, quiet=False, verbose=False, install=False, _remaining_args=None, **options):
from bonobo import settings
settings.QUIET.set_if_true(quiet)
settings.DEBUG.set_if_true(verbose)
self.install = install
graph, params = self.read(args=_remaining_args, **options)
params['plugins'] = set(params.pop('plugins', ())).union(set(options.pop('plugins', ())))
return bonobo.run(graph, **params)
def register_generic_run_arguments(parser, required=True):
"""

28
bonobo/examples/environ.py Normal file
View File

@ -0,0 +1,28 @@
import os
import bonobo
def extract_environ():
yield from sorted(os.environ.items())
def get_graph():
"""
This function builds the graph that needs to be executed.
:return: bonobo.Graph
"""
graph = bonobo.Graph()
graph.add_chain(extract_environ, print)
return graph
# The __main__ block actually execute the graph.
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
parser.add_argument('-v', action='append', dest='vars')
with bonobo.parse_args(parser):
bonobo.run(get_graph())

4
bonobo/examples/tutorials/tut02e02_write.py
View File

@ -8,9 +8,7 @@ def split_one(line):
graph = bonobo.Graph(
bonobo.FileReader('coffeeshops.txt'),
split_one,
bonobo.JsonWriter(
'coffeeshops.json', fs='fs.output'
),
bonobo.JsonWriter('coffeeshops.json', fs='fs.output'),
)

8
bonobo/examples/types/__main__.py
View File

@ -1,3 +1,7 @@
from bonobo.util.python import require
import bonobo
from bonobo.examples.types.strings import get_graph
graph = require('strings').graph
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
with bonobo.parse_args(parser):
bonobo.run(get_graph())

12
bonobo/examples/types/strings.py
View File

@ -14,7 +14,7 @@ Example on how to use symple python strings to communicate between transformatio
"""
from random import randint
from bonobo import Graph
import bonobo
def extract():
@ -31,9 +31,11 @@ def load(s: str):
print(s)
graph = Graph(extract, transform, load)
def get_graph():
return bonobo.Graph(extract, transform, load)
if __name__ == '__main__':
from bonobo import run
run(graph)
parser = bonobo.get_argument_parser()
with bonobo.parse_args(parser):
bonobo.run(get_graph())

6
bonobo/execution/node.py
View File

@ -138,7 +138,7 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext):
def isflag(param):
return isinstance(param, Token) and param in (NOT_MODIFIED,)
return isinstance(param, Token) and param in (NOT_MODIFIED, )
def split_tokens(output):
@ -150,11 +150,11 @@ def split_tokens(output):
"""
if isinstance(output, Token):
# just a flag
return (output,), ()
return (output, ), ()
if not istuple(output):
# no flag
return (), (output,)
return (), (output, )
i = 0
while isflag(output[i]):

1
bonobo/nodes/io/json.py
View File

@ -58,6 +58,7 @@ class LdjsonReader(FileReader):
class LdjsonWriter(FileWriter):
"""Write a stream of JSON objects, one object per line."""
def write(self, fs, file, lineno, **row):
lineno += 1 # class-level variable
file.write(json.dumps(row) + '\n')

154
bonobo/util/environ.py Normal file
View File

@ -0,0 +1,154 @@
import argparse
import codecs
import os
import re
import warnings
from contextlib import contextmanager
__escape_decoder = codecs.getdecoder('unicode_escape')
__posix_variable = re.compile('\$\{[^\}]*\}')
def parse_var(var):
name, value = var.split('=', 1)
def decode_escaped(escaped):
return __escape_decoder(escaped)[0]
if len(value) > 1:
c = value[0]
if c in ['"', "'"] and value[-1] == c:
value = decode_escaped(value[1:-1])
return name, value
def load_env_from_file(filename):
"""
Read an env file into a collection of (name, value) tuples.
"""
if not os.path.exists(filename):
raise FileNotFoundError('Environment file {} does not exist.'.format(filename))
with open(filename) as f:
for lineno, line in enumerate(f):
line = line.strip()
if not line or line.startswith('#'):
continue
if '=' not in line:
raise SyntaxError('Invalid environment file syntax in {} at line {}.'.format(filename, lineno + 1))
name, value = parse_var(line)
yield name, value
_parser = None
def get_argument_parser(parser=None):
"""
Creates an argument parser with arguments to override the system environment.
:api: bonobo.get_argument_parser
:param _parser:
:return:
"""
if parser is None:
import argparse
parser = argparse.ArgumentParser()
# Store globally to be able to warn the user about the fact he's probably wrong not to pass a parser to
# parse_args(), later.
global _parser
_parser = parser
_parser.add_argument('--default-env-file', '-E', action='append')
_parser.add_argument('--default-env', action='append')
_parser.add_argument('--env-file', action='append')
_parser.add_argument('--env', '-e', action='append')
return _parser
@contextmanager
def parse_args(mixed=None):
"""
Context manager to extract and apply environment related options from the provided argparser result.
A dictionnary with unknown options will be yielded, so the remaining options can be used by the caller.
:api: bonobo.patch_environ
:param mixed: ArgumentParser instance, Namespace, or dict.
:return:
"""
if mixed is None:
global _parser
if _parser is not None:
warnings.warn(
'You are calling bonobo.parse_args() without a parser argument, but it looks like you created a parser before. You probably want to pass your parser to this call, or if creating a new parser here is really what you want to do, please create a new one explicitely to silence this warning.'
)
# use the api from bonobo namespace, in case a command patched it.
import bonobo
mixed = bonobo.get_argument_parser()
if isinstance(mixed, argparse.ArgumentParser):
options = mixed.parse_args()
else:
options = mixed
if not isinstance(options, dict):
options = options.__dict__
# make a copy so we don't polute our parent variables.
options = dict(options)
# storage for values before patch.
_backup = {}
# Priority order: --env > --env-file > system > --default-env > --default-env-file
#
# * The code below is reading default-env before default-env-file as if the first sets something, default-env-file
# won't override it.
# * Then, env-file is read from before env, as the behaviour will be the oposite (env will override a var even if
# env-file sets something.)
try:
# Set default environment
for name, value in map(parse_var, options.pop('default_env', []) or []):
if not name in os.environ:
if not name in _backup:
_backup[name] = os.environ.get(name, None)
os.environ[name] = value
# Read and set default environment from file(s)
for filename in options.pop('default_env_file', []) or []:
for name, value in load_env_from_file(filename):
if not name in os.environ:
if not name in _backup:
_backup[name] = os.environ.get(name, None)
os.environ[name] = value
# Read and set environment from file(s)
for filename in options.pop('env_file', []) or []:
for name, value in load_env_from_file(filename):
if not name in _backup:
_backup[name] = os.environ.get(name, None)
os.environ[name] = value
# Set environment
for name, value in map(parse_var, options.pop('env', []) or []):
if not name in _backup:
_backup[name] = os.environ.get(name, None)
os.environ[name] = value
yield options
finally:
for name, value in _backup.items():
if value is None:
del os.environ[name]
else:
os.environ[name] = value

12
tests/features/test_not_modified.py
View File

@ -9,8 +9,12 @@ def useless(*args, **kwargs):
def test_not_modified():
input_messages = [
('foo', 'bar'),
{'foo': 'bar'},
('foo', {'bar': 'baz'}),
{
'foo': 'bar'
},
('foo', {
'bar': 'baz'
}),
(),
]
@ -18,7 +22,3 @@ def test_not_modified():
context.write_sync(*input_messages)
assert context.get_buffer() == input_messages

4
tests/io/test_csv.py
View File

@ -51,11 +51,9 @@ def test_read_csv_from_file_kwargs(tmpdir):
'a': 'a foo',
'b': 'b foo',
'c': 'c foo',
},
{
}, {
'a': 'a bar',
'b': 'b bar',
'c': 'c bar',
}
]

18
tests/io/test_json.py
View File

@ -20,10 +20,10 @@ def test_write_json_ioformat_arg0(tmpdir):
@pytest.mark.parametrize('add_kwargs', (
{},
{
'ioformat': settings.IOFORMAT_KWARGS,
},
{},
{
'ioformat': settings.IOFORMAT_KWARGS,
},
))
def test_write_json_kwargs(tmpdir, add_kwargs):
fs, filename, services = json_tester.get_services_for_writer(tmpdir)
@ -41,8 +41,7 @@ stream_json_tester.input_data = '''{"foo": "bar"}\n{"baz": "boz"}'''
def test_read_stream_json(tmpdir):
fs, filename, services = stream_json_tester.get_services_for_reader(tmpdir)
with BufferingNodeExecutionContext(LdjsonReader(filename),
services=services) as context:
with BufferingNodeExecutionContext(LdjsonReader(filename), services=services) as context:
context.write_sync(tuple())
actual = context.get_buffer()
@ -53,10 +52,11 @@ def test_read_stream_json(tmpdir):
def test_write_stream_json(tmpdir):
fs, filename, services = stream_json_tester.get_services_for_reader(tmpdir)
with BufferingNodeExecutionContext(LdjsonWriter(filename),
services=services) as context:
with BufferingNodeExecutionContext(LdjsonWriter(filename), services=services) as context:
context.write_sync(
{'foo': 'bar'},
{
'foo': 'bar'
},
{'baz': 'boz'},
)

354
tests/test_commands.py
View File

@ -45,7 +45,6 @@ def runner_module(args):
all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module])
single_runner = pytest.mark.parametrize('runner', [runner_module])
def test_entrypoint():
@ -158,238 +157,141 @@ def test_download_fails_non_example(runner):
runner('download', '/something/entirely/different.txt')
@all_runners
class TestDefaultEnvFile(object):
def test_run_file_with_default_env_file(self, runner):
out, err = runner(
'run', '--quiet', '--default-env-file', '.env_one',
get_examples_path('environment/env_files/get_passed_env_file.py')
)
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] != 'marzo'
@pytest.fixture
def env1(tmpdir):
env_file = tmpdir.join('.env_one')
env_file.write('\n'.join((
'SECRET=unknown',
'PASSWORD=sweet',
'PATH=first',
)))
return str(env_file)
def test_run_file_with_multiple_default_env_files(self, runner):
out, err = runner(
'run', '--quiet', '--default-env-file', '.env_one', '--default-env-file', '.env_two',
get_examples_path('environment/env_files/get_passed_env_file.py')
)
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] != 'marzo'
def test_run_module_with_default_env_file(self, runner):
out, err = runner(
'run', '--quiet', '-m', 'bonobo.examples.environment.env_files.get_passed_env_file', '--default-env-file',
'.env_one'
)
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] != 'marzo'
@pytest.fixture
def env2(tmpdir):
env_file = tmpdir.join('.env_two')
env_file.write('\n'.join((
'PASSWORD=bitter',
"PATH='second'",
)))
return str(env_file)
def test_run_module_with_multiple_default_env_files(self, runner):
out, err = runner(
'run',
'--quiet',
all_environ_targets = pytest.mark.parametrize(
'target', [
(get_examples_path('environ.py'), ),
(
'-m',
'bonobo.examples.environment.env_files.get_passed_env_file',
'--default-env-file',
'.env_one',
'--default-env-file',
'.env_two',
)
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] != 'marzo'
'bonobo.examples.environ',
),
]
)
@all_runners
class TestEnvFile(object):
def test_run_file_with_file(self, runner):
out, err = runner(
'run',
'--quiet',
get_examples_path('environment/env_files/get_passed_env_file.py'),
'--env-file',
'.env_one',
@all_environ_targets
class EnvironmentTestCase():
def run_quiet(self, runner, *args):
return runner('run', '--quiet', *args)
def run_environ(self, runner, *args, environ=None):
_environ = {'PATH': '/usr/bin'}
if environ:
_environ.update(environ)
with patch.dict('os.environ', _environ, clear=True):
out, err = self.run_quiet(runner, *args)
assert 'SECRET' not in os.environ
assert 'PASSWORD' not in os.environ
if 'PATH' in _environ:
assert 'PATH' in os.environ
assert os.environ['PATH'] == _environ['PATH']
assert err == ''
return dict(map(lambda line: line.split(' ', 1), filter(None, out.split('\n'))))
class TestDefaultEnvFile(EnvironmentTestCase):
def test_run_with_default_env_file(self, runner, target, env1):
env = self.run_environ(runner, *target, '--default-env-file', env1)
assert env.get('SECRET') == 'unknown'
assert env.get('PASSWORD') == 'sweet'
assert env.get('PATH') == '/usr/bin'
def test_run_with_multiple_default_env_files(self, runner, target, env1, env2):
env = self.run_environ(runner, *target, '--default-env-file', env1, '--default-env-file', env2)
assert env.get('SECRET') == 'unknown'
assert env.get('PASSWORD') == 'sweet'
assert env.get('PATH') == '/usr/bin'
env = self.run_environ(runner, *target, '--default-env-file', env2, '--default-env-file', env1)
assert env.get('SECRET') == 'unknown'
assert env.get('PASSWORD') == 'bitter'
assert env.get('PATH') == '/usr/bin'
class TestEnvFile(EnvironmentTestCase):
def test_run_with_file(self, runner, target, env1):
env = self.run_environ(runner, *target, '--env-file', env1)
assert env.get('SECRET') == 'unknown'
assert env.get('PASSWORD') == 'sweet'
assert env.get('PATH') == 'first'
def test_run_with_multiple_files(self, runner, target, env1, env2):
env = self.run_environ(runner, *target, '--env-file', env1, '--env-file', env2)
assert env.get('SECRET') == 'unknown'
assert env.get('PASSWORD') == 'bitter'
assert env.get('PATH') == 'second'
env = self.run_environ(runner, *target, '--env-file', env2, '--env-file', env1)
assert env.get('SECRET') == 'unknown'
assert env.get('PASSWORD') == 'sweet'
assert env.get('PATH') == 'first'
class TestEnvFileCombinations(EnvironmentTestCase):
def test_run_with_both_env_files(self, runner, target, env1, env2):
env = self.run_environ(runner, *target, '--default-env-file', env1, '--env-file', env2)
assert env.get('SECRET') == 'unknown'
assert env.get('PASSWORD') == 'bitter'
assert env.get('PATH') == 'second'
def test_run_with_both_env_files_then_overrides(self, runner, target, env1, env2):
env = self.run_environ(
runner, *target, '--default-env-file', env1, '--env-file', env2, '--env', 'PASSWORD=mine', '--env',
'SECRET=s3cr3t'
)
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] == 'marzo'
assert env.get('SECRET') == 's3cr3t'
assert env.get('PASSWORD') == 'mine'
assert env.get('PATH') == 'second'
def test_run_file_with_multiple_files(self, runner):
out, err = runner(
'run',
'--quiet',
get_examples_path('environment/env_files/get_passed_env_file.py'),
'--env-file',
'.env_one',
'--env-file',
'.env_two',
class TestEnvVars(EnvironmentTestCase):
def test_run_no_env(self, runner, target):
env = self.run_environ(runner, *target, environ={'USER': 'romain'})
assert env.get('USER') == 'romain'
def test_run_env(self, runner, target):
env = self.run_environ(runner, *target, '--env', 'USER=serious', environ={'USER': 'romain'})
assert env.get('USER') == 'serious'
def test_run_env_mixed(self, runner, target):
env = self.run_environ(runner, *target, '--env', 'ONE=1', '--env', 'TWO="2"', environ={'USER': 'romain'})
assert env.get('USER') == 'romain'
assert env.get('ONE') == '1'
assert env.get('TWO') == '2'
def test_run_default_env(self, runner, target):
env = self.run_environ(runner, *target, '--default-env', 'USER=clown')
assert env.get('USER') == 'clown'
env = self.run_environ(runner, *target, '--default-env', 'USER=clown', environ={'USER': 'romain'})
assert env.get('USER') == 'romain'
env = self.run_environ(
runner, *target, '--env', 'USER=serious', '--default-env', 'USER=clown', environ={
'USER': 'romain'
}
)
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'not_sweet_password'
assert out[2] == 'abril'
def test_run_module_with_file(self, runner):
out, err = runner(
'run',
'--quiet',
'-m',
'bonobo.examples.environment.env_files.get_passed_env_file',
'--env-file',
'.env_one',
)
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] == 'marzo'
def test_run_module_with_multiple_files(self, runner):
out, err = runner(
'run',
'--quiet',
'-m',
'bonobo.examples.environment.env_files.get_passed_env_file',
'--env-file',
'.env_one',
'--env-file',
'.env_two',
)
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'not_sweet_password'
assert out[2] == 'abril'
@all_runners
class TestEnvFileCombinations:
def test_run_file_with_default_env_file_and_env_file(self, runner):
out, err = runner(
'run',
'--quiet',
get_examples_path('environment/env_files/get_passed_env_file.py'),
'--default-env-file',
'.env_one',
'--env-file',
'.env_two',
)
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'not_sweet_password'
assert out[2] == 'abril'
def test_run_file_with_default_env_file_and_env_file_and_env_vars(self, runner):
out, err = runner(
'run',
'--quiet',
get_examples_path('environment/env_files/get_passed_env_file.py'),
'--default-env-file',
'.env_one',
'--env-file',
'.env_two',
'--env',
'TEST_USER_PASSWORD=SWEETpassWORD',
'--env',
'MY_SECRET=444',
)
out = out.split('\n')
assert out[0] == '444'
assert out[1] == 'SWEETpassWORD'
assert out[2] == 'abril'
@all_runners
class TestDefaultEnvVars:
def test_run_file_with_default_env_var(self, runner):
out, err = runner(
'run', '--quiet',
get_examples_path('environment/env_vars/get_passed_env.py'), '--default-env', 'USER=clowncity', '--env',
'USER=ted'
)
out = out.split('\n')
assert out[0] == 'user'
assert out[1] == 'number'
assert out[2] == 'string'
assert out[3] != 'clowncity'
def test_run_file_with_default_env_vars(self, runner):
out, err = runner(
'run', '--quiet',
get_examples_path('environment/env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env',
'ENV_TEST_USER=cwandrews', '--default-env', "ENV_TEST_STRING='my_test_string'"
)
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
def test_run_module_with_default_env_var(self, runner):
out, err = runner(
'run', '--quiet', '-m', 'bonobo.examples.environment.env_vars.get_passed_env', '--env',
'ENV_TEST_NUMBER=123', '--default-env', 'ENV_TEST_STRING=string'
)
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] != 'string'
def test_run_module_with_default_env_vars(self, runner):
out, err = runner(
'run', '--quiet', '-m', 'bonobo.examples.environment.env_vars.get_passed_env', '--env',
'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', '--default-env', "ENV_TEST_STRING='string'"
)
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] != 'string'
@all_runners
class TestEnvVars:
def test_run_file_with_env_var(self, runner):
out, err = runner(
'run', '--quiet',
get_examples_path('environment/env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123'
)
out = out.split('\n')
assert out[0] != 'test_user'
assert out[1] == '123'
assert out[2] == 'my_test_string'
def test_run_file_with_env_vars(self, runner):
out, err = runner(
'run', '--quiet',
get_examples_path('environment/env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env',
'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
)
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
def test_run_module_with_env_var(self, runner):
out, err = runner(
'run', '--quiet', '-m', 'bonobo.examples.environment.env_vars.get_passed_env', '--env',
'ENV_TEST_NUMBER=123'
)
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
def test_run_module_with_env_vars(self, runner):
out, err = runner(
'run', '--quiet', '-m', 'bonobo.examples.environment.env_vars.get_passed_env', '--env',
'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
)
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
assert env.get('USER') == 'serious'