Merge branch 'develop' of github.com:python-bonobo/bonobo into develop

This commit is contained in:
Romain Dorgueil
2017-10-21 12:56:06 +02:00
16 changed files with 394 additions and 69 deletions

View File

@ -45,6 +45,7 @@ python.add_requirements(
'psutil >=5.2,<6.0',
'requests >=2.0,<3.0',
'stevedore >=1.21,<2.0',
'python-dotenv >=0.7.1,<1.0',
dev=[
'cookiecutter >=1.5,<1.6',
'pytest-sugar >=0.8,<0.9',

View File

@ -1,7 +1,10 @@
import codecs
import os
from pathlib import Path
import bonobo
from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME
from dotenv import load_dotenv
DEFAULT_GRAPH_FILENAMES = (
'__main__.py',
@ -43,8 +46,8 @@ def _install_requirements(requirements):
importlib.reload(site)
def read(filename, module, install=False, quiet=False, verbose=False, env=None):
import re
def read(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None):
import runpy
from bonobo import Graph, settings
@ -54,12 +57,6 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None):
if verbose:
settings.DEBUG.set(True)
if env:
quote_killer = re.compile('["\']')
for e in env:
var_name, var_value = e.split('=')
os.environ[var_name] = quote_killer.sub('', var_value)
if filename:
if os.path.isdir(filename):
if install:
@ -83,6 +80,22 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None):
else:
raise RuntimeError('UNEXPECTED: argparse should not allow this.')
env_dir = Path(filename).parent or Path(module).parent
if default_env_file:
for f in default_env_file:
env_file_path = str(env_dir.joinpath(f))
load_dotenv(env_file_path)
if default_env:
for e in default_env:
set_env_var(e)
if env_file:
for f in env_file:
env_file_path = str(env_dir.joinpath(f))
load_dotenv(env_file_path, override=True)
if env:
for e in env:
set_env_var(e, override=True)
graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph))
assert len(graphs) == 1, (
@ -99,8 +112,25 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None):
return graph, plugins, services
def execute(filename, module, install=False, quiet=False, verbose=False, env=None):
graph, plugins, services = read(filename, module, install, quiet, verbose, env)
def set_env_var(e, override=False):
__escape_decoder = codecs.getdecoder('unicode_escape')
ename, evalue = e.split('=', 1)
def decode_escaped(escaped):
return __escape_decoder(escaped)[0]
if len(evalue) > 0:
if evalue[0] == evalue[len(evalue) - 1] in ['"', "'"]:
evalue = decode_escaped(evalue[1:-1])
if override:
os.environ[ename] = evalue
else:
os.environ.setdefault(ename, evalue)
def execute(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None):
graph, plugins, services = read(filename, module, install, quiet, verbose, default_env_file, default_env, env_file, env)
return bonobo.run(graph, plugins=plugins, services=services)
@ -109,6 +139,9 @@ def register_generic_run_arguments(parser, required=True):
source_group = parser.add_mutually_exclusive_group(required=required)
source_group.add_argument('filename', nargs='?', type=str)
source_group.add_argument('--module', '-m', type=str)
parser.add_argument('--default-env-file', action='append')
parser.add_argument('--default-env', action='append')
parser.add_argument('--env-file', action='append')
parser.add_argument('--env', '-e', action='append')
return parser

View File

@ -1,20 +0,0 @@
import os
import bonobo
def extract():
env_test_user = os.getenv('ENV_TEST_USER')
env_test_number = os.getenv('ENV_TEST_NUMBER')
env_test_string = os.getenv('ENV_TEST_STRING')
return env_test_user, env_test_number, env_test_string
def load(s: str):
print(s)
graph = bonobo.Graph(extract, load)
if __name__ == '__main__':
bonobo.run(graph)

View File

@ -0,0 +1,3 @@
MY_SECRET=321
TEST_USER_PASSWORD=sweetpassword
PATH=marzo

View File

@ -0,0 +1,2 @@
TEST_USER_PASSWORD=not_sweet_password
PATH='abril'

View File

@ -0,0 +1,21 @@
import os
import bonobo
def extract():
my_secret = os.getenv('MY_SECRET')
test_user_password = os.getenv('TEST_USER_PASSWORD')
path = os.getenv('PATH')
return my_secret, test_user_password, path
def load(s: str):
print(s)
graph = bonobo.Graph(extract, load)
if __name__ == '__main__':
bonobo.run(graph)

View File

@ -0,0 +1,21 @@
import os
import bonobo
def extract():
env_test_user = os.getenv('ENV_TEST_USER', 'user')
env_test_number = os.getenv('ENV_TEST_NUMBER', 'number')
env_test_string = os.getenv('ENV_TEST_STRING', 'string')
env_user = os.getenv('USER')
return env_test_user, env_test_number, env_test_string, env_user
def load(s: str):
print(s)
graph = bonobo.Graph(extract, load)
if __name__ == '__main__':
bonobo.run(graph)

View File

@ -23,25 +23,76 @@ simply to use the optional ``--env`` argument when running bonobo from the shell
syntax ``VAR_NAME=VAR_VALUE``. Multiple environment variables can be passed by using multiple ``--env`` / ``-e`` flags
(i.e. ``bonobo run --env FIZZ=buzz ...`` and ``bonobo run --env FIZZ=buzz --env Foo=bar ...``). Additionally, in bash
you can also set environment variables by listing those you wish to set before the `bonobo run` command with space
separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``).
separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``). Additionally,
bonobo is able to pull environment variables from local '.env' files rather than having to pass each key-value pair
individually at runtime. Importantly, a strict 'order of priority' is followed when setting environment variables so
it is advisable to read and understand the order listed below to prevent
The order of priority is from lower to higher with the higher "winning" if set:
1. default values
``os.getenv("VARNAME", default_value)``
The user/writer/creator of the graph is responsible for setting these.
2. ``--default-env-file`` values
Specify file to read default env values from. Each env var in the file is used if the var isn't already a corresponding value set at the system environment (system environment vars not overwritten).
3. ``--default-env`` values
Works like #2 but the default ``NAME=var`` are passed individually, with one ``key=value`` pair for each ``--default-env`` flag rather than gathered from a specified file.
4. system environment values
Env vars already set at the system level. It is worth noting that passed env vars via ``NAME=value bonobo run ...`` falls here in the order of priority.
5. ``--env-file`` values
Env vars specified here are set like those in #2 albeit that these values have priority over those set at the system level.
6. ``--env`` values
Env vars set using the ``--env`` / ``-e`` flag work like #3 but take priority over all other env vars.
Examples
::::::::
The Examples below demonstrate setting one or multiple variables using both of these methods:
.. code-block:: bash
# Using one environment variable via --env flag:
# Using one environment variable via a --env or --defualt-env flag:
bonobo run csvsanitizer --env SECRET_TOKEN=secret123
bonobo run csvsanitizer --defaul-env SECRET_TOKEN=secret123
# Using multiple environment variables via -e (env) flag:
# Using multiple environment variables via -e (env) and --default-env flags:
bonobo run csvsanitizer -e SRC_FILE=inventory.txt -e DST_FILE=inventory_processed.csv
bonobo run csvsanitizer --default-env SRC_FILE=inventory.txt --default-env DST_FILE=inventory_processed.csv
# Using one environment variable inline (bash only):
# Using one environment variable inline (bash-like shells only):
SECRET_TOKEN=secret123 bonobo run csvsanitizer
# Using multiple environment variables inline (bash only):
# Using multiple environment variables inline (bash-like shells only):
SRC_FILE=inventory.txt DST_FILE=inventory_processed.csv bonobo run csvsanitizer
*Though not-yet implemented, the bonobo roadmap includes implementing environment / .env files as well.*
# Using an env file for default env values:
bonobo run csvsanitizer --default-env-file .env
# Using an env file for env values:
bonobo run csvsanitizer --env-file '.env.private'
ENV File Structure
::::::::::::::::::
The file structure for env files is incredibly simple. The only text in the file
should be `NAME=value` pairs with one pair per line like the below.
.. code-block:: text
# .env
DB_USER='bonobo'
DB_PASS='cicero'
Accessing Environment Variables from within the Graph Context
:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

View File

@ -2,6 +2,7 @@
appdirs==1.4.3
certifi==2017.7.27.1
chardet==3.0.4
click==6.7
colorama==0.3.9
fs==2.0.12
idna==2.6
@ -9,6 +10,7 @@ packaging==16.8
pbr==3.1.1
psutil==5.4.0
pyparsing==2.2.0
python-dotenv==0.7.1
pytz==2017.2
requests==2.18.4
six==1.11.0

View File

@ -54,7 +54,7 @@ setup(
include_package_data=True,
install_requires=[
'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'packaging (>= 16, < 17)', 'psutil (>= 5.2, < 6.0)',
'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)'
'python-dotenv (>= 0.7.1, < 1.0)', 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)'
],
extras_require={
'dev': [

View File

@ -67,7 +67,7 @@ def test_defaults():
assert o.required_str == 'hello'
assert o.default_str == 'foo'
assert o.integer == None
assert o.integer is None
def test_str_type_factory():
@ -78,7 +78,7 @@ def test_str_type_factory():
assert o.required_str == '42'
assert o.default_str == 'foo'
assert o.integer == None
assert o.integer is None
def test_int_type_factory():
@ -100,8 +100,8 @@ def test_bool_type_factory():
assert o.required_str == 'yes'
assert o.default_str == 'foo'
assert o.integer == None
assert o.also_required == True
assert o.integer is None
assert o.also_required is True
def test_option_resolution_order():
@ -112,7 +112,7 @@ def test_option_resolution_order():
assert o.required_str == 'kaboom'
assert o.default_str == 'foo'
assert o.integer == None
assert o.integer is None
def test_option_positional():

View File

@ -103,27 +103,238 @@ def test_version(runner, capsys):
@all_runners
def test_run_with_env(runner, capsys):
runner(
'run', '--quiet',
get_examples_path('env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env',
'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
class TestDefaultEnvFile(object):
def test_run_file_with_default_env_file(self, runner, capsys):
runner(
'run', '--quiet', '--default-env-file', '.env_one',
get_examples_path('environment/env_files/get_passed_env_file.py')
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] != 'marzo'
def test_run_file_with_multiple_default_env_files(self, runner, capsys):
runner(
'run', '--quiet', '--default-env-file', '.env_one',
'--default-env-file', '.env_two',
get_examples_path('environment/env_files/get_passed_env_file.py')
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] != 'marzo'
def test_run_module_with_default_env_file(self, runner, capsys):
runner(
'run', '--quiet', '-m',
'bonobo.examples.environment.env_files.get_passed_env_file',
'--default-env-file', '.env_one'
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] != 'marzo'
def test_run_module_with_multiple_default_env_files(self, runner, capsys):
runner(
'run', '--quiet', '-m',
'bonobo.examples.environment.env_files.get_passed_env_file',
'--default-env-file', '.env_one', '--default-env-file', '.env_two',
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] != 'marzo'
@all_runners
def test_run_module_with_env(runner, capsys):
runner(
'run', '--quiet', '-m', 'bonobo.examples.env_vars.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env',
'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
class TestEnvFile(object):
def test_run_file_with_file(self, runner, capsys):
runner(
'run', '--quiet',
get_examples_path('environment/env_files/get_passed_env_file.py'),
'--env-file', '.env_one',
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] == 'marzo'
def test_run_file_with_multiple_files(self, runner, capsys):
runner(
'run', '--quiet',
get_examples_path('environment/env_files/get_passed_env_file.py'),
'--env-file', '.env_one', '--env-file', '.env_two',
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'not_sweet_password'
assert out[2] == 'abril'
def test_run_module_with_file(self, runner, capsys):
runner(
'run', '--quiet', '-m',
'bonobo.examples.environment.env_files.get_passed_env_file',
'--env-file', '.env_one',
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'sweetpassword'
assert out[2] == 'marzo'
def test_run_module_with_multiple_files(self, runner, capsys):
runner(
'run', '--quiet', '-m',
'bonobo.examples.environment.env_files.get_passed_env_file',
'--env-file', '.env_one', '--env-file', '.env_two',
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'not_sweet_password'
assert out[2] == 'abril'
@all_runners
class TestEnvFileCombinations(object):
def test_run_file_with_default_env_file_and_env_file(self, runner, capsys):
runner(
'run', '--quiet',
get_examples_path('environment/env_files/get_passed_env_file.py'),
'--default-env-file', '.env_one', '--env-file', '.env_two',
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '321'
assert out[1] == 'not_sweet_password'
assert out[2] == 'abril'
def test_run_file_with_default_env_file_and_env_file_and_env_vars(self, runner, capsys):
runner(
'run', '--quiet',
get_examples_path('environment/env_files/get_passed_env_file.py'),
'--default-env-file', '.env_one', '--env-file', '.env_two',
'--env', 'TEST_USER_PASSWORD=SWEETpassWORD', '--env',
'MY_SECRET=444',
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == '444'
assert out[1] == 'SWEETpassWORD'
assert out[2] == 'abril'
@all_runners
class TestDefaultEnvVars(object):
def test_run_file_with_default_env_var(self, runner, capsys):
runner(
'run', '--quiet',
get_examples_path('environment/env_vars/get_passed_env.py'),
'--default-env', 'USER=clowncity', '--env', 'USER=ted'
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'user'
assert out[1] == 'number'
assert out[2] == 'string'
assert out[3] != 'clowncity'
def test_run_file_with_default_env_vars(self, runner, capsys):
runner(
'run', '--quiet',
get_examples_path('environment/env_vars/get_passed_env.py'),
'--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
'--default-env', "ENV_TEST_STRING='my_test_string'"
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
def test_run_module_with_default_env_var(self, runner, capsys):
runner(
'run', '--quiet', '-m',
'bonobo.examples.environment.env_vars.get_passed_env',
'--env', 'ENV_TEST_NUMBER=123',
'--default-env', 'ENV_TEST_STRING=string'
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] != 'string'
def test_run_module_with_default_env_vars(self, runner, capsys):
runner(
'run', '--quiet', '-m',
'bonobo.examples.environment.env_vars.get_passed_env',
'--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
'--default-env', "ENV_TEST_STRING='string'"
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] != 'string'
@all_runners
class TestEnvVars(object):
def test_run_file_with_env_var(self, runner, capsys):
runner(
'run', '--quiet',
get_examples_path('environment/env_vars/get_passed_env.py'),
'--env', 'ENV_TEST_NUMBER=123'
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] != 'test_user'
assert out[1] == '123'
assert out[2] == 'my_test_string'
def test_run_file_with_env_vars(self, runner, capsys):
runner(
'run', '--quiet',
get_examples_path('environment/env_vars/get_passed_env.py'),
'--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
'--env', "ENV_TEST_STRING='my_test_string'"
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
def test_run_module_with_env_var(self, runner, capsys):
runner(
'run', '--quiet', '-m',
'bonobo.examples.environment.env_vars.get_passed_env',
'--env', 'ENV_TEST_NUMBER=123'
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
def test_run_module_with_env_vars(self, runner, capsys):
runner(
'run', '--quiet', '-m',
'bonobo.examples.environment.env_vars.get_passed_env',
'--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
'--env', "ENV_TEST_STRING='my_test_string'"
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'

View File

@ -42,9 +42,9 @@ def test_setting():
def test_default_settings():
settings.clear_all()
assert settings.DEBUG.get() == False
assert settings.PROFILE.get() == False
assert settings.QUIET.get() == False
assert settings.DEBUG.get() is False
assert settings.PROFILE.get() is False
assert settings.QUIET.get() is False
assert settings.LOGGING_LEVEL.get() == logging._checkLevel('INFO')
with patch.dict(environ, {'DEBUG': 't'}):

View File

@ -18,5 +18,5 @@ def test_force_iterator_with_generator():
yield 'ccc'
iterator = force_iterator(generator())
assert type(iterator) == types.GeneratorType
assert isinstance(iterator, types.GeneratorType)
assert list(iterator) == ['aaa', 'bbb', 'ccc']