From 2de72b52bd764ab030c8cd81d1d511a0e3e9b2f2 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 8 Oct 2017 19:50:15 -0400 Subject: [PATCH 01/19] Refactored setting of env vars passed via the env flag. --- bonobo/commands/run.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index a37282c..23298f5 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -1,3 +1,4 @@ +import codecs import os import bonobo @@ -41,7 +42,7 @@ def _install_requirements(requirements): def read(filename, module, install=False, quiet=False, verbose=False, env=None): - import re + import runpy from bonobo import Graph, settings @@ -52,10 +53,21 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): settings.DEBUG.set(True) if env: - quote_killer = re.compile('["\']') + __escape_decoder = codecs.getdecoder('unicode_escape') + + def decode_escaped(escaped): + return __escape_decoder(escaped)[0] + for e in env: - var_name, var_value = e.split('=') - os.environ[var_name] = quote_killer.sub('', var_value) + ename, evalue = e.split('=', 1) + + if len(evalue) > 0: + quoted = evalue[0] == evalue[len(evalue) - 1] in ['"', "'"] + + if quoted: + evalue = decode_escaped(evalue[1:-1]) + + os.environ[ename] = evalue if filename: if os.path.isdir(filename): From e3125e7e02d461ee7f47e4b5d70dcd3493e68783 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 8 Oct 2017 19:52:25 -0400 Subject: [PATCH 02/19] Further Refactored the setting of env vars passed via the env flag. --- bonobo/commands/run.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 23298f5..4757a54 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -62,9 +62,7 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): ename, evalue = e.split('=', 1) if len(evalue) > 0: - quoted = evalue[0] == evalue[len(evalue) - 1] in ['"', "'"] - - if quoted: + if evalue[0] == evalue[len(evalue) - 1] in ['"', "'"]: evalue = decode_escaped(evalue[1:-1]) os.environ[ename] = evalue From 88956ba6fefb01e359cb9b1fbe912a77f1a1c5ad Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 8 Oct 2017 21:02:31 -0400 Subject: [PATCH 03/19] default-env-file, default-env, and env-file now in place alongside env. default-env-file and default-env both use os.environ.setdefault so as not to overwrite existing variables (system environment) while env-file and env will overwrite existing variables. All four allow for multiple values (***How might this affect multiple default-env and default-env-file values, I expect that unlike env-file and env the first passed variables would win). --- bonobo/commands/run.py | 69 ++++++++++++++----- .../examples/env_vars/get_passed_env_file.py | 22 ++++++ tests/test_commands.py | 27 ++++++++ 3 files changed, 100 insertions(+), 18 deletions(-) create mode 100644 bonobo/examples/env_vars/get_passed_env_file.py diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 4757a54..59190be 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -1,8 +1,10 @@ import codecs import os +from pathlib import Path import bonobo from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME +from dotenv import load_dotenv DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', ) DEFAULT_GRAPH_ATTR = 'get_graph' @@ -41,7 +43,7 @@ def _install_requirements(requirements): importlib.reload(site) -def read(filename, module, install=False, quiet=False, verbose=False, env=None): +def read(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None): import runpy from bonobo import Graph, settings @@ -52,21 +54,6 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): if verbose: settings.DEBUG.set(True) - if env: - __escape_decoder = codecs.getdecoder('unicode_escape') - - def decode_escaped(escaped): - return __escape_decoder(escaped)[0] - - for e in env: - ename, evalue = e.split('=', 1) - - if len(evalue) > 0: - if evalue[0] == evalue[len(evalue) - 1] in ['"', "'"]: - evalue = decode_escaped(evalue[1:-1]) - - os.environ[ename] = evalue - if filename: if os.path.isdir(filename): if install: @@ -84,12 +71,38 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): requirements = os.path.join(os.path.dirname(filename), 'requirements.txt') _install_requirements(requirements) context = runpy.run_path(filename, run_name='__bonobo__') + env_dir = Path(filename).parent elif module: context = runpy.run_module(module, run_name='__bonobo__') filename = context['__file__'] + env_dir = Path(module) else: raise RuntimeError('UNEXPECTED: argparse should not allow this.') + if default_env_file: + for f in default_env_file: + env_file_path = env_dir.joinpath(f) + load_dotenv(env_file_path) + else: + try: + env_file_path = env_dir.joinpath('.env') + load_dotenv(env_file_path) + except FileNotFoundError: + pass + + if default_env: + for e in default_env: + set_env_var(e) + + if env_file: + for f in env_file: + env_file_path = env_dir.joinpath(f) + load_dotenv(env_file_path, override=True) + + if env: + for e in env: + set_env_var(e, override=True) + graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph)) assert len(graphs) == 1, ( @@ -106,8 +119,25 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): return graph, plugins, services -def execute(filename, module, install=False, quiet=False, verbose=False, env=None): - graph, plugins, services = read(filename, module, install, quiet, verbose, env) +def set_env_var(e, override=False): + __escape_decoder = codecs.getdecoder('unicode_escape') + ename, evalue = e.split('=', 1) + + def decode_escaped(escaped): + return __escape_decoder(escaped)[0] + + if len(evalue) > 0: + if evalue[0] == evalue[len(evalue) - 1] in ['"', "'"]: + evalue = decode_escaped(evalue[1:-1]) + + if override: + os.environ[ename] = evalue + else: + os.environ.setdefault(ename, evalue) + + +def execute(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None): + graph, plugins, services = read(filename, module, install, quiet, verbose, default_env_file, default_env, env_file, env) return bonobo.run(graph, plugins=plugins, services=services) @@ -116,6 +146,9 @@ def register_generic_run_arguments(parser, required=True): source_group = parser.add_mutually_exclusive_group(required=required) source_group.add_argument('filename', nargs='?', type=str) source_group.add_argument('--module', '-m', type=str) + parser.add_argument('--default-env-file', action='append') + parser.add_argument('--default-env', action='append') + parser.add_argument('--env-file', action='append') parser.add_argument('--env', '-e', action='append') return parser diff --git a/bonobo/examples/env_vars/get_passed_env_file.py b/bonobo/examples/env_vars/get_passed_env_file.py new file mode 100644 index 0000000..e7a0952 --- /dev/null +++ b/bonobo/examples/env_vars/get_passed_env_file.py @@ -0,0 +1,22 @@ +import os + +import bonobo + + +def extract(): + my_secret = os.getenv('MY_SECRET') + test_user_password = os.getenv('TEST_USER_PASSWORD') + user = os.getenv('USERNAME') + path = os.getenv('PATH') + + return my_secret, test_user_password, user, path + + +def load(s: str): + print(s) + + +graph = bonobo.Graph(extract, load) + +if __name__ == '__main__': + bonobo.run(graph) diff --git a/tests/test_commands.py b/tests/test_commands.py index a29465c..ce7b582 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -98,6 +98,33 @@ def test_version(runner, capsys): assert __version__ in out +@all_runners +def test_run_module_with_default_env_file(runner, capsys): + runner( + 'run', '--quiet', get_examples_path('env_vars/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'not_cwandrews_123' + assert out[3] != 'marzo' + + +# @all_runners +# def test_run_with_env_file(runner, capsys): +# runner( +# 'run', '--quiet', +# get_examples_path('env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env', +# 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" +# ) +# out, err = capsys.readouterr() +# out = out.split('\n') +# assert out[0] == 'cwandrews' +# assert out[1] == '123' +# assert out[2] == 'my_test_string' + + @all_runners def test_run_with_env(runner, capsys): runner( From e469ba30ba66f743e52b9b52f70d827087b77250 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 8 Oct 2017 21:13:51 -0400 Subject: [PATCH 04/19] Updated requirements.txt and requirements-dev.txt to include python-dotenv and dependencies. --- requirements-dev.txt | 1 + requirements.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4e005a7..2fa4ef8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,6 +23,7 @@ pytest-sugar==0.8.0 pytest-timeout==1.2.0 pytest==3.2.3 python-dateutil==2.6.1 +python-dotenv==0.7.1 pytz==2017.2 requests==2.18.4 six==1.11.0 diff --git a/requirements.txt b/requirements.txt index d6439df..9e604ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ appdirs==1.4.3 certifi==2017.7.27.1 chardet==3.0.4 +click==6.7 colorama==0.3.9 fs==2.0.11 idna==2.6 @@ -9,6 +10,7 @@ packaging==16.8 pbr==3.1.1 psutil==5.3.1 pyparsing==2.2.0 +python-dotenv==0.7.1 pytz==2017.2 requests==2.18.4 six==1.11.0 From af15647ab4832a4442aa6ba4f5470ddc97631a09 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Tue, 10 Oct 2017 22:54:31 -0400 Subject: [PATCH 05/19] Added tests for running file with combinations of multiple default env files, env files, and env vars. Also reorganized environment directory in examples. --- bonobo/commands/run.py | 6 - bonobo/examples/environment/env_files/.env2 | 2 + .../env_files}/__init__.py | 0 .../env_files}/get_passed_env_file.py | 3 +- .../examples/environment/env_vars/__init__.py | 0 .../env_vars/get_passed_env.py | 0 tests/test_commands.py | 104 ++++++++++++++---- 7 files changed, 83 insertions(+), 32 deletions(-) create mode 100644 bonobo/examples/environment/env_files/.env2 rename bonobo/examples/{env_vars => environment/env_files}/__init__.py (100%) rename bonobo/examples/{env_vars => environment/env_files}/get_passed_env_file.py (77%) create mode 100644 bonobo/examples/environment/env_vars/__init__.py rename bonobo/examples/{ => environment}/env_vars/get_passed_env.py (100%) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 59190be..033e17c 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -83,12 +83,6 @@ def read(filename, module, install=False, quiet=False, verbose=False, default_en for f in default_env_file: env_file_path = env_dir.joinpath(f) load_dotenv(env_file_path) - else: - try: - env_file_path = env_dir.joinpath('.env') - load_dotenv(env_file_path) - except FileNotFoundError: - pass if default_env: for e in default_env: diff --git a/bonobo/examples/environment/env_files/.env2 b/bonobo/examples/environment/env_files/.env2 new file mode 100644 index 0000000..1b91848 --- /dev/null +++ b/bonobo/examples/environment/env_files/.env2 @@ -0,0 +1,2 @@ +TEST_USER_PASSWORD='not_sweet_password' +PATH='abril' \ No newline at end of file diff --git a/bonobo/examples/env_vars/__init__.py b/bonobo/examples/environment/env_files/__init__.py similarity index 100% rename from bonobo/examples/env_vars/__init__.py rename to bonobo/examples/environment/env_files/__init__.py diff --git a/bonobo/examples/env_vars/get_passed_env_file.py b/bonobo/examples/environment/env_files/get_passed_env_file.py similarity index 77% rename from bonobo/examples/env_vars/get_passed_env_file.py rename to bonobo/examples/environment/env_files/get_passed_env_file.py index e7a0952..bb83e67 100644 --- a/bonobo/examples/env_vars/get_passed_env_file.py +++ b/bonobo/examples/environment/env_files/get_passed_env_file.py @@ -6,10 +6,9 @@ import bonobo def extract(): my_secret = os.getenv('MY_SECRET') test_user_password = os.getenv('TEST_USER_PASSWORD') - user = os.getenv('USERNAME') path = os.getenv('PATH') - return my_secret, test_user_password, user, path + return my_secret, test_user_password, path def load(s: str): diff --git a/bonobo/examples/environment/env_vars/__init__.py b/bonobo/examples/environment/env_vars/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bonobo/examples/env_vars/get_passed_env.py b/bonobo/examples/environment/env_vars/get_passed_env.py similarity index 100% rename from bonobo/examples/env_vars/get_passed_env.py rename to bonobo/examples/environment/env_vars/get_passed_env.py diff --git a/tests/test_commands.py b/tests/test_commands.py index ce7b582..fbede11 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -99,38 +99,92 @@ def test_version(runner, capsys): @all_runners -def test_run_module_with_default_env_file(runner, capsys): +def test_run_file_with_default_env_file(runner, capsys): runner( - 'run', '--quiet', get_examples_path('env_vars/get_passed_env_file.py') + 'run', '--quiet', '--default-env-file', '.env', + get_examples_path('environment/env_files/get_passed_env_file.py') ) out, err = capsys.readouterr() out = out.split('\n') assert out[0] == '321' assert out[1] == 'sweetpassword' - assert out[2] != 'not_cwandrews_123' - assert out[3] != 'marzo' - - -# @all_runners -# def test_run_with_env_file(runner, capsys): -# runner( -# 'run', '--quiet', -# get_examples_path('env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env', -# 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" -# ) -# out, err = capsys.readouterr() -# out = out.split('\n') -# assert out[0] == 'cwandrews' -# assert out[1] == '123' -# assert out[2] == 'my_test_string' + assert out[2] != 'marzo' @all_runners -def test_run_with_env(runner, capsys): +def test_run_file_with_multiple_default_env_files(runner, capsys): + runner( + 'run', '--quiet', '--default-env-file', '.env', + '--default-env-file', '.env2', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' + + +@all_runners +def test_run_file_with_env_file(runner, capsys): + runner( + 'run', '--quiet', '--env-file', '.env', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] == 'marzo' + + +@all_runners +def test_run_file_with_multiple_env_files(runner, capsys): + runner( + 'run', '--quiet', '--env-file', '.env', '--env-file', '.env2', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' + + +@all_runners +def test_run_file_with_default_env_file_and_env_file(runner, capsys): + runner( + 'run', '--quiet', '--default-env-file', '.env', '--env-file', '.env2', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' + + +@all_runners +def test_run_file_with_default_env_file_and_env_file_and_env_vars(runner, capsys): + runner( + 'run', '--quiet', '--default-env-file', '.env', '--env-file', '.env2', + '--env', 'TEST_USER_PASSWORD=SWEETpass', '--env', 'MY_SECRET=444', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '444' + assert out[1] == 'SWEETpass' + assert out[2] == 'abril' + + +@all_runners +def test_run_file_with_env_vars(runner, capsys): runner( 'run', '--quiet', - get_examples_path('env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env', - 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'" ) out, err = capsys.readouterr() out = out.split('\n') @@ -140,10 +194,12 @@ def test_run_with_env(runner, capsys): @all_runners -def test_run_module_with_env(runner, capsys): +def test_run_module_with_env_vars(runner, capsys): runner( - 'run', '--quiet', '-m', 'bonobo.examples.env_vars.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env', - 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'" ) out, err = capsys.readouterr() out = out.split('\n') From 1aada995965f1b2738837246a8be4bb391d5740a Mon Sep 17 00:00:00 2001 From: cwandrews Date: Tue, 10 Oct 2017 22:56:58 -0400 Subject: [PATCH 06/19] Small adjustment to test parameters. --- tests/test_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index fbede11..92c279a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -168,13 +168,13 @@ def test_run_file_with_default_env_file_and_env_file(runner, capsys): def test_run_file_with_default_env_file_and_env_file_and_env_vars(runner, capsys): runner( 'run', '--quiet', '--default-env-file', '.env', '--env-file', '.env2', - '--env', 'TEST_USER_PASSWORD=SWEETpass', '--env', 'MY_SECRET=444', + '--env', 'TEST_USER_PASSWORD=SWEETpassWORD', '--env', 'MY_SECRET=444', get_examples_path('environment/env_files/get_passed_env_file.py') ) out, err = capsys.readouterr() out = out.split('\n') assert out[0] == '444' - assert out[1] == 'SWEETpass' + assert out[1] == 'SWEETpassWORD' assert out[2] == 'abril' From 489d22cbcb0e33609fff246484d3d27a51459d70 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Wed, 11 Oct 2017 20:49:57 -0400 Subject: [PATCH 07/19] Moved default-env-file tests to class. --- tests/test_commands.py | 73 +++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 92c279a..adb0317 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -98,31 +98,58 @@ def test_version(runner, capsys): assert __version__ in out -@all_runners -def test_run_file_with_default_env_file(runner, capsys): - runner( - 'run', '--quiet', '--default-env-file', '.env', - get_examples_path('environment/env_files/get_passed_env_file.py') - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == '321' - assert out[1] == 'sweetpassword' - assert out[2] != 'marzo' +class TestDefaultEnvFile(object): + @all_runners + def test_run_file_with_default_env_file(self, runner, capsys): + runner( + 'run', '--quiet', '--default-env-file', '.env', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' -@all_runners -def test_run_file_with_multiple_default_env_files(runner, capsys): - runner( - 'run', '--quiet', '--default-env-file', '.env', - '--default-env-file', '.env2', - get_examples_path('environment/env_files/get_passed_env_file.py') - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == '321' - assert out[1] == 'sweetpassword' - assert out[2] != 'marzo' + @all_runners + def test_run_file_with_multiple_default_env_files(self, runner, capsys): + runner( + 'run', '--quiet', '--default-env-file', '.env', + '--default-env-file', '.env2', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' + + @all_runners + def test_run_module_with_default_env_file(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--default-env-file', '.env' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' + + @all_runners + def test_run_module_with_multiple_default_env_files(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--default-env-file', '.env', '--default-env-file', '.env2', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' @all_runners From 53f6cc055fb9bbc1cb15f393ef0cec434101eb82 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Wed, 11 Oct 2017 21:49:39 -0400 Subject: [PATCH 08/19] Fixed bug involved in finding env when running module. --- bonobo/commands/run.py | 4 +- bonobo/examples/environment/env_files/.env2 | 2 - .../examples/environment/env_files/.env_two | 2 + tests/test_commands.py | 87 ++++++++++++------- 4 files changed, 59 insertions(+), 36 deletions(-) delete mode 100644 bonobo/examples/environment/env_files/.env2 create mode 100644 bonobo/examples/environment/env_files/.env_two diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 033e17c..7b23b9d 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -71,14 +71,14 @@ def read(filename, module, install=False, quiet=False, verbose=False, default_en requirements = os.path.join(os.path.dirname(filename), 'requirements.txt') _install_requirements(requirements) context = runpy.run_path(filename, run_name='__bonobo__') - env_dir = Path(filename).parent elif module: context = runpy.run_module(module, run_name='__bonobo__') filename = context['__file__'] - env_dir = Path(module) else: raise RuntimeError('UNEXPECTED: argparse should not allow this.') + env_dir = Path(filename).parent or Path(module).parent + if default_env_file: for f in default_env_file: env_file_path = env_dir.joinpath(f) diff --git a/bonobo/examples/environment/env_files/.env2 b/bonobo/examples/environment/env_files/.env2 deleted file mode 100644 index 1b91848..0000000 --- a/bonobo/examples/environment/env_files/.env2 +++ /dev/null @@ -1,2 +0,0 @@ -TEST_USER_PASSWORD='not_sweet_password' -PATH='abril' \ No newline at end of file diff --git a/bonobo/examples/environment/env_files/.env_two b/bonobo/examples/environment/env_files/.env_two new file mode 100644 index 0000000..672d6d2 --- /dev/null +++ b/bonobo/examples/environment/env_files/.env_two @@ -0,0 +1,2 @@ +TEST_USER_PASSWORD=not_sweet_password +PATH='abril' \ No newline at end of file diff --git a/tests/test_commands.py b/tests/test_commands.py index adb0317..59f3db8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -98,9 +98,8 @@ def test_version(runner, capsys): assert __version__ in out +@all_runners class TestDefaultEnvFile(object): - - @all_runners def test_run_file_with_default_env_file(self, runner, capsys): runner( 'run', '--quiet', '--default-env-file', '.env', @@ -112,11 +111,10 @@ class TestDefaultEnvFile(object): assert out[1] == 'sweetpassword' assert out[2] != 'marzo' - @all_runners def test_run_file_with_multiple_default_env_files(self, runner, capsys): runner( 'run', '--quiet', '--default-env-file', '.env', - '--default-env-file', '.env2', + '--default-env-file', '.env_two', get_examples_path('environment/env_files/get_passed_env_file.py') ) out, err = capsys.readouterr() @@ -125,7 +123,6 @@ class TestDefaultEnvFile(object): assert out[1] == 'sweetpassword' assert out[2] != 'marzo' - @all_runners def test_run_module_with_default_env_file(self, runner, capsys): runner( 'run', '--quiet', '-m', @@ -138,12 +135,11 @@ class TestDefaultEnvFile(object): assert out[1] == 'sweetpassword' assert out[2] != 'marzo' - @all_runners def test_run_module_with_multiple_default_env_files(self, runner, capsys): runner( 'run', '--quiet', '-m', 'bonobo.examples.environment.env_files.get_passed_env_file', - '--default-env-file', '.env', '--default-env-file', '.env2', + '--default-env-file', '.env', '--default-env-file', '.env_two', ) out, err = capsys.readouterr() out = out.split('\n') @@ -153,36 +149,62 @@ class TestDefaultEnvFile(object): @all_runners -def test_run_file_with_env_file(runner, capsys): - runner( - 'run', '--quiet', '--env-file', '.env', - get_examples_path('environment/env_files/get_passed_env_file.py') - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == '321' - assert out[1] == 'sweetpassword' - assert out[2] == 'marzo' +class TestEnvFile(object): + def test_run_file_with_file(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--env-file', '.env', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] == 'marzo' + def test_run_file_with_multiple_files(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--env-file', '.env', '--env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' -@all_runners -def test_run_file_with_multiple_env_files(runner, capsys): - runner( - 'run', '--quiet', '--env-file', '.env', '--env-file', '.env2', - get_examples_path('environment/env_files/get_passed_env_file.py') - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == '321' - assert out[1] == 'not_sweet_password' - assert out[2] == 'abril' + def test_run_module_with_file(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--env-file', '.env', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] == 'marzo' + + def test_run_module_with_multiple_files(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--env-file', '.env', '--env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' @all_runners def test_run_file_with_default_env_file_and_env_file(runner, capsys): runner( - 'run', '--quiet', '--default-env-file', '.env', '--env-file', '.env2', - get_examples_path('environment/env_files/get_passed_env_file.py') + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--default-env-file', '.env', '--env-file', '.env_two', ) out, err = capsys.readouterr() out = out.split('\n') @@ -194,9 +216,10 @@ def test_run_file_with_default_env_file_and_env_file(runner, capsys): @all_runners def test_run_file_with_default_env_file_and_env_file_and_env_vars(runner, capsys): runner( - 'run', '--quiet', '--default-env-file', '.env', '--env-file', '.env2', + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--default-env-file', '.env', '--env-file', '.env_two', '--env', 'TEST_USER_PASSWORD=SWEETpassWORD', '--env', 'MY_SECRET=444', - get_examples_path('environment/env_files/get_passed_env_file.py') ) out, err = capsys.readouterr() out = out.split('\n') From fe4964b9c7da08d52e81630ad94a2709e5abe159 Mon Sep 17 00:00:00 2001 From: mouadhkaabachi Date: Thu, 12 Oct 2017 17:46:28 +0200 Subject: [PATCH 09/19] comparison to None|True|False should be 'if cond is None:' --- tests/config/test_configurables.py | 10 +++++----- tests/test_settings.py | 6 +++--- tests/util/test_iterators.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/config/test_configurables.py b/tests/config/test_configurables.py index e1555d7..bee501e 100644 --- a/tests/config/test_configurables.py +++ b/tests/config/test_configurables.py @@ -67,7 +67,7 @@ def test_defaults(): assert o.required_str == 'hello' assert o.default_str == 'foo' - assert o.integer == None + assert o.integer is None def test_str_type_factory(): @@ -78,7 +78,7 @@ def test_str_type_factory(): assert o.required_str == '42' assert o.default_str == 'foo' - assert o.integer == None + assert o.integer is None def test_int_type_factory(): @@ -100,8 +100,8 @@ def test_bool_type_factory(): assert o.required_str == 'yes' assert o.default_str == 'foo' - assert o.integer == None - assert o.also_required == True + assert o.integer is None + assert o.also_required is True def test_option_resolution_order(): @@ -112,7 +112,7 @@ def test_option_resolution_order(): assert o.required_str == 'kaboom' assert o.default_str == 'foo' - assert o.integer == None + assert o.integer is None def test_option_positional(): diff --git a/tests/test_settings.py b/tests/test_settings.py index c8313c5..5771dc4 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -42,9 +42,9 @@ def test_setting(): def test_default_settings(): settings.clear_all() - assert settings.DEBUG.get() == False - assert settings.PROFILE.get() == False - assert settings.QUIET.get() == False + assert settings.DEBUG.get() is False + assert settings.PROFILE.get() is False + assert settings.QUIET.get() is False assert settings.LOGGING_LEVEL.get() == logging._checkLevel('INFO') with patch.dict(environ, {'DEBUG': 't'}): diff --git a/tests/util/test_iterators.py b/tests/util/test_iterators.py index 22b34d7..3d0249e 100644 --- a/tests/util/test_iterators.py +++ b/tests/util/test_iterators.py @@ -18,5 +18,5 @@ def test_force_iterator_with_generator(): yield 'ccc' iterator = force_iterator(generator()) - assert type(iterator) == types.GeneratorType + assert isinstance(iterator, types.GeneratorType) assert list(iterator) == ['aaa', 'bbb', 'ccc'] From 84e197b209dc1ae0381a02af66a31dade90545fd Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 15 Oct 2017 15:39:23 -0400 Subject: [PATCH 10/19] Updated .env >>> .env_one to include in repo (.env ignored). --- .../examples/environment/env_files/.env_one | 3 +++ tests/test_commands.py | 20 +++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 bonobo/examples/environment/env_files/.env_one diff --git a/bonobo/examples/environment/env_files/.env_one b/bonobo/examples/environment/env_files/.env_one new file mode 100644 index 0000000..65f2b17 --- /dev/null +++ b/bonobo/examples/environment/env_files/.env_one @@ -0,0 +1,3 @@ +MY_SECRET=321 +TEST_USER_PASSWORD=sweetpassword +PATH=marzo \ No newline at end of file diff --git a/tests/test_commands.py b/tests/test_commands.py index 59f3db8..3f996cb 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -102,7 +102,7 @@ def test_version(runner, capsys): class TestDefaultEnvFile(object): def test_run_file_with_default_env_file(self, runner, capsys): runner( - 'run', '--quiet', '--default-env-file', '.env', + 'run', '--quiet', '--default-env-file', '.env_one', get_examples_path('environment/env_files/get_passed_env_file.py') ) out, err = capsys.readouterr() @@ -113,7 +113,7 @@ class TestDefaultEnvFile(object): def test_run_file_with_multiple_default_env_files(self, runner, capsys): runner( - 'run', '--quiet', '--default-env-file', '.env', + 'run', '--quiet', '--default-env-file', '.env_one', '--default-env-file', '.env_two', get_examples_path('environment/env_files/get_passed_env_file.py') ) @@ -127,7 +127,7 @@ class TestDefaultEnvFile(object): runner( 'run', '--quiet', '-m', 'bonobo.examples.environment.env_files.get_passed_env_file', - '--default-env-file', '.env' + '--default-env-file', '.env_one' ) out, err = capsys.readouterr() out = out.split('\n') @@ -139,7 +139,7 @@ class TestDefaultEnvFile(object): runner( 'run', '--quiet', '-m', 'bonobo.examples.environment.env_files.get_passed_env_file', - '--default-env-file', '.env', '--default-env-file', '.env_two', + '--default-env-file', '.env_one', '--default-env-file', '.env_two', ) out, err = capsys.readouterr() out = out.split('\n') @@ -154,7 +154,7 @@ class TestEnvFile(object): runner( 'run', '--quiet', get_examples_path('environment/env_files/get_passed_env_file.py'), - '--env-file', '.env', + '--env-file', '.env_one', ) out, err = capsys.readouterr() out = out.split('\n') @@ -166,7 +166,7 @@ class TestEnvFile(object): runner( 'run', '--quiet', get_examples_path('environment/env_files/get_passed_env_file.py'), - '--env-file', '.env', '--env-file', '.env_two', + '--env-file', '.env_one', '--env-file', '.env_two', ) out, err = capsys.readouterr() out = out.split('\n') @@ -178,7 +178,7 @@ class TestEnvFile(object): runner( 'run', '--quiet', '-m', 'bonobo.examples.environment.env_files.get_passed_env_file', - '--env-file', '.env', + '--env-file', '.env_one', ) out, err = capsys.readouterr() out = out.split('\n') @@ -190,7 +190,7 @@ class TestEnvFile(object): runner( 'run', '--quiet', '-m', 'bonobo.examples.environment.env_files.get_passed_env_file', - '--env-file', '.env', '--env-file', '.env_two', + '--env-file', '.env_one', '--env-file', '.env_two', ) out, err = capsys.readouterr() out = out.split('\n') @@ -204,7 +204,7 @@ def test_run_file_with_default_env_file_and_env_file(runner, capsys): runner( 'run', '--quiet', get_examples_path('environment/env_files/get_passed_env_file.py'), - '--default-env-file', '.env', '--env-file', '.env_two', + '--default-env-file', '.env_one', '--env-file', '.env_two', ) out, err = capsys.readouterr() out = out.split('\n') @@ -218,7 +218,7 @@ def test_run_file_with_default_env_file_and_env_file_and_env_vars(runner, capsys runner( 'run', '--quiet', get_examples_path('environment/env_files/get_passed_env_file.py'), - '--default-env-file', '.env', '--env-file', '.env_two', + '--default-env-file', '.env_one', '--env-file', '.env_two', '--env', 'TEST_USER_PASSWORD=SWEETpassWORD', '--env', 'MY_SECRET=444', ) out, err = capsys.readouterr() From dc34ab4a8b560ec163cba5e9dd98f7d8025df335 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 15 Oct 2017 15:49:14 -0400 Subject: [PATCH 11/19] Moved env vars tests to class. --- .../environment/env_vars/get_passed_env.py | 6 +- tests/test_commands.py | 73 ++++++++++++------- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/bonobo/examples/environment/env_vars/get_passed_env.py b/bonobo/examples/environment/env_vars/get_passed_env.py index 54a3280..6ef64d1 100644 --- a/bonobo/examples/environment/env_vars/get_passed_env.py +++ b/bonobo/examples/environment/env_vars/get_passed_env.py @@ -4,9 +4,9 @@ import bonobo def extract(): - env_test_user = os.getenv('ENV_TEST_USER') - env_test_number = os.getenv('ENV_TEST_NUMBER') - env_test_string = os.getenv('ENV_TEST_STRING') + env_test_user = os.getenv('ENV_TEST_USER', 'user') + env_test_number = os.getenv('ENV_TEST_NUMBER', 'number') + env_test_string = os.getenv('ENV_TEST_STRING', 'string') return env_test_user, env_test_number, env_test_string diff --git a/tests/test_commands.py b/tests/test_commands.py index 3f996cb..4508869 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -229,30 +229,53 @@ def test_run_file_with_default_env_file_and_env_file_and_env_vars(runner, capsys @all_runners -def test_run_file_with_env_vars(runner, capsys): - runner( - 'run', '--quiet', - get_examples_path('environment/env_vars/get_passed_env.py'), - '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', - '--env', "ENV_TEST_STRING='my_test_string'" - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == 'cwandrews' - assert out[1] == '123' - assert out[2] == 'my_test_string' +class TestEnvVars(object): + def test_run_file_with_env_var(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] != 'test_user' + assert out[1] == '123' + assert out[2] == 'string' + def test_run_file_with_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' -@all_runners -def test_run_module_with_env_vars(runner, capsys): - runner( - 'run', '--quiet', '-m', - 'bonobo.examples.environment.env_vars.get_passed_env', - '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', - '--env', "ENV_TEST_STRING='my_test_string'" - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == 'cwandrews' - assert out[1] == '123' - assert out[2] == 'my_test_string' + def test_run_module_with_env_var(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_module_with_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' From cb7a18f20f9ea4ea7e13413a8d8d03981d1f4ad7 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 15 Oct 2017 16:14:14 -0400 Subject: [PATCH 12/19] Added more tests and moved all env and env file testing to classes (it might make more sense to just move them to separate files?). --- bonobo/commands/run.py | 4 - .../environment/env_vars/get_passed_env.py | 3 +- tests/test_commands.py | 103 ++++++++++++++---- 3 files changed, 81 insertions(+), 29 deletions(-) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 7b23b9d..8c0186c 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -78,21 +78,17 @@ def read(filename, module, install=False, quiet=False, verbose=False, default_en raise RuntimeError('UNEXPECTED: argparse should not allow this.') env_dir = Path(filename).parent or Path(module).parent - if default_env_file: for f in default_env_file: env_file_path = env_dir.joinpath(f) load_dotenv(env_file_path) - if default_env: for e in default_env: set_env_var(e) - if env_file: for f in env_file: env_file_path = env_dir.joinpath(f) load_dotenv(env_file_path, override=True) - if env: for e in env: set_env_var(e, override=True) diff --git a/bonobo/examples/environment/env_vars/get_passed_env.py b/bonobo/examples/environment/env_vars/get_passed_env.py index 6ef64d1..f236ba7 100644 --- a/bonobo/examples/environment/env_vars/get_passed_env.py +++ b/bonobo/examples/environment/env_vars/get_passed_env.py @@ -7,7 +7,8 @@ def extract(): env_test_user = os.getenv('ENV_TEST_USER', 'user') env_test_number = os.getenv('ENV_TEST_NUMBER', 'number') env_test_string = os.getenv('ENV_TEST_STRING', 'string') - return env_test_user, env_test_number, env_test_string + env_user = os.getenv('USER') + return env_test_user, env_test_number, env_test_string, env_user def load(s: str): diff --git a/tests/test_commands.py b/tests/test_commands.py index 4508869..0e6205d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -200,32 +200,87 @@ class TestEnvFile(object): @all_runners -def test_run_file_with_default_env_file_and_env_file(runner, capsys): - runner( - 'run', '--quiet', - get_examples_path('environment/env_files/get_passed_env_file.py'), - '--default-env-file', '.env_one', '--env-file', '.env_two', - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == '321' - assert out[1] == 'not_sweet_password' - assert out[2] == 'abril' +class TestEnvFileCombinations(object): + def test_run_file_with_default_env_file_and_env_file(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--default-env-file', '.env_one', '--env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' + + def test_run_file_with_default_env_file_and_env_file_and_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--default-env-file', '.env_one', '--env-file', '.env_two', + '--env', 'TEST_USER_PASSWORD=SWEETpassWORD', '--env', + 'MY_SECRET=444', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '444' + assert out[1] == 'SWEETpassWORD' + assert out[2] == 'abril' @all_runners -def test_run_file_with_default_env_file_and_env_file_and_env_vars(runner, capsys): - runner( - 'run', '--quiet', - get_examples_path('environment/env_files/get_passed_env_file.py'), - '--default-env-file', '.env_one', '--env-file', '.env_two', - '--env', 'TEST_USER_PASSWORD=SWEETpassWORD', '--env', 'MY_SECRET=444', - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == '444' - assert out[1] == 'SWEETpassWORD' - assert out[2] == 'abril' +class TestDefaultEnvVars(object): + def test_run_file_with_default_env_var(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--default-env', 'USER=clowncity' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'user' + assert out[1] == 'number' + assert out[2] == 'string' + assert out[3] != 'clowncity' + + def test_run_file_with_default_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--default-env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_module_with_default_env_var(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', + '--default-env', 'ENV_TEST_STRING=string' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] != 'string' + + def test_run_module_with_default_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--default-env', "ENV_TEST_STRING='string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] != 'string' @all_runners @@ -240,7 +295,7 @@ class TestEnvVars(object): out = out.split('\n') assert out[0] != 'test_user' assert out[1] == '123' - assert out[2] == 'string' + assert out[2] == 'my_test_string' def test_run_file_with_env_vars(self, runner, capsys): runner( From d6d063ad43e499a69dcd94630c26f2cc54f02368 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 15 Oct 2017 16:43:28 -0400 Subject: [PATCH 13/19] Updated environment documentation in guides to account for env files. --- docs/guide/environment.rst | 67 +++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/docs/guide/environment.rst b/docs/guide/environment.rst index 203368d..ba21e62 100644 --- a/docs/guide/environment.rst +++ b/docs/guide/environment.rst @@ -23,25 +23,76 @@ simply to use the optional ``--env`` argument when running bonobo from the shell syntax ``VAR_NAME=VAR_VALUE``. Multiple environment variables can be passed by using multiple ``--env`` / ``-e`` flags (i.e. ``bonobo run --env FIZZ=buzz ...`` and ``bonobo run --env FIZZ=buzz --env Foo=bar ...``). Additionally, in bash you can also set environment variables by listing those you wish to set before the `bonobo run` command with space -separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``). +separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``). Additionally, +bonobo is able to pull environment variables from local '.env' files rather than having to pass each key-value pair +individually at runtime. Importantly, a strict 'order of priority' is followed when setting environment variables so +it is advisable to read and understand the order listed below to prevent + + +The order of priority is from lower to higher with the higher "winning" if set: + + 1. default values + ``os.getenv("VARNAME", default_value)`` + The user/writer/creator of the graph is responsible for setting these. + + 2. ``--default-env-file`` values + Specify file to read default env values from. Each env var in the file is used if the var isn't already a corresponding value set at the system environment (system environment vars not overwritten). + + 3. ``--default-env`` values + Works like #2 but the default ``NAME=var`` are passed individually, with one ``key=value`` pair for each ``--default-env`` flag rather than gathered from a specified file. + + 4. system environment values + Env vars already set at the system level. It is worth noting that passed env vars via ``NAME=value bonobo run ...`` falls here in the order of priority. + + 5. ``--env-file`` values + Env vars specified here are set like those in #2 albeit that these values have priority over those set at the system level. + + 6. ``--env`` values + Env vars set using the ``--env`` / ``-e`` flag work like #3 but take priority over all other env vars. + + + +Examples +:::::::: The Examples below demonstrate setting one or multiple variables using both of these methods: .. code-block:: bash - # Using one environment variable via --env flag: + # Using one environment variable via --env and --defualt-env flags: bonobo run csvsanitizer --env SECRET_TOKEN=secret123 + bonobo run csvsanitizer --defaul-env SECRET_TOKEN=secret123 - # Using multiple environment variables via -e (env) flag: + # Using multiple environment variables via -e (env) and --default-env flags: bonobo run csvsanitizer -e SRC_FILE=inventory.txt -e DST_FILE=inventory_processed.csv - - # Using one environment variable inline (bash only): + bonobo run csvsanitizer --default-env SRC_FILE=inventory.txt --default-env DST_FILE=inventory_processed.csv + + # Using one environment variable inline (bash-like shells only): SECRET_TOKEN=secret123 bonobo run csvsanitizer - # Using multiple environment variables inline (bash only): + # Using multiple environment variables inline (bash-like shells only): SRC_FILE=inventory.txt DST_FILE=inventory_processed.csv bonobo run csvsanitizer - -*Though not-yet implemented, the bonobo roadmap includes implementing environment / .env files as well.* + + # Using an env file for default env values: + bonobo run csvsanitizer --default-env-file .default_env + + # Using an env file for env values: + bonobo run csvsanitizer --env-file '.env' + + +ENV File Structure +:::::::::::::::::: + +The file structure for env files is incredibly simple. The only text in the file +should be `NAME=value` pairs with one pair per line like the below. + +.. code-block:: text + + # .env + + DB_USER='bonobo' + DB_PASS='cicero' + Accessing Environment Variables from within the Graph Context ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: From bad598a4d7ce3986f61b4db4d0f6e0572bf61ac6 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 15 Oct 2017 16:52:14 -0400 Subject: [PATCH 14/19] Cast env_dir to string before passing to load_dotenv as passing a PosixPath to load_dotenv raises an exception in 3.5. --- bonobo/commands/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 8c0186c..a2a4edc 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -80,14 +80,14 @@ def read(filename, module, install=False, quiet=False, verbose=False, default_en env_dir = Path(filename).parent or Path(module).parent if default_env_file: for f in default_env_file: - env_file_path = env_dir.joinpath(f) + env_file_path = str(env_dir.joinpath(f)) load_dotenv(env_file_path) if default_env: for e in default_env: set_env_var(e) if env_file: for f in env_file: - env_file_path = env_dir.joinpath(f) + env_file_path = str(env_dir.joinpath(f)) load_dotenv(env_file_path, override=True) if env: for e in env: From 945d8501391d69ddc2c18948b63aa28b2daf917c Mon Sep 17 00:00:00 2001 From: CW Andrews Date: Sun, 15 Oct 2017 17:11:10 -0400 Subject: [PATCH 15/19] Update environment.rst Fixed typo (I think). --- docs/guide/environment.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/environment.rst b/docs/guide/environment.rst index ba21e62..ce199a1 100644 --- a/docs/guide/environment.rst +++ b/docs/guide/environment.rst @@ -59,7 +59,7 @@ The Examples below demonstrate setting one or multiple variables using both of t .. code-block:: bash - # Using one environment variable via --env and --defualt-env flags: + # Using one environment variable via a --env or --defualt-env flag: bonobo run csvsanitizer --env SECRET_TOKEN=secret123 bonobo run csvsanitizer --defaul-env SECRET_TOKEN=secret123 From 5d41f6df2d074908b56608f7d932652f61b1f042 Mon Sep 17 00:00:00 2001 From: CW Andrews Date: Sun, 15 Oct 2017 17:15:30 -0400 Subject: [PATCH 16/19] Update environment.rst Updated examples to use preferred naming conventions for default and private/local .env files per request of @hartym. --- docs/guide/environment.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/environment.rst b/docs/guide/environment.rst index ce199a1..972f45d 100644 --- a/docs/guide/environment.rst +++ b/docs/guide/environment.rst @@ -74,10 +74,10 @@ The Examples below demonstrate setting one or multiple variables using both of t SRC_FILE=inventory.txt DST_FILE=inventory_processed.csv bonobo run csvsanitizer # Using an env file for default env values: - bonobo run csvsanitizer --default-env-file .default_env + bonobo run csvsanitizer --default-env-file .env # Using an env file for env values: - bonobo run csvsanitizer --env-file '.env' + bonobo run csvsanitizer --env-file '.env.private' ENV File Structure From d8c04138f6c3c7f9f0a1d38788e0ba4f90b128f1 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 15 Oct 2017 19:45:54 -0400 Subject: [PATCH 17/19] Updated Projectfile to include python-dotenv dependency. --- Projectfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Projectfile b/Projectfile index 0973c1f..73efcaa 100644 --- a/Projectfile +++ b/Projectfile @@ -45,6 +45,7 @@ python.add_requirements( 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0', + 'python-dotenv >=0.7.1,<1.0', dev=[ 'cookiecutter >=1.5,<1.6', 'pytest-sugar >=0.8,<0.9', From 3f3bda632cec45713f174c900db74b27b51a5a85 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 15 Oct 2017 19:50:27 -0400 Subject: [PATCH 18/19] Updated requirements files using edgy-project. --- Makefile | 6 +++--- requirements-dev.txt | 1 - requirements-docker.txt | 6 +++--- requirements-jupyter.txt | 3 +-- requirements.txt | 4 ++-- setup.py | 2 +- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 221b012..c2f6f48 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-10-05 18:56:33.985014 +# Updated at 2017-10-15 19:49:24.146904 PACKAGE ?= bonobo PYTHON ?= $(shell which python) @@ -27,13 +27,13 @@ VERSION ?= $(shell git describe 2>/dev/null || echo dev) # Installs the local project dependencies. install: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ + $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ fi # Installs the local project dependencies, including development-only libraries. install-dev: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ + $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ fi # Cleans up the local mess. diff --git a/requirements-dev.txt b/requirements-dev.txt index 2fa4ef8..4e005a7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,7 +23,6 @@ pytest-sugar==0.8.0 pytest-timeout==1.2.0 pytest==3.2.3 python-dateutil==2.6.1 -python-dotenv==0.7.1 pytz==2017.2 requests==2.18.4 six==1.11.0 diff --git a/requirements-docker.txt b/requirements-docker.txt index 976b56d..9e68208 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,16 +1,16 @@ -e .[docker] appdirs==1.4.3 -bonobo-docker==0.2.11 +bonobo-docker==0.2.12 certifi==2017.7.27.1 chardet==3.0.4 colorama==0.3.9 docker-pycreds==0.2.1 docker==2.3.0 -fs==2.0.11 +fs==2.0.12 idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.3.1 +psutil==5.4.0 pyparsing==2.2.0 pytz==2017.2 requests==2.18.4 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index e1b0ba7..e860433 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -1,5 +1,4 @@ -e .[jupyter] -appnope==0.1.0 bleach==2.1.1 decorator==4.1.2 entrypoints==0.2.3 @@ -19,7 +18,7 @@ markupsafe==1.0 mistune==0.7.4 nbconvert==5.3.1 nbformat==4.4.0 -notebook==5.1.0 +notebook==5.2.0 pandocfilters==1.4.2 parso==0.1.0 pexpect==4.2.1 diff --git a/requirements.txt b/requirements.txt index 9e604ed..7384e3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,11 @@ certifi==2017.7.27.1 chardet==3.0.4 click==6.7 colorama==0.3.9 -fs==2.0.11 +fs==2.0.12 idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.3.1 +psutil==5.4.0 pyparsing==2.2.0 python-dotenv==0.7.1 pytz==2017.2 diff --git a/setup.py b/setup.py index be97d0c..7ff3be3 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( include_package_data=True, install_requires=[ 'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'packaging (>= 16, < 17)', 'psutil (>= 5.2, < 6.0)', - 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)' + 'python-dotenv (>= 0.7.1, < 1.0)', 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)' ], extras_require={ 'dev': [ From b87f674eb251eb1446d270b42d412691826b5953 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Sun, 15 Oct 2017 19:55:37 -0400 Subject: [PATCH 19/19] Test tweak to work for Windows CI. --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 0e6205d..281a996 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -234,7 +234,7 @@ class TestDefaultEnvVars(object): runner( 'run', '--quiet', get_examples_path('environment/env_vars/get_passed_env.py'), - '--default-env', 'USER=clowncity' + '--default-env', 'USER=clowncity', '--env', 'USER=ted' ) out, err = capsys.readouterr() out = out.split('\n')