diff --git a/Projectfile b/Projectfile index c812fc1..c1abd79 100644 --- a/Projectfile +++ b/Projectfile @@ -45,6 +45,7 @@ python.add_requirements( 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0', + 'python-dotenv >=0.7.1,<1.0', dev=[ 'cookiecutter >=1.5,<1.6', 'pytest-sugar >=0.8,<0.9', diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 0a11577..ad19230 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -1,7 +1,10 @@ +import codecs import os +from pathlib import Path import bonobo from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME +from dotenv import load_dotenv DEFAULT_GRAPH_FILENAMES = ( '__main__.py', @@ -43,8 +46,8 @@ def _install_requirements(requirements): importlib.reload(site) -def read(filename, module, install=False, quiet=False, verbose=False, env=None): - import re +def read(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None): + import runpy from bonobo import Graph, settings @@ -54,12 +57,6 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): if verbose: settings.DEBUG.set(True) - if env: - quote_killer = re.compile('["\']') - for e in env: - var_name, var_value = e.split('=') - os.environ[var_name] = quote_killer.sub('', var_value) - if filename: if os.path.isdir(filename): if install: @@ -83,6 +80,22 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): else: raise RuntimeError('UNEXPECTED: argparse should not allow this.') + env_dir = Path(filename).parent or Path(module).parent + if default_env_file: + for f in default_env_file: + env_file_path = str(env_dir.joinpath(f)) + load_dotenv(env_file_path) + if default_env: + for e in default_env: + set_env_var(e) + if env_file: + for f in env_file: + env_file_path = str(env_dir.joinpath(f)) + load_dotenv(env_file_path, override=True) + if env: + for e in env: + set_env_var(e, override=True) + graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph)) assert len(graphs) == 1, ( @@ -99,8 +112,25 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): return graph, plugins, services -def execute(filename, module, install=False, quiet=False, verbose=False, env=None): - graph, plugins, services = read(filename, module, install, quiet, verbose, env) +def set_env_var(e, override=False): + __escape_decoder = codecs.getdecoder('unicode_escape') + ename, evalue = e.split('=', 1) + + def decode_escaped(escaped): + return __escape_decoder(escaped)[0] + + if len(evalue) > 0: + if evalue[0] == evalue[len(evalue) - 1] in ['"', "'"]: + evalue = decode_escaped(evalue[1:-1]) + + if override: + os.environ[ename] = evalue + else: + os.environ.setdefault(ename, evalue) + + +def execute(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None): + graph, plugins, services = read(filename, module, install, quiet, verbose, default_env_file, default_env, env_file, env) return bonobo.run(graph, plugins=plugins, services=services) @@ -109,6 +139,9 @@ def register_generic_run_arguments(parser, required=True): source_group = parser.add_mutually_exclusive_group(required=required) source_group.add_argument('filename', nargs='?', type=str) source_group.add_argument('--module', '-m', type=str) + parser.add_argument('--default-env-file', action='append') + parser.add_argument('--default-env', action='append') + parser.add_argument('--env-file', action='append') parser.add_argument('--env', '-e', action='append') return parser diff --git a/bonobo/examples/env_vars/get_passed_env.py b/bonobo/examples/env_vars/get_passed_env.py deleted file mode 100644 index 54a3280..0000000 --- a/bonobo/examples/env_vars/get_passed_env.py +++ /dev/null @@ -1,20 +0,0 @@ -import os - -import bonobo - - -def extract(): - env_test_user = os.getenv('ENV_TEST_USER') - env_test_number = os.getenv('ENV_TEST_NUMBER') - env_test_string = os.getenv('ENV_TEST_STRING') - return env_test_user, env_test_number, env_test_string - - -def load(s: str): - print(s) - - -graph = bonobo.Graph(extract, load) - -if __name__ == '__main__': - bonobo.run(graph) diff --git a/bonobo/examples/environment/env_files/.env_one b/bonobo/examples/environment/env_files/.env_one new file mode 100644 index 0000000..65f2b17 --- /dev/null +++ b/bonobo/examples/environment/env_files/.env_one @@ -0,0 +1,3 @@ +MY_SECRET=321 +TEST_USER_PASSWORD=sweetpassword +PATH=marzo \ No newline at end of file diff --git a/bonobo/examples/environment/env_files/.env_two b/bonobo/examples/environment/env_files/.env_two new file mode 100644 index 0000000..672d6d2 --- /dev/null +++ b/bonobo/examples/environment/env_files/.env_two @@ -0,0 +1,2 @@ +TEST_USER_PASSWORD=not_sweet_password +PATH='abril' \ No newline at end of file diff --git a/bonobo/examples/env_vars/__init__.py b/bonobo/examples/environment/env_files/__init__.py similarity index 100% rename from bonobo/examples/env_vars/__init__.py rename to bonobo/examples/environment/env_files/__init__.py diff --git a/bonobo/examples/environment/env_files/get_passed_env_file.py b/bonobo/examples/environment/env_files/get_passed_env_file.py new file mode 100644 index 0000000..bb83e67 --- /dev/null +++ b/bonobo/examples/environment/env_files/get_passed_env_file.py @@ -0,0 +1,21 @@ +import os + +import bonobo + + +def extract(): + my_secret = os.getenv('MY_SECRET') + test_user_password = os.getenv('TEST_USER_PASSWORD') + path = os.getenv('PATH') + + return my_secret, test_user_password, path + + +def load(s: str): + print(s) + + +graph = bonobo.Graph(extract, load) + +if __name__ == '__main__': + bonobo.run(graph) diff --git a/bonobo/examples/environment/env_vars/__init__.py b/bonobo/examples/environment/env_vars/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bonobo/examples/environment/env_vars/get_passed_env.py b/bonobo/examples/environment/env_vars/get_passed_env.py new file mode 100644 index 0000000..f236ba7 --- /dev/null +++ b/bonobo/examples/environment/env_vars/get_passed_env.py @@ -0,0 +1,21 @@ +import os + +import bonobo + + +def extract(): + env_test_user = os.getenv('ENV_TEST_USER', 'user') + env_test_number = os.getenv('ENV_TEST_NUMBER', 'number') + env_test_string = os.getenv('ENV_TEST_STRING', 'string') + env_user = os.getenv('USER') + return env_test_user, env_test_number, env_test_string, env_user + + +def load(s: str): + print(s) + + +graph = bonobo.Graph(extract, load) + +if __name__ == '__main__': + bonobo.run(graph) diff --git a/docs/guide/environment.rst b/docs/guide/environment.rst index 203368d..972f45d 100644 --- a/docs/guide/environment.rst +++ b/docs/guide/environment.rst @@ -23,25 +23,76 @@ simply to use the optional ``--env`` argument when running bonobo from the shell syntax ``VAR_NAME=VAR_VALUE``. Multiple environment variables can be passed by using multiple ``--env`` / ``-e`` flags (i.e. ``bonobo run --env FIZZ=buzz ...`` and ``bonobo run --env FIZZ=buzz --env Foo=bar ...``). Additionally, in bash you can also set environment variables by listing those you wish to set before the `bonobo run` command with space -separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``). +separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``). Additionally, +bonobo is able to pull environment variables from local '.env' files rather than having to pass each key-value pair +individually at runtime. Importantly, a strict 'order of priority' is followed when setting environment variables so +it is advisable to read and understand the order listed below to prevent + + +The order of priority is from lower to higher with the higher "winning" if set: + + 1. default values + ``os.getenv("VARNAME", default_value)`` + The user/writer/creator of the graph is responsible for setting these. + + 2. ``--default-env-file`` values + Specify file to read default env values from. Each env var in the file is used if the var isn't already a corresponding value set at the system environment (system environment vars not overwritten). + + 3. ``--default-env`` values + Works like #2 but the default ``NAME=var`` are passed individually, with one ``key=value`` pair for each ``--default-env`` flag rather than gathered from a specified file. + + 4. system environment values + Env vars already set at the system level. It is worth noting that passed env vars via ``NAME=value bonobo run ...`` falls here in the order of priority. + + 5. ``--env-file`` values + Env vars specified here are set like those in #2 albeit that these values have priority over those set at the system level. + + 6. ``--env`` values + Env vars set using the ``--env`` / ``-e`` flag work like #3 but take priority over all other env vars. + + + +Examples +:::::::: The Examples below demonstrate setting one or multiple variables using both of these methods: .. code-block:: bash - # Using one environment variable via --env flag: + # Using one environment variable via a --env or --defualt-env flag: bonobo run csvsanitizer --env SECRET_TOKEN=secret123 + bonobo run csvsanitizer --defaul-env SECRET_TOKEN=secret123 - # Using multiple environment variables via -e (env) flag: + # Using multiple environment variables via -e (env) and --default-env flags: bonobo run csvsanitizer -e SRC_FILE=inventory.txt -e DST_FILE=inventory_processed.csv - - # Using one environment variable inline (bash only): + bonobo run csvsanitizer --default-env SRC_FILE=inventory.txt --default-env DST_FILE=inventory_processed.csv + + # Using one environment variable inline (bash-like shells only): SECRET_TOKEN=secret123 bonobo run csvsanitizer - # Using multiple environment variables inline (bash only): + # Using multiple environment variables inline (bash-like shells only): SRC_FILE=inventory.txt DST_FILE=inventory_processed.csv bonobo run csvsanitizer - -*Though not-yet implemented, the bonobo roadmap includes implementing environment / .env files as well.* + + # Using an env file for default env values: + bonobo run csvsanitizer --default-env-file .env + + # Using an env file for env values: + bonobo run csvsanitizer --env-file '.env.private' + + +ENV File Structure +:::::::::::::::::: + +The file structure for env files is incredibly simple. The only text in the file +should be `NAME=value` pairs with one pair per line like the below. + +.. code-block:: text + + # .env + + DB_USER='bonobo' + DB_PASS='cicero' + Accessing Environment Variables from within the Graph Context ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/requirements.txt b/requirements.txt index 13d5113..7384e3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ appdirs==1.4.3 certifi==2017.7.27.1 chardet==3.0.4 +click==6.7 colorama==0.3.9 fs==2.0.12 idna==2.6 @@ -9,6 +10,7 @@ packaging==16.8 pbr==3.1.1 psutil==5.4.0 pyparsing==2.2.0 +python-dotenv==0.7.1 pytz==2017.2 requests==2.18.4 six==1.11.0 diff --git a/setup.py b/setup.py index 7b513dc..8278209 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( include_package_data=True, install_requires=[ 'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'packaging (>= 16, < 17)', 'psutil (>= 5.2, < 6.0)', - 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)' + 'python-dotenv (>= 0.7.1, < 1.0)', 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)' ], extras_require={ 'dev': [ diff --git a/tests/config/test_configurables.py b/tests/config/test_configurables.py index e1555d7..bee501e 100644 --- a/tests/config/test_configurables.py +++ b/tests/config/test_configurables.py @@ -67,7 +67,7 @@ def test_defaults(): assert o.required_str == 'hello' assert o.default_str == 'foo' - assert o.integer == None + assert o.integer is None def test_str_type_factory(): @@ -78,7 +78,7 @@ def test_str_type_factory(): assert o.required_str == '42' assert o.default_str == 'foo' - assert o.integer == None + assert o.integer is None def test_int_type_factory(): @@ -100,8 +100,8 @@ def test_bool_type_factory(): assert o.required_str == 'yes' assert o.default_str == 'foo' - assert o.integer == None - assert o.also_required == True + assert o.integer is None + assert o.also_required is True def test_option_resolution_order(): @@ -112,7 +112,7 @@ def test_option_resolution_order(): assert o.required_str == 'kaboom' assert o.default_str == 'foo' - assert o.integer == None + assert o.integer is None def test_option_positional(): diff --git a/tests/test_commands.py b/tests/test_commands.py index 59cf5f4..1b09a45 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -103,27 +103,238 @@ def test_version(runner, capsys): @all_runners -def test_run_with_env(runner, capsys): - runner( - 'run', '--quiet', - get_examples_path('env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env', - 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == 'cwandrews' - assert out[1] == '123' - assert out[2] == 'my_test_string' +class TestDefaultEnvFile(object): + def test_run_file_with_default_env_file(self, runner, capsys): + runner( + 'run', '--quiet', '--default-env-file', '.env_one', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' + + def test_run_file_with_multiple_default_env_files(self, runner, capsys): + runner( + 'run', '--quiet', '--default-env-file', '.env_one', + '--default-env-file', '.env_two', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' + + def test_run_module_with_default_env_file(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--default-env-file', '.env_one' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' + + def test_run_module_with_multiple_default_env_files(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--default-env-file', '.env_one', '--default-env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' @all_runners -def test_run_module_with_env(runner, capsys): - runner( - 'run', '--quiet', '-m', 'bonobo.examples.env_vars.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env', - 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == 'cwandrews' - assert out[1] == '123' - assert out[2] == 'my_test_string' +class TestEnvFile(object): + def test_run_file_with_file(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--env-file', '.env_one', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] == 'marzo' + + def test_run_file_with_multiple_files(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--env-file', '.env_one', '--env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' + + def test_run_module_with_file(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--env-file', '.env_one', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] == 'marzo' + + def test_run_module_with_multiple_files(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--env-file', '.env_one', '--env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' + + +@all_runners +class TestEnvFileCombinations(object): + def test_run_file_with_default_env_file_and_env_file(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--default-env-file', '.env_one', '--env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' + + def test_run_file_with_default_env_file_and_env_file_and_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--default-env-file', '.env_one', '--env-file', '.env_two', + '--env', 'TEST_USER_PASSWORD=SWEETpassWORD', '--env', + 'MY_SECRET=444', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '444' + assert out[1] == 'SWEETpassWORD' + assert out[2] == 'abril' + + +@all_runners +class TestDefaultEnvVars(object): + def test_run_file_with_default_env_var(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--default-env', 'USER=clowncity', '--env', 'USER=ted' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'user' + assert out[1] == 'number' + assert out[2] == 'string' + assert out[3] != 'clowncity' + + def test_run_file_with_default_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--default-env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_module_with_default_env_var(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', + '--default-env', 'ENV_TEST_STRING=string' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] != 'string' + + def test_run_module_with_default_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--default-env', "ENV_TEST_STRING='string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] != 'string' + + +@all_runners +class TestEnvVars(object): + def test_run_file_with_env_var(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] != 'test_user' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_file_with_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_module_with_env_var(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_module_with_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' diff --git a/tests/test_settings.py b/tests/test_settings.py index c8313c5..5771dc4 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -42,9 +42,9 @@ def test_setting(): def test_default_settings(): settings.clear_all() - assert settings.DEBUG.get() == False - assert settings.PROFILE.get() == False - assert settings.QUIET.get() == False + assert settings.DEBUG.get() is False + assert settings.PROFILE.get() is False + assert settings.QUIET.get() is False assert settings.LOGGING_LEVEL.get() == logging._checkLevel('INFO') with patch.dict(environ, {'DEBUG': 't'}): diff --git a/tests/util/test_iterators.py b/tests/util/test_iterators.py index 22b34d7..3d0249e 100644 --- a/tests/util/test_iterators.py +++ b/tests/util/test_iterators.py @@ -18,5 +18,5 @@ def test_force_iterator_with_generator(): yield 'ccc' iterator = force_iterator(generator()) - assert type(iterator) == types.GeneratorType + assert isinstance(iterator, types.GeneratorType) assert list(iterator) == ['aaa', 'bbb', 'ccc']