diff --git a/Makefile b/Makefile index 221b012..c2f6f48 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-10-05 18:56:33.985014 +# Updated at 2017-10-15 19:49:24.146904 PACKAGE ?= bonobo PYTHON ?= $(shell which python) @@ -27,13 +27,13 @@ VERSION ?= $(shell git describe 2>/dev/null || echo dev) # Installs the local project dependencies. install: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ + $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ fi # Installs the local project dependencies, including development-only libraries. install-dev: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ + $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ fi # Cleans up the local mess. diff --git a/Projectfile b/Projectfile index 0973c1f..73efcaa 100644 --- a/Projectfile +++ b/Projectfile @@ -45,6 +45,7 @@ python.add_requirements( 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0', + 'python-dotenv >=0.7.1,<1.0', dev=[ 'cookiecutter >=1.5,<1.6', 'pytest-sugar >=0.8,<0.9', diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index a37282c..a2a4edc 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -1,7 +1,10 @@ +import codecs import os +from pathlib import Path import bonobo from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME +from dotenv import load_dotenv DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', ) DEFAULT_GRAPH_ATTR = 'get_graph' @@ -40,8 +43,8 @@ def _install_requirements(requirements): importlib.reload(site) -def read(filename, module, install=False, quiet=False, verbose=False, env=None): - import re +def read(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None): + import runpy from bonobo import Graph, settings @@ -51,12 +54,6 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): if verbose: settings.DEBUG.set(True) - if env: - quote_killer = re.compile('["\']') - for e in env: - var_name, var_value = e.split('=') - os.environ[var_name] = quote_killer.sub('', var_value) - if filename: if os.path.isdir(filename): if install: @@ -80,6 +77,22 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): else: raise RuntimeError('UNEXPECTED: argparse should not allow this.') + env_dir = Path(filename).parent or Path(module).parent + if default_env_file: + for f in default_env_file: + env_file_path = str(env_dir.joinpath(f)) + load_dotenv(env_file_path) + if default_env: + for e in default_env: + set_env_var(e) + if env_file: + for f in env_file: + env_file_path = str(env_dir.joinpath(f)) + load_dotenv(env_file_path, override=True) + if env: + for e in env: + set_env_var(e, override=True) + graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph)) assert len(graphs) == 1, ( @@ -96,8 +109,25 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): return graph, plugins, services -def execute(filename, module, install=False, quiet=False, verbose=False, env=None): - graph, plugins, services = read(filename, module, install, quiet, verbose, env) +def set_env_var(e, override=False): + __escape_decoder = codecs.getdecoder('unicode_escape') + ename, evalue = e.split('=', 1) + + def decode_escaped(escaped): + return __escape_decoder(escaped)[0] + + if len(evalue) > 0: + if evalue[0] == evalue[len(evalue) - 1] in ['"', "'"]: + evalue = decode_escaped(evalue[1:-1]) + + if override: + os.environ[ename] = evalue + else: + os.environ.setdefault(ename, evalue) + + +def execute(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None): + graph, plugins, services = read(filename, module, install, quiet, verbose, default_env_file, default_env, env_file, env) return bonobo.run(graph, plugins=plugins, services=services) @@ -106,6 +136,9 @@ def register_generic_run_arguments(parser, required=True): source_group = parser.add_mutually_exclusive_group(required=required) source_group.add_argument('filename', nargs='?', type=str) source_group.add_argument('--module', '-m', type=str) + parser.add_argument('--default-env-file', action='append') + parser.add_argument('--default-env', action='append') + parser.add_argument('--env-file', action='append') parser.add_argument('--env', '-e', action='append') return parser diff --git a/bonobo/examples/env_vars/get_passed_env.py b/bonobo/examples/env_vars/get_passed_env.py deleted file mode 100644 index 54a3280..0000000 --- a/bonobo/examples/env_vars/get_passed_env.py +++ /dev/null @@ -1,20 +0,0 @@ -import os - -import bonobo - - -def extract(): - env_test_user = os.getenv('ENV_TEST_USER') - env_test_number = os.getenv('ENV_TEST_NUMBER') - env_test_string = os.getenv('ENV_TEST_STRING') - return env_test_user, env_test_number, env_test_string - - -def load(s: str): - print(s) - - -graph = bonobo.Graph(extract, load) - -if __name__ == '__main__': - bonobo.run(graph) diff --git a/bonobo/examples/environment/env_files/.env_one b/bonobo/examples/environment/env_files/.env_one new file mode 100644 index 0000000..65f2b17 --- /dev/null +++ b/bonobo/examples/environment/env_files/.env_one @@ -0,0 +1,3 @@ +MY_SECRET=321 +TEST_USER_PASSWORD=sweetpassword +PATH=marzo \ No newline at end of file diff --git a/bonobo/examples/environment/env_files/.env_two b/bonobo/examples/environment/env_files/.env_two new file mode 100644 index 0000000..672d6d2 --- /dev/null +++ b/bonobo/examples/environment/env_files/.env_two @@ -0,0 +1,2 @@ +TEST_USER_PASSWORD=not_sweet_password +PATH='abril' \ No newline at end of file diff --git a/bonobo/examples/env_vars/__init__.py b/bonobo/examples/environment/env_files/__init__.py similarity index 100% rename from bonobo/examples/env_vars/__init__.py rename to bonobo/examples/environment/env_files/__init__.py diff --git a/bonobo/examples/environment/env_files/get_passed_env_file.py b/bonobo/examples/environment/env_files/get_passed_env_file.py new file mode 100644 index 0000000..bb83e67 --- /dev/null +++ b/bonobo/examples/environment/env_files/get_passed_env_file.py @@ -0,0 +1,21 @@ +import os + +import bonobo + + +def extract(): + my_secret = os.getenv('MY_SECRET') + test_user_password = os.getenv('TEST_USER_PASSWORD') + path = os.getenv('PATH') + + return my_secret, test_user_password, path + + +def load(s: str): + print(s) + + +graph = bonobo.Graph(extract, load) + +if __name__ == '__main__': + bonobo.run(graph) diff --git a/bonobo/examples/environment/env_vars/__init__.py b/bonobo/examples/environment/env_vars/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bonobo/examples/environment/env_vars/get_passed_env.py b/bonobo/examples/environment/env_vars/get_passed_env.py new file mode 100644 index 0000000..f236ba7 --- /dev/null +++ b/bonobo/examples/environment/env_vars/get_passed_env.py @@ -0,0 +1,21 @@ +import os + +import bonobo + + +def extract(): + env_test_user = os.getenv('ENV_TEST_USER', 'user') + env_test_number = os.getenv('ENV_TEST_NUMBER', 'number') + env_test_string = os.getenv('ENV_TEST_STRING', 'string') + env_user = os.getenv('USER') + return env_test_user, env_test_number, env_test_string, env_user + + +def load(s: str): + print(s) + + +graph = bonobo.Graph(extract, load) + +if __name__ == '__main__': + bonobo.run(graph) diff --git a/docs/guide/environment.rst b/docs/guide/environment.rst index 203368d..972f45d 100644 --- a/docs/guide/environment.rst +++ b/docs/guide/environment.rst @@ -23,25 +23,76 @@ simply to use the optional ``--env`` argument when running bonobo from the shell syntax ``VAR_NAME=VAR_VALUE``. Multiple environment variables can be passed by using multiple ``--env`` / ``-e`` flags (i.e. ``bonobo run --env FIZZ=buzz ...`` and ``bonobo run --env FIZZ=buzz --env Foo=bar ...``). Additionally, in bash you can also set environment variables by listing those you wish to set before the `bonobo run` command with space -separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``). +separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``). Additionally, +bonobo is able to pull environment variables from local '.env' files rather than having to pass each key-value pair +individually at runtime. Importantly, a strict 'order of priority' is followed when setting environment variables so +it is advisable to read and understand the order listed below to prevent + + +The order of priority is from lower to higher with the higher "winning" if set: + + 1. default values + ``os.getenv("VARNAME", default_value)`` + The user/writer/creator of the graph is responsible for setting these. + + 2. ``--default-env-file`` values + Specify file to read default env values from. Each env var in the file is used if the var isn't already a corresponding value set at the system environment (system environment vars not overwritten). + + 3. ``--default-env`` values + Works like #2 but the default ``NAME=var`` are passed individually, with one ``key=value`` pair for each ``--default-env`` flag rather than gathered from a specified file. + + 4. system environment values + Env vars already set at the system level. It is worth noting that passed env vars via ``NAME=value bonobo run ...`` falls here in the order of priority. + + 5. ``--env-file`` values + Env vars specified here are set like those in #2 albeit that these values have priority over those set at the system level. + + 6. ``--env`` values + Env vars set using the ``--env`` / ``-e`` flag work like #3 but take priority over all other env vars. + + + +Examples +:::::::: The Examples below demonstrate setting one or multiple variables using both of these methods: .. code-block:: bash - # Using one environment variable via --env flag: + # Using one environment variable via a --env or --defualt-env flag: bonobo run csvsanitizer --env SECRET_TOKEN=secret123 + bonobo run csvsanitizer --defaul-env SECRET_TOKEN=secret123 - # Using multiple environment variables via -e (env) flag: + # Using multiple environment variables via -e (env) and --default-env flags: bonobo run csvsanitizer -e SRC_FILE=inventory.txt -e DST_FILE=inventory_processed.csv - - # Using one environment variable inline (bash only): + bonobo run csvsanitizer --default-env SRC_FILE=inventory.txt --default-env DST_FILE=inventory_processed.csv + + # Using one environment variable inline (bash-like shells only): SECRET_TOKEN=secret123 bonobo run csvsanitizer - # Using multiple environment variables inline (bash only): + # Using multiple environment variables inline (bash-like shells only): SRC_FILE=inventory.txt DST_FILE=inventory_processed.csv bonobo run csvsanitizer - -*Though not-yet implemented, the bonobo roadmap includes implementing environment / .env files as well.* + + # Using an env file for default env values: + bonobo run csvsanitizer --default-env-file .env + + # Using an env file for env values: + bonobo run csvsanitizer --env-file '.env.private' + + +ENV File Structure +:::::::::::::::::: + +The file structure for env files is incredibly simple. The only text in the file +should be `NAME=value` pairs with one pair per line like the below. + +.. code-block:: text + + # .env + + DB_USER='bonobo' + DB_PASS='cicero' + Accessing Environment Variables from within the Graph Context ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/requirements-docker.txt b/requirements-docker.txt index 976b56d..9e68208 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,16 +1,16 @@ -e .[docker] appdirs==1.4.3 -bonobo-docker==0.2.11 +bonobo-docker==0.2.12 certifi==2017.7.27.1 chardet==3.0.4 colorama==0.3.9 docker-pycreds==0.2.1 docker==2.3.0 -fs==2.0.11 +fs==2.0.12 idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.3.1 +psutil==5.4.0 pyparsing==2.2.0 pytz==2017.2 requests==2.18.4 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index e1b0ba7..e860433 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -1,5 +1,4 @@ -e .[jupyter] -appnope==0.1.0 bleach==2.1.1 decorator==4.1.2 entrypoints==0.2.3 @@ -19,7 +18,7 @@ markupsafe==1.0 mistune==0.7.4 nbconvert==5.3.1 nbformat==4.4.0 -notebook==5.1.0 +notebook==5.2.0 pandocfilters==1.4.2 parso==0.1.0 pexpect==4.2.1 diff --git a/requirements.txt b/requirements.txt index d6439df..7384e3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,15 @@ appdirs==1.4.3 certifi==2017.7.27.1 chardet==3.0.4 +click==6.7 colorama==0.3.9 -fs==2.0.11 +fs==2.0.12 idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.3.1 +psutil==5.4.0 pyparsing==2.2.0 +python-dotenv==0.7.1 pytz==2017.2 requests==2.18.4 six==1.11.0 diff --git a/setup.py b/setup.py index be97d0c..7ff3be3 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( include_package_data=True, install_requires=[ 'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'packaging (>= 16, < 17)', 'psutil (>= 5.2, < 6.0)', - 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)' + 'python-dotenv (>= 0.7.1, < 1.0)', 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)' ], extras_require={ 'dev': [ diff --git a/tests/test_commands.py b/tests/test_commands.py index a29465c..281a996 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -99,27 +99,238 @@ def test_version(runner, capsys): @all_runners -def test_run_with_env(runner, capsys): - runner( - 'run', '--quiet', - get_examples_path('env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env', - 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == 'cwandrews' - assert out[1] == '123' - assert out[2] == 'my_test_string' +class TestDefaultEnvFile(object): + def test_run_file_with_default_env_file(self, runner, capsys): + runner( + 'run', '--quiet', '--default-env-file', '.env_one', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' + + def test_run_file_with_multiple_default_env_files(self, runner, capsys): + runner( + 'run', '--quiet', '--default-env-file', '.env_one', + '--default-env-file', '.env_two', + get_examples_path('environment/env_files/get_passed_env_file.py') + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' + + def test_run_module_with_default_env_file(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--default-env-file', '.env_one' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' + + def test_run_module_with_multiple_default_env_files(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--default-env-file', '.env_one', '--default-env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] != 'marzo' @all_runners -def test_run_module_with_env(runner, capsys): - runner( - 'run', '--quiet', '-m', 'bonobo.examples.env_vars.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env', - 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" - ) - out, err = capsys.readouterr() - out = out.split('\n') - assert out[0] == 'cwandrews' - assert out[1] == '123' - assert out[2] == 'my_test_string' +class TestEnvFile(object): + def test_run_file_with_file(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--env-file', '.env_one', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] == 'marzo' + + def test_run_file_with_multiple_files(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--env-file', '.env_one', '--env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' + + def test_run_module_with_file(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--env-file', '.env_one', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'sweetpassword' + assert out[2] == 'marzo' + + def test_run_module_with_multiple_files(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_files.get_passed_env_file', + '--env-file', '.env_one', '--env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' + + +@all_runners +class TestEnvFileCombinations(object): + def test_run_file_with_default_env_file_and_env_file(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--default-env-file', '.env_one', '--env-file', '.env_two', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '321' + assert out[1] == 'not_sweet_password' + assert out[2] == 'abril' + + def test_run_file_with_default_env_file_and_env_file_and_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_files/get_passed_env_file.py'), + '--default-env-file', '.env_one', '--env-file', '.env_two', + '--env', 'TEST_USER_PASSWORD=SWEETpassWORD', '--env', + 'MY_SECRET=444', + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == '444' + assert out[1] == 'SWEETpassWORD' + assert out[2] == 'abril' + + +@all_runners +class TestDefaultEnvVars(object): + def test_run_file_with_default_env_var(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--default-env', 'USER=clowncity', '--env', 'USER=ted' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'user' + assert out[1] == 'number' + assert out[2] == 'string' + assert out[3] != 'clowncity' + + def test_run_file_with_default_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--default-env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_module_with_default_env_var(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', + '--default-env', 'ENV_TEST_STRING=string' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] != 'string' + + def test_run_module_with_default_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--default-env', "ENV_TEST_STRING='string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] != 'string' + + +@all_runners +class TestEnvVars(object): + def test_run_file_with_env_var(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] != 'test_user' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_file_with_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', + get_examples_path('environment/env_vars/get_passed_env.py'), + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_module_with_env_var(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123' + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + def test_run_module_with_env_vars(self, runner, capsys): + runner( + 'run', '--quiet', '-m', + 'bonobo.examples.environment.env_vars.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string'