From 8b9dac50eca8d2737e377c225fdd5e4c4dc948c8 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Mon, 18 Sep 2017 15:24:27 -0400 Subject: [PATCH 1/2] Added optional passing of one or multiple environment variables via --env flag to the bonobo cli. --- bonobo/commands/run.py | 12 +++++++++++- tests/test_commands.py | 27 +++++++++++++++++++++++++++ tests/util/get_passed_env.py | 22 ++++++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tests/util/get_passed_env.py diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index fb93e77..c9bb39e 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -40,7 +40,10 @@ def _install_requirements(requirements): importlib.reload(site) -def execute(filename, module, install=False, quiet=False, verbose=False): +def execute(filename, module, install=False, quiet=False, verbose=False, + env=None): + import re + import runpy from bonobo import Graph, run, settings @@ -50,6 +53,12 @@ def execute(filename, module, install=False, quiet=False, verbose=False): if verbose: settings.DEBUG.set(True) + if env: + quote_killer = re.compile('["\']') + for e in env: + var_name, var_value = e.split('=') + os.environ[var_name] = quote_killer.sub('', var_value) + if filename: if os.path.isdir(filename): if install: @@ -106,4 +115,5 @@ def register(parser): verbosity_group.add_argument('--quiet', '-q', action='store_true') verbosity_group.add_argument('--verbose', '-v', action='store_true') parser.add_argument('--install', '-I', action='store_true') + parser.add_argument('--env', '-e', action='append') return execute diff --git a/tests/test_commands.py b/tests/test_commands.py index daf245f..cff9e38 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,6 +3,7 @@ import runpy import sys from unittest.mock import patch +import pathlib import pkg_resources import pytest @@ -96,3 +97,29 @@ def test_version(runner, capsys): out = out.strip() assert out.startswith('bonobo ') assert __version__ in out + + +@all_runners +def test_run_with_env(runner, capsys): + runner('run', '--quiet', + str(pathlib.Path(os.path.dirname(__file__), + 'util', 'get_passed_env.py')), + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'") + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + +@all_runners +def test_run_module_with_env(runner, capsys): + runner('run', '--quiet', '-m', 'tests.util.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'") + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' diff --git a/tests/util/get_passed_env.py b/tests/util/get_passed_env.py new file mode 100644 index 0000000..d9c4ba6 --- /dev/null +++ b/tests/util/get_passed_env.py @@ -0,0 +1,22 @@ +import os + +from bonobo import Graph + + +def extract(): + env_test_user = os.getenv('ENV_TEST_USER') + env_test_number = os.getenv('ENV_TEST_NUMBER') + env_test_string = os.getenv('ENV_TEST_STRING') + return env_test_user, env_test_number, env_test_string + + +def load(s: str): + print(s) + + +graph = Graph(extract, load) + +if __name__ == '__main__': + from bonobo import run + + run(graph) From 21514ad670f99da42b2967b89a6ece3d2e7e5ba1 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 30 Sep 2017 09:54:30 +0200 Subject: [PATCH 2/2] Update dependencies. --- Makefile | 6 +++--- requirements-dev.txt | 20 ++++++++++---------- requirements-docker.txt | 16 ++++++++-------- requirements-jupyter.txt | 31 ++++++++++++++++--------------- requirements.txt | 16 ++++++++-------- 5 files changed, 45 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index cb5f8bc..8175b3e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-07-16 10:52:31.093416 +# Updated at 2017-09-30 09:50:47.806007 PACKAGE ?= bonobo PYTHON ?= $(shell which python) @@ -27,13 +27,13 @@ VERSION ?= $(shell git describe 2>/dev/null || echo dev) # Installs the local project dependencies. install: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ fi # Installs the local project dependencies, including development-only libraries. install-dev: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ fi # Cleans up the local mess. diff --git a/requirements-dev.txt b/requirements-dev.txt index 69d64d8..92123d5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,16 +1,16 @@ -e .[dev] alabaster==0.7.10 arrow==0.10.0 -babel==2.4.0 -binaryornot==0.4.3 -certifi==2017.4.17 +babel==2.5.1 +binaryornot==0.4.4 +certifi==2017.7.27.1 chardet==3.0.4 click==6.7 cookiecutter==1.5.1 coverage==4.4.1 -docutils==0.13.1 +docutils==0.14 future==0.16.0 -idna==2.5 +idna==2.6 imagesize==0.7.1 jinja2-time==0.2.0 jinja2==2.9.6 @@ -21,14 +21,14 @@ pygments==2.2.0 pytest-cov==2.5.1 pytest-sugar==0.8.0 pytest-timeout==1.2.0 -pytest==3.1.3 +pytest==3.2.2 python-dateutil==2.6.1 pytz==2017.2 -requests==2.18.1 -six==1.10.0 +requests==2.18.4 +six==1.11.0 snowballstemmer==1.2.1 -sphinx==1.6.3 +sphinx==1.6.4 sphinxcontrib-websupport==1.0.1 termcolor==1.1.0 -urllib3==1.21.1 +urllib3==1.22 whichcraft==0.4.1 diff --git a/requirements-docker.txt b/requirements-docker.txt index f5e74fc..870223d 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,20 +1,20 @@ -e .[docker] appdirs==1.4.3 bonobo-docker==0.2.11 -certifi==2017.4.17 +certifi==2017.7.27.1 chardet==3.0.4 colorama==0.3.9 docker-pycreds==0.2.1 docker==2.3.0 -fs==2.0.4 -idna==2.5 +fs==2.0.11 +idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.2.2 +psutil==5.3.1 pyparsing==2.2.0 pytz==2017.2 -requests==2.18.1 -six==1.10.0 -stevedore==1.24.0 -urllib3==1.21.1 +requests==2.18.4 +six==1.11.0 +stevedore==1.27.0 +urllib3==1.22 websocket-client==0.44.0 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 2542040..94b10ea 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -1,40 +1,41 @@ -e .[jupyter] appnope==0.1.0 -bleach==2.0.0 -decorator==4.1.1 +bleach==2.1 +decorator==4.1.2 entrypoints==0.2.3 html5lib==0.999999999 ipykernel==4.6.1 ipython-genutils==0.2.0 -ipython==6.1.0 -ipywidgets==6.0.0 -jedi==0.10.2 +ipython==6.2.1 +ipywidgets==6.0.1 +jedi==0.11.0 jinja2==2.9.6 jsonschema==2.6.0 jupyter-client==5.1.0 -jupyter-console==5.1.0 +jupyter-console==5.2.0 jupyter-core==4.3.0 jupyter==1.0.0 markupsafe==1.0 mistune==0.7.4 -nbconvert==5.2.1 -nbformat==4.3.0 -notebook==5.0.0 -pandocfilters==1.4.1 +nbconvert==5.3.1 +nbformat==4.4.0 +notebook==5.1.0 +pandocfilters==1.4.2 +parso==0.1.0 pexpect==4.2.1 pickleshare==0.7.4 -prompt-toolkit==1.0.14 +prompt-toolkit==1.0.15 ptyprocess==0.5.2 pygments==2.2.0 python-dateutil==2.6.1 pyzmq==16.0.2 -qtconsole==4.3.0 +qtconsole==4.3.1 simplegeneric==0.8.1 -six==1.10.0 +six==1.11.0 terminado==0.6 testpath==0.3.1 -tornado==4.5.1 +tornado==4.5.2 traitlets==4.3.2 wcwidth==0.1.7 webencodings==0.5.1 -widgetsnbextension==2.0.0 +widgetsnbextension==2.0.1 diff --git a/requirements.txt b/requirements.txt index 5ddbb01..61ab27d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -e . appdirs==1.4.3 -certifi==2017.4.17 +certifi==2017.7.27.1 chardet==3.0.4 colorama==0.3.9 -fs==2.0.4 -idna==2.5 +fs==2.0.11 +idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.2.2 +psutil==5.3.1 pyparsing==2.2.0 pytz==2017.2 -requests==2.18.1 -six==1.10.0 -stevedore==1.24.0 -urllib3==1.21.1 +requests==2.18.4 +six==1.11.0 +stevedore==1.27.0 +urllib3==1.22