From 8b9dac50eca8d2737e377c225fdd5e4c4dc948c8 Mon Sep 17 00:00:00 2001 From: cwandrews Date: Mon, 18 Sep 2017 15:24:27 -0400 Subject: [PATCH 1/4] Added optional passing of one or multiple environment variables via --env flag to the bonobo cli. --- bonobo/commands/run.py | 12 +++++++++++- tests/test_commands.py | 27 +++++++++++++++++++++++++++ tests/util/get_passed_env.py | 22 ++++++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tests/util/get_passed_env.py diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index fb93e77..c9bb39e 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -40,7 +40,10 @@ def _install_requirements(requirements): importlib.reload(site) -def execute(filename, module, install=False, quiet=False, verbose=False): +def execute(filename, module, install=False, quiet=False, verbose=False, + env=None): + import re + import runpy from bonobo import Graph, run, settings @@ -50,6 +53,12 @@ def execute(filename, module, install=False, quiet=False, verbose=False): if verbose: settings.DEBUG.set(True) + if env: + quote_killer = re.compile('["\']') + for e in env: + var_name, var_value = e.split('=') + os.environ[var_name] = quote_killer.sub('', var_value) + if filename: if os.path.isdir(filename): if install: @@ -106,4 +115,5 @@ def register(parser): verbosity_group.add_argument('--quiet', '-q', action='store_true') verbosity_group.add_argument('--verbose', '-v', action='store_true') parser.add_argument('--install', '-I', action='store_true') + parser.add_argument('--env', '-e', action='append') return execute diff --git a/tests/test_commands.py b/tests/test_commands.py index daf245f..cff9e38 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,6 +3,7 @@ import runpy import sys from unittest.mock import patch +import pathlib import pkg_resources import pytest @@ -96,3 +97,29 @@ def test_version(runner, capsys): out = out.strip() assert out.startswith('bonobo ') assert __version__ in out + + +@all_runners +def test_run_with_env(runner, capsys): + runner('run', '--quiet', + str(pathlib.Path(os.path.dirname(__file__), + 'util', 'get_passed_env.py')), + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'") + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + +@all_runners +def test_run_module_with_env(runner, capsys): + runner('run', '--quiet', '-m', 'tests.util.get_passed_env', + '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', + '--env', "ENV_TEST_STRING='my_test_string'") + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' diff --git a/tests/util/get_passed_env.py b/tests/util/get_passed_env.py new file mode 100644 index 0000000..d9c4ba6 --- /dev/null +++ b/tests/util/get_passed_env.py @@ -0,0 +1,22 @@ +import os + +from bonobo import Graph + + +def extract(): + env_test_user = os.getenv('ENV_TEST_USER') + env_test_number = os.getenv('ENV_TEST_NUMBER') + env_test_string = os.getenv('ENV_TEST_STRING') + return env_test_user, env_test_number, env_test_string + + +def load(s: str): + print(s) + + +graph = Graph(extract, load) + +if __name__ == '__main__': + from bonobo import run + + run(graph) From 21514ad670f99da42b2967b89a6ece3d2e7e5ba1 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 30 Sep 2017 09:54:30 +0200 Subject: [PATCH 2/4] Update dependencies. --- Makefile | 6 +++--- requirements-dev.txt | 20 ++++++++++---------- requirements-docker.txt | 16 ++++++++-------- requirements-jupyter.txt | 31 ++++++++++++++++--------------- requirements.txt | 16 ++++++++-------- 5 files changed, 45 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index cb5f8bc..8175b3e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-07-16 10:52:31.093416 +# Updated at 2017-09-30 09:50:47.806007 PACKAGE ?= bonobo PYTHON ?= $(shell which python) @@ -27,13 +27,13 @@ VERSION ?= $(shell git describe 2>/dev/null || echo dev) # Installs the local project dependencies. install: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ fi # Installs the local project dependencies, including development-only libraries. install-dev: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ fi # Cleans up the local mess. diff --git a/requirements-dev.txt b/requirements-dev.txt index 69d64d8..92123d5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,16 +1,16 @@ -e .[dev] alabaster==0.7.10 arrow==0.10.0 -babel==2.4.0 -binaryornot==0.4.3 -certifi==2017.4.17 +babel==2.5.1 +binaryornot==0.4.4 +certifi==2017.7.27.1 chardet==3.0.4 click==6.7 cookiecutter==1.5.1 coverage==4.4.1 -docutils==0.13.1 +docutils==0.14 future==0.16.0 -idna==2.5 +idna==2.6 imagesize==0.7.1 jinja2-time==0.2.0 jinja2==2.9.6 @@ -21,14 +21,14 @@ pygments==2.2.0 pytest-cov==2.5.1 pytest-sugar==0.8.0 pytest-timeout==1.2.0 -pytest==3.1.3 +pytest==3.2.2 python-dateutil==2.6.1 pytz==2017.2 -requests==2.18.1 -six==1.10.0 +requests==2.18.4 +six==1.11.0 snowballstemmer==1.2.1 -sphinx==1.6.3 +sphinx==1.6.4 sphinxcontrib-websupport==1.0.1 termcolor==1.1.0 -urllib3==1.21.1 +urllib3==1.22 whichcraft==0.4.1 diff --git a/requirements-docker.txt b/requirements-docker.txt index f5e74fc..870223d 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,20 +1,20 @@ -e .[docker] appdirs==1.4.3 bonobo-docker==0.2.11 -certifi==2017.4.17 +certifi==2017.7.27.1 chardet==3.0.4 colorama==0.3.9 docker-pycreds==0.2.1 docker==2.3.0 -fs==2.0.4 -idna==2.5 +fs==2.0.11 +idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.2.2 +psutil==5.3.1 pyparsing==2.2.0 pytz==2017.2 -requests==2.18.1 -six==1.10.0 -stevedore==1.24.0 -urllib3==1.21.1 +requests==2.18.4 +six==1.11.0 +stevedore==1.27.0 +urllib3==1.22 websocket-client==0.44.0 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 2542040..94b10ea 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -1,40 +1,41 @@ -e .[jupyter] appnope==0.1.0 -bleach==2.0.0 -decorator==4.1.1 +bleach==2.1 +decorator==4.1.2 entrypoints==0.2.3 html5lib==0.999999999 ipykernel==4.6.1 ipython-genutils==0.2.0 -ipython==6.1.0 -ipywidgets==6.0.0 -jedi==0.10.2 +ipython==6.2.1 +ipywidgets==6.0.1 +jedi==0.11.0 jinja2==2.9.6 jsonschema==2.6.0 jupyter-client==5.1.0 -jupyter-console==5.1.0 +jupyter-console==5.2.0 jupyter-core==4.3.0 jupyter==1.0.0 markupsafe==1.0 mistune==0.7.4 -nbconvert==5.2.1 -nbformat==4.3.0 -notebook==5.0.0 -pandocfilters==1.4.1 +nbconvert==5.3.1 +nbformat==4.4.0 +notebook==5.1.0 +pandocfilters==1.4.2 +parso==0.1.0 pexpect==4.2.1 pickleshare==0.7.4 -prompt-toolkit==1.0.14 +prompt-toolkit==1.0.15 ptyprocess==0.5.2 pygments==2.2.0 python-dateutil==2.6.1 pyzmq==16.0.2 -qtconsole==4.3.0 +qtconsole==4.3.1 simplegeneric==0.8.1 -six==1.10.0 +six==1.11.0 terminado==0.6 testpath==0.3.1 -tornado==4.5.1 +tornado==4.5.2 traitlets==4.3.2 wcwidth==0.1.7 webencodings==0.5.1 -widgetsnbextension==2.0.0 +widgetsnbextension==2.0.1 diff --git a/requirements.txt b/requirements.txt index 5ddbb01..61ab27d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -e . appdirs==1.4.3 -certifi==2017.4.17 +certifi==2017.7.27.1 chardet==3.0.4 colorama==0.3.9 -fs==2.0.4 -idna==2.5 +fs==2.0.11 +idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.2.2 +psutil==5.3.1 pyparsing==2.2.0 pytz==2017.2 -requests==2.18.1 -six==1.10.0 -stevedore==1.24.0 -urllib3==1.21.1 +requests==2.18.4 +six==1.11.0 +stevedore==1.27.0 +urllib3==1.22 From 7ca3369f7141246823e0fe2dabb0d98e5d91af5a Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 30 Sep 2017 11:01:53 +0200 Subject: [PATCH 3/4] Rename "bonobo graph" to "bonobo inspect". For now, graphviz is default but there will probably be a humand default in the future, with graphviz source generation set if --graph (or -g) flag is passed. --- Makefile | 2 +- Projectfile | 2 +- bonobo/commands/graph.py | 32 -------------------------------- bonobo/commands/inspect.py | 33 +++++++++++++++++++++++++++++++++ bonobo/commands/run.py | 8 ++------ bonobo/util/graphviz.py | 3 +-- setup.py | 2 +- tests/test_commands.py | 17 +++++++++-------- 8 files changed, 48 insertions(+), 51 deletions(-) delete mode 100644 bonobo/commands/graph.py create mode 100644 bonobo/commands/inspect.py diff --git a/Makefile b/Makefile index 1e617cb..aac0c92 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-09-30 10:24:51.699716 +# Updated at 2017-09-30 10:57:00.855477 PACKAGE ?= bonobo PYTHON ?= $(shell which python) diff --git a/Projectfile b/Projectfile index eea8cc5..8e848b3 100644 --- a/Projectfile +++ b/Projectfile @@ -30,7 +30,7 @@ python.setup( ], 'bonobo.commands': [ 'init = bonobo.commands.init:register', - 'graph = bonobo.commands.graph:register', + 'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register', ], diff --git a/bonobo/commands/graph.py b/bonobo/commands/graph.py deleted file mode 100644 index 7afa8de..0000000 --- a/bonobo/commands/graph.py +++ /dev/null @@ -1,32 +0,0 @@ -import json - -import itertools - -from bonobo.util.objects import get_name -from bonobo.commands.run import read, register_generic_run_arguments -from bonobo.constants import BEGIN - - -def execute(filename, module, install=False, quiet=False, verbose=False): - graph, plugins, services = read(filename, module, install, quiet, verbose) - - print('digraph {') - print(' rankdir = LR;') - print(' "BEGIN" [shape="point"];') - - for i in graph.outputs_of(BEGIN): - print(' "BEGIN" -> ' + json.dumps(get_name(graph[i])) + ';') - - for ix in graph.topologically_sorted_indexes: - for iy in graph.outputs_of(ix): - print(' {} -> {};'.format( - json.dumps(get_name(graph[ix])), - json.dumps(get_name(graph[iy])) - )) - - print('}') - - -def register(parser): - register_generic_run_arguments(parser) - return execute diff --git a/bonobo/commands/inspect.py b/bonobo/commands/inspect.py new file mode 100644 index 0000000..83b770e --- /dev/null +++ b/bonobo/commands/inspect.py @@ -0,0 +1,33 @@ +import json + +from bonobo.commands.run import read, register_generic_run_arguments +from bonobo.constants import BEGIN +from bonobo.util.objects import get_name + +OUTPUT_GRAPHVIZ = 'graphviz' + +def execute(*, output, **kwargs): + graph, plugins, services = read(**kwargs) + + if output == OUTPUT_GRAPHVIZ: + print('digraph {') + print(' rankdir = LR;') + print(' "BEGIN" [shape="point"];') + + for i in graph.outputs_of(BEGIN): + print(' "BEGIN" -> ' + json.dumps(get_name(graph[i])) + ';') + + for ix in graph.topologically_sorted_indexes: + for iy in graph.outputs_of(ix): + print(' {} -> {};'.format(json.dumps(get_name(graph[ix])), json.dumps(get_name(graph[iy])))) + + print('}') + else: + raise NotImplementedError('Output type not implemented.') + + +def register(parser): + register_generic_run_arguments(parser) + parser.add_argument('--graph', '-g', dest='output', action='store_const', const=OUTPUT_GRAPHVIZ) + parser.set_defaults(output=OUTPUT_GRAPHVIZ) + return execute diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 27a2329..2204a3b 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -3,7 +3,7 @@ import os import bonobo from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME -DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py',) +DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', ) DEFAULT_GRAPH_ATTR = 'get_graph' @@ -99,11 +99,7 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None): def execute(filename, module, install=False, quiet=False, verbose=False, env=None): graph, plugins, services = read(filename, module, install, quiet, verbose, env) - return bonobo.run( - graph, - plugins=plugins, - services=services - ) + return bonobo.run(graph, plugins=plugins, services=services) def register_generic_run_arguments(parser, required=True): diff --git a/bonobo/util/graphviz.py b/bonobo/util/graphviz.py index fa88974..588e374 100644 --- a/bonobo/util/graphviz.py +++ b/bonobo/util/graphviz.py @@ -1,4 +1,3 @@ - def render_as_dot(graph): """ @@ -6,4 +5,4 @@ def render_as_dot(graph): :return: str """ - pass \ No newline at end of file + pass diff --git a/setup.py b/setup.py index 0abee00..08b84e0 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup( }, entry_points={ 'bonobo.commands': [ - 'init = bonobo.commands.init:register', 'graph = bonobo.commands.graph:register', + 'init = bonobo.commands.init:register', 'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register' ], 'console_scripts': ['bonobo = bonobo.commands:entrypoint'] diff --git a/tests/test_commands.py b/tests/test_commands.py index cff9e38..730bc0b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -101,11 +101,11 @@ def test_version(runner, capsys): @all_runners def test_run_with_env(runner, capsys): - runner('run', '--quiet', - str(pathlib.Path(os.path.dirname(__file__), - 'util', 'get_passed_env.py')), - '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', - '--env', "ENV_TEST_STRING='my_test_string'") + runner( + 'run', '--quiet', + str(pathlib.Path(os.path.dirname(__file__), 'util', 'get_passed_env.py')), '--env', 'ENV_TEST_NUMBER=123', + '--env', 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" + ) out, err = capsys.readouterr() out = out.split('\n') assert out[0] == 'cwandrews' @@ -115,9 +115,10 @@ def test_run_with_env(runner, capsys): @all_runners def test_run_module_with_env(runner, capsys): - runner('run', '--quiet', '-m', 'tests.util.get_passed_env', - '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews', - '--env', "ENV_TEST_STRING='my_test_string'") + runner( + 'run', '--quiet', '-m', 'tests.util.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env', + 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" + ) out, err = capsys.readouterr() out = out.split('\n') assert out[0] == 'cwandrews' From b49ccaa7a77aad78a4560c8469c1e09d37a74abb Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 30 Sep 2017 11:26:22 +0200 Subject: [PATCH 4/4] Formating. --- bonobo/commands/inspect.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bonobo/commands/inspect.py b/bonobo/commands/inspect.py index 83b770e..bb82704 100644 --- a/bonobo/commands/inspect.py +++ b/bonobo/commands/inspect.py @@ -6,6 +6,7 @@ from bonobo.util.objects import get_name OUTPUT_GRAPHVIZ = 'graphviz' + def execute(*, output, **kwargs): graph, plugins, services = read(**kwargs)