diff --git a/Makefile b/Makefile index cb5f8bc..d082166 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-07-16 10:52:31.093416 +# Updated at 2017-09-30 11:26:44.075878 PACKAGE ?= bonobo PYTHON ?= $(shell which python) @@ -27,13 +27,13 @@ VERSION ?= $(shell git describe 2>/dev/null || echo dev) # Installs the local project dependencies. install: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ fi # Installs the local project dependencies, including development-only libraries. install-dev: if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ fi # Cleans up the local mess. diff --git a/Projectfile b/Projectfile index 6fe6c2b..0973c1f 100644 --- a/Projectfile +++ b/Projectfile @@ -29,7 +29,9 @@ python.setup( 'bonobo = bonobo.commands:entrypoint', ], 'bonobo.commands': [ + 'convert = bonobo.commands.convert:register', 'init = bonobo.commands.init:register', + 'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register', ], @@ -56,3 +58,5 @@ python.add_requirements( 'ipywidgets >=6.0.0,<7', ] ) + +# vim: ft=python: diff --git a/bin/imgcat b/bin/imgcat new file mode 100755 index 0000000..001d2b8 --- /dev/null +++ b/bin/imgcat @@ -0,0 +1,112 @@ +#!/bin/bash + +# tmux requires unrecognized OSC sequences to be wrapped with DCS tmux; +# ST, and for all ESCs in to be replaced with ESC ESC. It +# only accepts ESC backslash for ST. +function print_osc() { + if [[ $TERM == screen* ]] ; then + printf "\033Ptmux;\033\033]" + else + printf "\033]" + fi +} + +# More of the tmux workaround described above. +function print_st() { + if [[ $TERM == screen* ]] ; then + printf "\a\033\\" + else + printf "\a" + fi +} + +# print_image filename inline base64contents print_filename +# filename: Filename to convey to client +# inline: 0 or 1 +# base64contents: Base64-encoded contents +# print_filename: If non-empty, print the filename +# before outputting the image +function print_image() { + print_osc + printf '1337;File=' + if [[ -n "$1" ]]; then + printf 'name='`printf "%s" "$1" | base64`";" + fi + + VERSION=$(base64 --version 2>&1) + if [[ "$VERSION" =~ fourmilab ]]; then + BASE64ARG=-d + elif [[ "$VERSION" =~ GNU ]]; then + BASE64ARG=-di + else + BASE64ARG=-D + fi + + printf "%s" "$3" | base64 $BASE64ARG | wc -c | awk '{printf "size=%d",$1}' + printf ";inline=$2" + printf ":" + printf "%s" "$3" + print_st + printf '\n' + if [[ -n "$4" ]]; then + echo $1 + fi +} + +function error() { + echo "ERROR: $*" 1>&2 +} + +function show_help() { + echo "Usage: imgcat [-p] filename ..." 1>& 2 + echo " or: cat filename | imgcat" 1>& 2 +} + +## Main + +if [ -t 0 ]; then + has_stdin=f +else + has_stdin=t +fi + +# Show help if no arguments and no stdin. +if [ $has_stdin = f -a $# -eq 0 ]; then + show_help + exit +fi + +# Look for command line flags. +while [ $# -gt 0 ]; do + case "$1" in + -h|--h|--help) + show_help + exit + ;; + -p|--p|--print) + print_filename=1 + ;; + -*) + error "Unknown option flag: $1" + show_help + exit 1 + ;; + *) + if [ -r "$1" ] ; then + has_stdin=f + print_image "$1" 1 "$(base64 < "$1")" "$print_filename" + else + error "imgcat: $1: No such file or directory" + exit 2 + fi + ;; + esac + shift +done + +# Read and print stdin +if [ $has_stdin = t ]; then + print_image "" 1 "$(cat | base64)" "" +fi + +exit 0 diff --git a/bin/test_graph b/bin/test_graph new file mode 100644 index 0000000..1e5fd85 --- /dev/null +++ b/bin/test_graph @@ -0,0 +1 @@ +bonobo graph bonobo/examples/tutorials/tut02_03_writeasmap.py | dot -otest.png -Tpng && bin/imgcat test.png diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py new file mode 100644 index 0000000..3c26724 --- /dev/null +++ b/bonobo/commands/convert.py @@ -0,0 +1,78 @@ +import mimetypes +import os + +import bonobo + +SHORTCUTS = { + 'csv': 'text/csv', + 'json': 'application/json', + 'pickle': 'pickle', + 'plain': 'text/plain', + 'text': 'text/plain', + 'txt': 'text/plain', +} + +REGISTRY = { + 'application/json': (bonobo.JsonReader, bonobo.JsonWriter), + 'pickle': (bonobo.PickleReader, bonobo.PickleWriter), + 'text/csv': (bonobo.CsvReader, bonobo.CsvWriter), + 'text/plain': (bonobo.FileReader, bonobo.FileWriter), +} + +READER = 'reader' +WRITER = 'writer' + + +def resolve_factory(name, filename, factory_type): + """ + Try to resolve which transformation factory to use for this filename. User eventually provided a name, which has + priority, otherwise we try to detect it using the mimetype detection on filename. + + """ + if name is None: + name = mimetypes.guess_type(filename)[0] + + if name in SHORTCUTS: + name = SHORTCUTS[name] + + if name is None: + _, _ext = os.path.splitext(filename) + if _ext: + _ext = _ext[1:] + if _ext in SHORTCUTS: + name = SHORTCUTS[_ext] + + if not name in REGISTRY: + raise RuntimeError( + 'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} .'.format( + name=name, filename=filename, factory_type=factory_type, opt=factory_type[0])) + + if factory_type == READER: + return REGISTRY[name][0] + elif factory_type == WRITER: + return REGISTRY[name][1] + else: + raise ValueError('Invalid factory type.') + + +def execute(input, output, reader=None, reader_options=None, writer=None, writer_options=None, options=None): + reader = resolve_factory(reader, input, READER)(input) + writer = resolve_factory(writer, output, WRITER)(output) + + graph = bonobo.Graph() + graph.add_chain(reader, writer) + + return bonobo.run(graph, services={ + 'fs': bonobo.open_fs(), + }) + + +def register(parser): + parser.add_argument('input') + parser.add_argument('output') + parser.add_argument('--' + READER, '-r') + parser.add_argument('--' + WRITER, '-w') + # parser.add_argument('--reader-option', '-ro', dest='reader_options') + # parser.add_argument('--writer-option', '-wo', dest='writer_options') + # parser.add_argument('--option', '-o', dest='options') + return execute diff --git a/bonobo/commands/inspect.py b/bonobo/commands/inspect.py new file mode 100644 index 0000000..bb82704 --- /dev/null +++ b/bonobo/commands/inspect.py @@ -0,0 +1,34 @@ +import json + +from bonobo.commands.run import read, register_generic_run_arguments +from bonobo.constants import BEGIN +from bonobo.util.objects import get_name + +OUTPUT_GRAPHVIZ = 'graphviz' + + +def execute(*, output, **kwargs): + graph, plugins, services = read(**kwargs) + + if output == OUTPUT_GRAPHVIZ: + print('digraph {') + print(' rankdir = LR;') + print(' "BEGIN" [shape="point"];') + + for i in graph.outputs_of(BEGIN): + print(' "BEGIN" -> ' + json.dumps(get_name(graph[i])) + ';') + + for ix in graph.topologically_sorted_indexes: + for iy in graph.outputs_of(ix): + print(' {} -> {};'.format(json.dumps(get_name(graph[ix])), json.dumps(get_name(graph[iy])))) + + print('}') + else: + raise NotImplementedError('Output type not implemented.') + + +def register(parser): + register_generic_run_arguments(parser) + parser.add_argument('--graph', '-g', dest='output', action='store_const', const=OUTPUT_GRAPHVIZ) + parser.set_defaults(output=OUTPUT_GRAPHVIZ) + return execute diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index fb93e77..2204a3b 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -1,7 +1,7 @@ import os -DEFAULT_SERVICES_FILENAME = '_services.py' -DEFAULT_SERVICES_ATTR = 'get_services' +import bonobo +from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', ) DEFAULT_GRAPH_ATTR = 'get_graph' @@ -40,9 +40,10 @@ def _install_requirements(requirements): importlib.reload(site) -def execute(filename, module, install=False, quiet=False, verbose=False): +def read(filename, module, install=False, quiet=False, verbose=False, env=None): + import re import runpy - from bonobo import Graph, run, settings + from bonobo import Graph, settings if quiet: settings.QUIET.set(True) @@ -50,6 +51,12 @@ def execute(filename, module, install=False, quiet=False, verbose=False): if verbose: settings.DEBUG.set(True) + if env: + quote_killer = re.compile('["\']') + for e in env: + var_name, var_value = e.split('=') + os.environ[var_name] = quote_killer.sub('', var_value) + if filename: if os.path.isdir(filename): if install: @@ -81,17 +88,19 @@ def execute(filename, module, install=False, quiet=False, verbose=False): ).format(len(graphs)) graph = list(graphs.values())[0] - - # todo if console and not quiet, then add the console plugin - # todo when better console plugin, add it if console and just disable display - return run( - graph, - plugins=[], - services=get_default_services( - filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None - ) + plugins = [] + services = get_default_services( + filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None ) + return graph, plugins, services + + +def execute(filename, module, install=False, quiet=False, verbose=False, env=None): + graph, plugins, services = read(filename, module, install, quiet, verbose, env) + + return bonobo.run(graph, plugins=plugins, services=services) + def register_generic_run_arguments(parser, required=True): source_group = parser.add_mutually_exclusive_group(required=required) @@ -106,4 +115,5 @@ def register(parser): verbosity_group.add_argument('--quiet', '-q', action='store_true') verbosity_group.add_argument('--verbose', '-v', action='store_true') parser.add_argument('--install', '-I', action='store_true') + parser.add_argument('--env', '-e', action='append') return execute diff --git a/bonobo/constants.py b/bonobo/constants.py index d567229..4187197 100644 --- a/bonobo/constants.py +++ b/bonobo/constants.py @@ -4,3 +4,5 @@ BEGIN = Token('Begin') END = Token('End') INHERIT_INPUT = Token('InheritInput') NOT_MODIFIED = Token('NotModified') +DEFAULT_SERVICES_FILENAME = '_services.py' +DEFAULT_SERVICES_ATTR = 'get_services' \ No newline at end of file diff --git a/bonobo/util/graphviz.py b/bonobo/util/graphviz.py new file mode 100644 index 0000000..588e374 --- /dev/null +++ b/bonobo/util/graphviz.py @@ -0,0 +1,8 @@ +def render_as_dot(graph): + """ + + :param bonobo.Graph graph: + :return: str + """ + + pass diff --git a/docs/reference/commands.rst b/docs/reference/commands.rst index dcd054a..674d549 100644 --- a/docs/reference/commands.rst +++ b/docs/reference/commands.rst @@ -1,6 +1,21 @@ Command-line ============ + +Bonobo Convert +:::::::::::::: + +Build a simple bonobo graph with one reader and one writer, then execute it, allowing to use bonobo in "no code" mode +for simple file format conversions. + +Syntax: `bonobo convert [-r reader] input_filename [-w writer] output_filename` + +.. todo:: + + add a way to override default options of reader/writers, add a way to add "filters", for example this could be used + to read from csv and write to csv too (or other format) but adding a geocoder filter that would add some fields. + + Bonobo Init ::::::::::: @@ -8,7 +23,17 @@ Create an empty project, ready to use bonobo. Syntax: `bonobo init` -Requires `edgy.project`. +Requires `cookiecutter`. + + +Bonobo Inspect +:::::::::::::: + +Inspects a bonobo graph source files. For now, only support graphviz output. + +Syntax: `bonobo inspect [--graph|-g] filename` + +Requires graphviz if you want to generate an actual graph picture, although the command itself depends on nothing. Bonobo Run @@ -20,6 +45,7 @@ Syntax: `bonobo run [-c cmd | -m mod | file | -] [arg]` .. todo:: implement -m, check if -c is of any use and if yes, implement it too. Implement args, too. + Bonobo RunC ::::::::::: diff --git a/requirements-dev.txt b/requirements-dev.txt index 69d64d8..92123d5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,16 +1,16 @@ -e .[dev] alabaster==0.7.10 arrow==0.10.0 -babel==2.4.0 -binaryornot==0.4.3 -certifi==2017.4.17 +babel==2.5.1 +binaryornot==0.4.4 +certifi==2017.7.27.1 chardet==3.0.4 click==6.7 cookiecutter==1.5.1 coverage==4.4.1 -docutils==0.13.1 +docutils==0.14 future==0.16.0 -idna==2.5 +idna==2.6 imagesize==0.7.1 jinja2-time==0.2.0 jinja2==2.9.6 @@ -21,14 +21,14 @@ pygments==2.2.0 pytest-cov==2.5.1 pytest-sugar==0.8.0 pytest-timeout==1.2.0 -pytest==3.1.3 +pytest==3.2.2 python-dateutil==2.6.1 pytz==2017.2 -requests==2.18.1 -six==1.10.0 +requests==2.18.4 +six==1.11.0 snowballstemmer==1.2.1 -sphinx==1.6.3 +sphinx==1.6.4 sphinxcontrib-websupport==1.0.1 termcolor==1.1.0 -urllib3==1.21.1 +urllib3==1.22 whichcraft==0.4.1 diff --git a/requirements-docker.txt b/requirements-docker.txt index f5e74fc..870223d 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,20 +1,20 @@ -e .[docker] appdirs==1.4.3 bonobo-docker==0.2.11 -certifi==2017.4.17 +certifi==2017.7.27.1 chardet==3.0.4 colorama==0.3.9 docker-pycreds==0.2.1 docker==2.3.0 -fs==2.0.4 -idna==2.5 +fs==2.0.11 +idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.2.2 +psutil==5.3.1 pyparsing==2.2.0 pytz==2017.2 -requests==2.18.1 -six==1.10.0 -stevedore==1.24.0 -urllib3==1.21.1 +requests==2.18.4 +six==1.11.0 +stevedore==1.27.0 +urllib3==1.22 websocket-client==0.44.0 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 2542040..94b10ea 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -1,40 +1,41 @@ -e .[jupyter] appnope==0.1.0 -bleach==2.0.0 -decorator==4.1.1 +bleach==2.1 +decorator==4.1.2 entrypoints==0.2.3 html5lib==0.999999999 ipykernel==4.6.1 ipython-genutils==0.2.0 -ipython==6.1.0 -ipywidgets==6.0.0 -jedi==0.10.2 +ipython==6.2.1 +ipywidgets==6.0.1 +jedi==0.11.0 jinja2==2.9.6 jsonschema==2.6.0 jupyter-client==5.1.0 -jupyter-console==5.1.0 +jupyter-console==5.2.0 jupyter-core==4.3.0 jupyter==1.0.0 markupsafe==1.0 mistune==0.7.4 -nbconvert==5.2.1 -nbformat==4.3.0 -notebook==5.0.0 -pandocfilters==1.4.1 +nbconvert==5.3.1 +nbformat==4.4.0 +notebook==5.1.0 +pandocfilters==1.4.2 +parso==0.1.0 pexpect==4.2.1 pickleshare==0.7.4 -prompt-toolkit==1.0.14 +prompt-toolkit==1.0.15 ptyprocess==0.5.2 pygments==2.2.0 python-dateutil==2.6.1 pyzmq==16.0.2 -qtconsole==4.3.0 +qtconsole==4.3.1 simplegeneric==0.8.1 -six==1.10.0 +six==1.11.0 terminado==0.6 testpath==0.3.1 -tornado==4.5.1 +tornado==4.5.2 traitlets==4.3.2 wcwidth==0.1.7 webencodings==0.5.1 -widgetsnbextension==2.0.0 +widgetsnbextension==2.0.1 diff --git a/requirements.txt b/requirements.txt index 5ddbb01..61ab27d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -e . appdirs==1.4.3 -certifi==2017.4.17 +certifi==2017.7.27.1 chardet==3.0.4 colorama==0.3.9 -fs==2.0.4 -idna==2.5 +fs==2.0.11 +idna==2.6 packaging==16.8 pbr==3.1.1 -psutil==5.2.2 +psutil==5.3.1 pyparsing==2.2.0 pytz==2017.2 -requests==2.18.1 -six==1.10.0 -stevedore==1.24.0 -urllib3==1.21.1 +requests==2.18.4 +six==1.11.0 +stevedore==1.27.0 +urllib3==1.22 diff --git a/setup.py b/setup.py index 89c9ccd..be97d0c 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,8 @@ setup( }, entry_points={ 'bonobo.commands': [ - 'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register', + 'convert = bonobo.commands.convert:register', 'init = bonobo.commands.init:register', + 'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register' ], 'console_scripts': ['bonobo = bonobo.commands:entrypoint'] diff --git a/tests/test_commands.py b/tests/test_commands.py index daf245f..730bc0b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,6 +3,7 @@ import runpy import sys from unittest.mock import patch +import pathlib import pkg_resources import pytest @@ -96,3 +97,30 @@ def test_version(runner, capsys): out = out.strip() assert out.startswith('bonobo ') assert __version__ in out + + +@all_runners +def test_run_with_env(runner, capsys): + runner( + 'run', '--quiet', + str(pathlib.Path(os.path.dirname(__file__), 'util', 'get_passed_env.py')), '--env', 'ENV_TEST_NUMBER=123', + '--env', 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' + + +@all_runners +def test_run_module_with_env(runner, capsys): + runner( + 'run', '--quiet', '-m', 'tests.util.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env', + 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'" + ) + out, err = capsys.readouterr() + out = out.split('\n') + assert out[0] == 'cwandrews' + assert out[1] == '123' + assert out[2] == 'my_test_string' diff --git a/tests/util/get_passed_env.py b/tests/util/get_passed_env.py new file mode 100644 index 0000000..d9c4ba6 --- /dev/null +++ b/tests/util/get_passed_env.py @@ -0,0 +1,22 @@ +import os + +from bonobo import Graph + + +def extract(): + env_test_user = os.getenv('ENV_TEST_USER') + env_test_number = os.getenv('ENV_TEST_NUMBER') + env_test_string = os.getenv('ENV_TEST_STRING') + return env_test_user, env_test_number, env_test_string + + +def load(s: str): + print(s) + + +graph = Graph(extract, load) + +if __name__ == '__main__': + from bonobo import run + + run(graph)