Merge branch 'develop' into transformation_factory

Makefile

@@ -1,7 +1,7 @@
 # This file has been auto-generated.
 # All changes will be lost, see Projectfile.
 #
-# Updated at 2017-07-16 10:52:31.093416
+# Updated at 2017-09-30 11:26:44.075878
 
 PACKAGE ?= bonobo
 PYTHON ?= $(shell which python)
@@ -27,13 +27,13 @@ VERSION ?= $(shell git describe 2>/dev/null || echo dev)
 # Installs the local project dependencies.
 install:
 	if [ -z "$(QUICK)" ]; then \
-	    $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \
+	    $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \
 	fi
 
 # Installs the local project dependencies, including development-only libraries.
 install-dev:
 	if [ -z "$(QUICK)" ]; then \
-	    $(PIP) install -U pip wheel $(PYTHON_PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \
+	    $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \
 	fi
 
 # Cleans up the local mess.

Projectfile

@@ -29,7 +29,9 @@ python.setup(
             'bonobo = bonobo.commands:entrypoint',
         ],
         'bonobo.commands': [
+            'convert = bonobo.commands.convert:register',
             'init = bonobo.commands.init:register',
+            'inspect = bonobo.commands.inspect:register',
             'run = bonobo.commands.run:register',
             'version = bonobo.commands.version:register',
         ],
@@ -56,3 +58,5 @@ python.add_requirements(
         'ipywidgets >=6.0.0,<7',
     ]
 )
+
+# vim: ft=python:

bin/imgcat

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# tmux requires unrecognized OSC sequences to be wrapped with DCS tmux;
+# <sequence> ST, and for all ESCs in <sequence> to be replaced with ESC ESC. It
+# only accepts ESC backslash for ST.
+function print_osc() {
+    if [[ $TERM == screen* ]] ; then
+        printf "\033Ptmux;\033\033]"
+    else
+        printf "\033]"
+    fi
+}
+
+# More of the tmux workaround described above.
+function print_st() {
+    if [[ $TERM == screen* ]] ; then
+        printf "\a\033\\"
+    else
+        printf "\a"
+    fi
+}
+
+# print_image filename inline base64contents print_filename
+#   filename: Filename to convey to client
+#   inline: 0 or 1
+#   base64contents: Base64-encoded contents
+#   print_filename: If non-empty, print the filename 
+#                   before outputting the image
+function print_image() {
+    print_osc
+    printf '1337;File='
+    if [[ -n "$1" ]]; then
+      printf 'name='`printf "%s" "$1" | base64`";"
+    fi
+
+    VERSION=$(base64 --version 2>&1)
+    if [[ "$VERSION" =~ fourmilab ]]; then
+      BASE64ARG=-d
+    elif [[ "$VERSION" =~ GNU ]]; then
+      BASE64ARG=-di
+    else
+      BASE64ARG=-D
+    fi
+
+    printf "%s" "$3" | base64 $BASE64ARG | wc -c | awk '{printf "size=%d",$1}'
+    printf ";inline=$2"
+    printf ":"
+    printf "%s" "$3"
+    print_st
+    printf '\n'
+    if [[ -n "$4" ]]; then
+      echo $1
+    fi
+}
+
+function error() {
+    echo "ERROR: $*" 1>&2
+}
+
+function show_help() {
+    echo "Usage: imgcat [-p] filename ..." 1>& 2
+    echo "   or: cat filename | imgcat" 1>& 2
+}
+
+## Main
+
+if [ -t 0 ]; then
+    has_stdin=f
+else
+    has_stdin=t
+fi
+
+# Show help if no arguments and no stdin.
+if [ $has_stdin = f -a $# -eq 0 ]; then
+    show_help
+    exit
+fi
+
+# Look for command line flags.
+while [ $# -gt 0 ]; do
+    case "$1" in
+    -h|--h|--help)
+        show_help
+        exit
+        ;;
+    -p|--p|--print)
+        print_filename=1
+        ;;
+    -*)
+        error "Unknown option flag: $1"
+        show_help
+        exit 1
+      ;;
+    *)
+        if [ -r "$1" ] ; then
+            has_stdin=f
+            print_image "$1" 1 "$(base64 < "$1")" "$print_filename"
+        else
+            error "imgcat: $1: No such file or directory"
+            exit 2
+        fi
+        ;;
+    esac
+    shift
+done
+
+# Read and print stdin
+if [ $has_stdin = t ]; then
+    print_image "" 1 "$(cat | base64)" ""
+fi
+
+exit 0

bin/test_graph

@@ -0,0 +1 @@
+bonobo graph bonobo/examples/tutorials/tut02_03_writeasmap.py | dot -otest.png -Tpng && bin/imgcat test.png 

bonobo/commands/convert.py

@@ -0,0 +1,78 @@
+import mimetypes
+import os
+
+import bonobo
+
+SHORTCUTS = {
+    'csv': 'text/csv',
+    'json': 'application/json',
+    'pickle': 'pickle',
+    'plain': 'text/plain',
+    'text': 'text/plain',
+    'txt': 'text/plain',
+}
+
+REGISTRY = {
+    'application/json': (bonobo.JsonReader, bonobo.JsonWriter),
+    'pickle': (bonobo.PickleReader, bonobo.PickleWriter),
+    'text/csv': (bonobo.CsvReader, bonobo.CsvWriter),
+    'text/plain': (bonobo.FileReader, bonobo.FileWriter),
+}
+
+READER = 'reader'
+WRITER = 'writer'
+
+
+def resolve_factory(name, filename, factory_type):
+    """
+    Try to resolve which transformation factory to use for this filename. User eventually provided a name, which has
+    priority, otherwise we try to detect it using the mimetype detection on filename.
+
+    """
+    if name is None:
+        name = mimetypes.guess_type(filename)[0]
+
+    if name in SHORTCUTS:
+        name = SHORTCUTS[name]
+
+    if name is None:
+        _, _ext = os.path.splitext(filename)
+        if _ext:
+            _ext = _ext[1:]
+        if _ext in SHORTCUTS:
+            name = SHORTCUTS[_ext]
+
+    if not name in REGISTRY:
+        raise RuntimeError(
+            'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} <format>.'.format(
+                name=name, filename=filename, factory_type=factory_type, opt=factory_type[0]))
+
+    if factory_type == READER:
+        return REGISTRY[name][0]
+    elif factory_type == WRITER:
+        return REGISTRY[name][1]
+    else:
+        raise ValueError('Invalid factory type.')
+
+
+def execute(input, output, reader=None, reader_options=None, writer=None, writer_options=None, options=None):
+    reader = resolve_factory(reader, input, READER)(input)
+    writer = resolve_factory(writer, output, WRITER)(output)
+
+    graph = bonobo.Graph()
+    graph.add_chain(reader, writer)
+
+    return bonobo.run(graph, services={
+        'fs': bonobo.open_fs(),
+    })
+
+
+def register(parser):
+    parser.add_argument('input')
+    parser.add_argument('output')
+    parser.add_argument('--' + READER, '-r')
+    parser.add_argument('--' + WRITER, '-w')
+    # parser.add_argument('--reader-option', '-ro', dest='reader_options')
+    # parser.add_argument('--writer-option', '-wo', dest='writer_options')
+    # parser.add_argument('--option', '-o', dest='options')
+    return execute

bonobo/commands/inspect.py

@@ -0,0 +1,34 @@
+import json
+
+from bonobo.commands.run import read, register_generic_run_arguments
+from bonobo.constants import BEGIN
+from bonobo.util.objects import get_name
+
+OUTPUT_GRAPHVIZ = 'graphviz'
+
+
+def execute(*, output, **kwargs):
+    graph, plugins, services = read(**kwargs)
+
+    if output == OUTPUT_GRAPHVIZ:
+        print('digraph {')
+        print('  rankdir = LR;')
+        print('  "BEGIN" [shape="point"];')
+
+        for i in graph.outputs_of(BEGIN):
+            print('  "BEGIN" -> ' + json.dumps(get_name(graph[i])) + ';')
+
+        for ix in graph.topologically_sorted_indexes:
+            for iy in graph.outputs_of(ix):
+                print('  {} -> {};'.format(json.dumps(get_name(graph[ix])), json.dumps(get_name(graph[iy]))))
+
+        print('}')
+    else:
+        raise NotImplementedError('Output type not implemented.')
+
+
+def register(parser):
+    register_generic_run_arguments(parser)
+    parser.add_argument('--graph', '-g', dest='output', action='store_const', const=OUTPUT_GRAPHVIZ)
+    parser.set_defaults(output=OUTPUT_GRAPHVIZ)
+    return execute

bonobo/commands/run.py

@@ -1,7 +1,7 @@
 import os
 
-DEFAULT_SERVICES_FILENAME = '_services.py'
+import bonobo
-DEFAULT_SERVICES_ATTR = 'get_services'
+from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME
 
 DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', )
 DEFAULT_GRAPH_ATTR = 'get_graph'
@@ -40,9 +40,10 @@ def _install_requirements(requirements):
     importlib.reload(site)
 
 
-def execute(filename, module, install=False, quiet=False, verbose=False):
+def read(filename, module, install=False, quiet=False, verbose=False, env=None):
+    import re
     import runpy
-    from bonobo import Graph, run, settings
+    from bonobo import Graph, settings
 
     if quiet:
         settings.QUIET.set(True)
@@ -50,6 +51,12 @@ def execute(filename, module, install=False, quiet=False, verbose=False):
     if verbose:
         settings.DEBUG.set(True)
 
+    if env:
+        quote_killer = re.compile('["\']')
+        for e in env:
+            var_name, var_value = e.split('=')
+            os.environ[var_name] = quote_killer.sub('', var_value)
+
     if filename:
         if os.path.isdir(filename):
             if install:
@@ -81,17 +88,19 @@ def execute(filename, module, install=False, quiet=False, verbose=False):
     ).format(len(graphs))
 
     graph = list(graphs.values())[0]
-
+    plugins = []
-    # todo if console and not quiet, then add the console plugin
+    services = get_default_services(
-    # todo when better console plugin, add it if console and just disable display
+        filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
-    return run(
-        graph,
-        plugins=[],
-        services=get_default_services(
-            filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
-        )
     )
 
+    return graph, plugins, services
+
+
+def execute(filename, module, install=False, quiet=False, verbose=False, env=None):
+    graph, plugins, services = read(filename, module, install, quiet, verbose, env)
+
+    return bonobo.run(graph, plugins=plugins, services=services)
+
 
 def register_generic_run_arguments(parser, required=True):
     source_group = parser.add_mutually_exclusive_group(required=required)
@@ -106,4 +115,5 @@ def register(parser):
     verbosity_group.add_argument('--quiet', '-q', action='store_true')
     verbosity_group.add_argument('--verbose', '-v', action='store_true')
     parser.add_argument('--install', '-I', action='store_true')
+    parser.add_argument('--env', '-e', action='append')
     return execute

bonobo/constants.py

@@ -4,3 +4,5 @@ BEGIN = Token('Begin')
 END = Token('End')
 INHERIT_INPUT = Token('InheritInput')
 NOT_MODIFIED = Token('NotModified')
+DEFAULT_SERVICES_FILENAME = '_services.py'
+DEFAULT_SERVICES_ATTR = 'get_services'

bonobo/util/graphviz.py

@@ -0,0 +1,8 @@
+def render_as_dot(graph):
+    """
+    
+    :param bonobo.Graph graph: 
+    :return: str
+    """
+
+    pass

docs/reference/commands.rst

@@ -1,6 +1,21 @@
 Command-line
 ============
 
+
+Bonobo Convert
+::::::::::::::
+
+Build a simple bonobo graph with one reader and one writer, then execute it, allowing to use bonobo in "no code" mode
+for simple file format conversions.
+
+Syntax: `bonobo convert [-r reader] input_filename [-w writer] output_filename`
+
+.. todo::
+
+    add a way to override default options of reader/writers, add a way to add "filters", for example this could be used
+    to read from csv and write to csv too (or other format) but adding a geocoder filter that would add some fields.
+
+
 Bonobo Init
 :::::::::::
 
@@ -8,7 +23,17 @@ Create an empty project, ready to use bonobo.
 
 Syntax: `bonobo init`
 
-Requires `edgy.project`.
+Requires `cookiecutter`.
+
+
+Bonobo Inspect
+::::::::::::::
+
+Inspects a bonobo graph source files. For now, only support graphviz output.
+
+Syntax: `bonobo inspect [--graph|-g] filename`
+
+Requires graphviz if you want to generate an actual graph picture, although the command itself depends on nothing.
 
 
 Bonobo Run
@@ -20,6 +45,7 @@ Syntax: `bonobo run [-c cmd | -m mod | file | -] [arg]`
 
 .. todo:: implement -m, check if -c is of any use and if yes, implement it too. Implement args, too.
 
+
 Bonobo RunC
 :::::::::::
 

requirements-dev.txt

@@ -1,16 +1,16 @@
 -e .[dev]
 alabaster==0.7.10
 arrow==0.10.0
-babel==2.4.0
+babel==2.5.1
-binaryornot==0.4.3
+binaryornot==0.4.4
-certifi==2017.4.17
+certifi==2017.7.27.1
 chardet==3.0.4
 click==6.7
 cookiecutter==1.5.1
 coverage==4.4.1
-docutils==0.13.1
+docutils==0.14
 future==0.16.0
-idna==2.5
+idna==2.6
 imagesize==0.7.1
 jinja2-time==0.2.0
 jinja2==2.9.6
@@ -21,14 +21,14 @@ pygments==2.2.0
 pytest-cov==2.5.1
 pytest-sugar==0.8.0
 pytest-timeout==1.2.0
-pytest==3.1.3
+pytest==3.2.2
 python-dateutil==2.6.1
 pytz==2017.2
-requests==2.18.1
+requests==2.18.4
-six==1.10.0
+six==1.11.0
 snowballstemmer==1.2.1
-sphinx==1.6.3
+sphinx==1.6.4
 sphinxcontrib-websupport==1.0.1
 termcolor==1.1.0
-urllib3==1.21.1
+urllib3==1.22
 whichcraft==0.4.1

requirements-docker.txt

@@ -1,20 +1,20 @@
 -e .[docker]
 appdirs==1.4.3
 bonobo-docker==0.2.11
-certifi==2017.4.17
+certifi==2017.7.27.1
 chardet==3.0.4
 colorama==0.3.9
 docker-pycreds==0.2.1
 docker==2.3.0
-fs==2.0.4
+fs==2.0.11
-idna==2.5
+idna==2.6
 packaging==16.8
 pbr==3.1.1
-psutil==5.2.2
+psutil==5.3.1
 pyparsing==2.2.0
 pytz==2017.2
-requests==2.18.1
+requests==2.18.4
-six==1.10.0
+six==1.11.0
-stevedore==1.24.0
+stevedore==1.27.0
-urllib3==1.21.1
+urllib3==1.22
 websocket-client==0.44.0

requirements-jupyter.txt

@@ -1,40 +1,41 @@
 -e .[jupyter]
 appnope==0.1.0
-bleach==2.0.0
+bleach==2.1
-decorator==4.1.1
+decorator==4.1.2
 entrypoints==0.2.3
 html5lib==0.999999999
 ipykernel==4.6.1
 ipython-genutils==0.2.0
-ipython==6.1.0
+ipython==6.2.1
-ipywidgets==6.0.0
+ipywidgets==6.0.1
-jedi==0.10.2
+jedi==0.11.0
 jinja2==2.9.6
 jsonschema==2.6.0
 jupyter-client==5.1.0
-jupyter-console==5.1.0
+jupyter-console==5.2.0
 jupyter-core==4.3.0
 jupyter==1.0.0
 markupsafe==1.0
 mistune==0.7.4
-nbconvert==5.2.1
+nbconvert==5.3.1
-nbformat==4.3.0
+nbformat==4.4.0
-notebook==5.0.0
+notebook==5.1.0
-pandocfilters==1.4.1
+pandocfilters==1.4.2
+parso==0.1.0
 pexpect==4.2.1
 pickleshare==0.7.4
-prompt-toolkit==1.0.14
+prompt-toolkit==1.0.15
 ptyprocess==0.5.2
 pygments==2.2.0
 python-dateutil==2.6.1
 pyzmq==16.0.2
-qtconsole==4.3.0
+qtconsole==4.3.1
 simplegeneric==0.8.1
-six==1.10.0
+six==1.11.0
 terminado==0.6
 testpath==0.3.1
-tornado==4.5.1
+tornado==4.5.2
 traitlets==4.3.2
 wcwidth==0.1.7
 webencodings==0.5.1
-widgetsnbextension==2.0.0
+widgetsnbextension==2.0.1

requirements.txt

@@ -1,16 +1,16 @@
 -e .
 appdirs==1.4.3
-certifi==2017.4.17
+certifi==2017.7.27.1
 chardet==3.0.4
 colorama==0.3.9
-fs==2.0.4
+fs==2.0.11
-idna==2.5
+idna==2.6
 packaging==16.8
 pbr==3.1.1
-psutil==5.2.2
+psutil==5.3.1
 pyparsing==2.2.0
 pytz==2017.2
-requests==2.18.1
+requests==2.18.4
-six==1.10.0
+six==1.11.0
-stevedore==1.24.0
+stevedore==1.27.0
-urllib3==1.21.1
+urllib3==1.22

setup.py

@@ -67,7 +67,8 @@ setup(
     },
     entry_points={
         'bonobo.commands': [
-            'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register',
+            'convert = bonobo.commands.convert:register', 'init = bonobo.commands.init:register',
+            'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register',
             'version = bonobo.commands.version:register'
         ],
         'console_scripts': ['bonobo = bonobo.commands:entrypoint']

tests/test_commands.py

@@ -3,6 +3,7 @@ import runpy
 import sys
 from unittest.mock import patch
 
+import pathlib
 import pkg_resources
 import pytest
 
@@ -96,3 +97,30 @@ def test_version(runner, capsys):
     out = out.strip()
     assert out.startswith('bonobo ')
     assert __version__ in out
+
+
+@all_runners
+def test_run_with_env(runner, capsys):
+    runner(
+        'run', '--quiet',
+        str(pathlib.Path(os.path.dirname(__file__), 'util', 'get_passed_env.py')), '--env', 'ENV_TEST_NUMBER=123',
+        '--env', 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
+    )
+    out, err = capsys.readouterr()
+    out = out.split('\n')
+    assert out[0] == 'cwandrews'
+    assert out[1] == '123'
+    assert out[2] == 'my_test_string'
+
+
+@all_runners
+def test_run_module_with_env(runner, capsys):
+    runner(
+        'run', '--quiet', '-m', 'tests.util.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env',
+        'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
+    )
+    out, err = capsys.readouterr()
+    out = out.split('\n')
+    assert out[0] == 'cwandrews'
+    assert out[1] == '123'
+    assert out[2] == 'my_test_string'

tests/util/get_passed_env.py

@@ -0,0 +1,22 @@
+import os
+
+from bonobo import Graph
+
+
+def extract():
+    env_test_user = os.getenv('ENV_TEST_USER')
+    env_test_number = os.getenv('ENV_TEST_NUMBER')
+    env_test_string = os.getenv('ENV_TEST_STRING')
+    return env_test_user, env_test_number, env_test_string
+
+
+def load(s: str):
+    print(s)
+
+
+graph = Graph(extract, load)
+
+if __name__ == '__main__':
+    from bonobo import run
+
+    run(graph)