From 3059f112f1c8d3c7f909c2ac32d531f208629095 Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sun, 21 May 2017 20:42:54 +0200
Subject: [PATCH 01/16] WIP: factory for transformations

---
 bonobo/examples/datasets/fablabs.py         |  12 +-
 bonobo/examples/nodes/_services.py          |   5 +
 bonobo/examples/{types => nodes}/bags.py    |   0
 bonobo/examples/{types => nodes}/dicts.py   |   0
 bonobo/examples/nodes/factory.py            |  33 ++++
 bonobo/examples/{types => nodes}/strings.py |   0
 bonobo/examples/types/__init__.py           |   7 -
 bonobo/nodes/basics.py                      |   4 +-
 bonobo/nodes/factory.py                     | 208 ++++++++++++++++++++
 bonobo/nodes/io/json.py                     |   9 +
 10 files changed, 261 insertions(+), 17 deletions(-)
 create mode 100644 bonobo/examples/nodes/_services.py
 rename bonobo/examples/{types => nodes}/bags.py (100%)
 rename bonobo/examples/{types => nodes}/dicts.py (100%)
 create mode 100644 bonobo/examples/nodes/factory.py
 rename bonobo/examples/{types => nodes}/strings.py (100%)
 delete mode 100644 bonobo/examples/types/__init__.py
 create mode 100644 bonobo/nodes/factory.py

diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py
index be95fe1..333cb17 100644
--- a/bonobo/examples/datasets/fablabs.py
+++ b/bonobo/examples/datasets/fablabs.py
@@ -48,10 +48,6 @@ def normalize(row):
     return result
 
 
-def filter_france(row):
-    if row.get('country') == 'France':
-        yield row
-
 
 def display(row):
     print(Style.BRIGHT, row.get('name'), Style.RESET_ALL, sep='')
@@ -73,15 +69,15 @@ def display(row):
 
     print(
         '  - {}address{}: {address}'.
-        format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address))
+            format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address))
     )
     print(
         '  - {}links{}: {links}'.
-        format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links']))
+            format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links']))
     )
     print(
         '  - {}geometry{}: {geometry}'.
-        format(Fore.BLUE, Style.RESET_ALL, **row)
+            format(Fore.BLUE, Style.RESET_ALL, **row)
     )
     print(
         '  - {}source{}: {source}'.format(
@@ -95,7 +91,7 @@ graph = bonobo.Graph(
         dataset=API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris'
     ),
     normalize,
-    filter_france,
+    bonobo.Filter(filter=lambda row: row.get('country') == 'France'),
     bonobo.Tee(display),
     bonobo.JsonWriter(path='fablabs.txt'),
 )
diff --git a/bonobo/examples/nodes/_services.py b/bonobo/examples/nodes/_services.py
new file mode 100644
index 0000000..337bf6b
--- /dev/null
+++ b/bonobo/examples/nodes/_services.py
@@ -0,0 +1,5 @@
+from bonobo import get_examples_path, open_fs
+
+
+def get_services():
+    return {'fs': open_fs(get_examples_path())}
diff --git a/bonobo/examples/types/bags.py b/bonobo/examples/nodes/bags.py
similarity index 100%
rename from bonobo/examples/types/bags.py
rename to bonobo/examples/nodes/bags.py
diff --git a/bonobo/examples/types/dicts.py b/bonobo/examples/nodes/dicts.py
similarity index 100%
rename from bonobo/examples/types/dicts.py
rename to bonobo/examples/nodes/dicts.py
diff --git a/bonobo/examples/nodes/factory.py b/bonobo/examples/nodes/factory.py
new file mode 100644
index 0000000..d4702c6
--- /dev/null
+++ b/bonobo/examples/nodes/factory.py
@@ -0,0 +1,33 @@
+from functools import partial
+
+import itertools
+
+import bonobo
+from bonobo.commands.run import get_default_services
+from bonobo.config import Configurable
+from bonobo.nodes.factory import Factory
+from bonobo.nodes.io.json import JsonDictReader
+
+@Factory
+def Normalize(self):
+    self[0].str().title()
+    self.move(0, 'title')
+    self.move(0, 'address')
+
+
+
+
+class PrettyPrinter(Configurable):
+    def call(self, *args, **kwargs):
+        for i, (item, value) in enumerate(itertools.chain(enumerate(args), kwargs.items())):
+            print('  ' if i else '• ', item, '=', value)
+
+
+graph = bonobo.Graph(
+    JsonDictReader('datasets/coffeeshops.json'),
+    Normalize(),
+    PrettyPrinter(),
+)
+
+if __name__ == '__main__':
+    bonobo.run(graph, services=get_default_services(__file__))
diff --git a/bonobo/examples/types/strings.py b/bonobo/examples/nodes/strings.py
similarity index 100%
rename from bonobo/examples/types/strings.py
rename to bonobo/examples/nodes/strings.py
diff --git a/bonobo/examples/types/__init__.py b/bonobo/examples/types/__init__.py
deleted file mode 100644
index a2c0ceb..0000000
--- a/bonobo/examples/types/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from . import bags, dicts, strings
-
-__all__ = [
-    'bags',
-    'dicts',
-    'strings',
-]
\ No newline at end of file
diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py
index 195cd8e..094e8e6 100644
--- a/bonobo/nodes/basics.py
+++ b/bonobo/nodes/basics.py
@@ -3,8 +3,7 @@ from pprint import pprint as _pprint
 
 from colorama import Fore, Style
 
-from bonobo.config import Configurable, Option
-from bonobo.config.processors import ContextProcessor
+from bonobo.config import Configurable, ContextProcessor, Option
 from bonobo.structs.bags import Bag
 from bonobo.util.objects import ValueHolder
 from bonobo.util.term import CLEAR_EOL
@@ -74,6 +73,7 @@ pprint = Tee(_pprint)
 
 def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):
     from bonobo.constants import NOT_MODIFIED
+    from colorama import Fore, Style
 
     def _pprint(*args, **kwargs):
         nonlocal title_keys, sort, print_values
diff --git a/bonobo/nodes/factory.py b/bonobo/nodes/factory.py
new file mode 100644
index 0000000..af3f778
--- /dev/null
+++ b/bonobo/nodes/factory.py
@@ -0,0 +1,208 @@
+import functools
+from functools import partial
+
+from bonobo import Bag
+from bonobo.config import Configurable, Method
+
+_isarg = lambda item: type(item) is int
+_iskwarg = lambda item: type(item) is str
+
+
+class Operation():
+    def __init__(self, item, callable):
+        self.item = item
+        self.callable = callable
+
+    def __repr__(self):
+        return '<operation {} on {}>'.format(self.callable.__name__, self.item)
+
+    def apply(self, *args, **kwargs):
+        if _isarg(self.item):
+            return (*args[0:self.item], self.callable(args[self.item]), *args[self.item + 1:]), kwargs
+        if _iskwarg(self.item):
+            return args, {**kwargs, self.item: self.callable(kwargs.get(self.item))}
+        raise RuntimeError('Houston, we have a problem...')
+
+
+class FactoryOperation():
+    def __init__(self, factory, callable):
+        self.factory = factory
+        self.callable = callable
+
+    def __repr__(self):
+        return '<factory operation {}>'.format(self.callable.__name__)
+
+    def apply(self, *args, **kwargs):
+        return self.callable(*args, **kwargs)
+
+
+CURSOR_TYPES = {}
+
+
+def operation(mixed):
+    def decorator(m, ctype=mixed):
+        def lazy_operation(self, *args, **kwargs):
+            @functools.wraps(m)
+            def actual_operation(x):
+                return m(self, x, *args, **kwargs)
+
+            self.factory.operations.append(Operation(self.item, actual_operation))
+            return CURSOR_TYPES[ctype](self.factory, self.item) if ctype else self
+
+        return lazy_operation
+
+    return decorator if isinstance(mixed, str) else decorator(mixed, ctype=None)
+
+
+def factory_operation(m):
+    def lazy_operation(self, *config):
+        @functools.wraps(m)
+        def actual_operation(*args, **kwargs):
+            return m(self, *config, *args, **kwargs)
+
+        self.operations.append(FactoryOperation(self, actual_operation))
+        return self
+
+    return lazy_operation
+
+
+class Cursor():
+    _type = None
+
+    def __init__(self, factory, item):
+        self.factory = factory
+        self.item = item
+
+    @operation('dict')
+    def dict(self, x):
+        return x if isinstance(x, dict) else dict(x)
+
+    @operation('int')
+    def int(self):
+        pass
+
+    @operation('str')
+    def str(self, x):
+        return x if isinstance(x, str) else str(x)
+
+    @operation('list')
+    def list(self):
+        pass
+
+    @operation('tuple')
+    def tuple(self):
+        pass
+
+    def __getattr__(self, item):
+        """
+        Fallback to type methods if they exist, for example StrCursor.upper will use str.upper if not overriden, etc.
+
+        :param item:
+        """
+        if self._type and item in self._type.__dict__:
+            method = self._type.__dict__[item]
+
+            @operation
+            @functools.wraps(method)
+            def _operation(self, x, *args, **kwargs):
+                return method(x, *args, **kwargs)
+
+            setattr(self, item, partial(_operation, self))
+            return getattr(self, item)
+
+        raise AttributeError('Unknown operation {}.{}().'.format(type(self).__name__, item, ))
+
+
+CURSOR_TYPES['default'] = Cursor
+
+
+class DictCursor(Cursor):
+    _type = dict
+
+    @operation('default')
+    def get(self, x, path):
+        return x.get(path)
+
+    @operation
+    def map_keys(self, x, mapping):
+        return {mapping.get(k): v for k, v in x.items()}
+
+
+CURSOR_TYPES['dict'] = DictCursor
+
+
+class StringCursor(Cursor):
+    _type = str
+
+
+CURSOR_TYPES['str'] = StringCursor
+
+
+class Factory(Configurable):
+    setup = Method()
+
+    def __init__(self):
+        self.default_cursor_type = 'default'
+        self.operations = []
+        self.setup()
+
+    @factory_operation
+    def move(self, _from, _to, *args, **kwargs):
+        if _from == _to:
+            return args, kwargs
+
+        if _isarg(_from):
+            value = args[_from]
+            args = args[:_from] + args[_from + 1:]
+        elif _iskwarg(_from):
+            value = kwargs[_from]
+            kwargs = {k: v for k, v in kwargs if k != _from}
+        else:
+            raise RuntimeError('Houston, we have a problem...')
+
+        if _isarg(_to):
+            return (*args[:_to], value, *args[_to + 1:]), kwargs
+        elif _iskwarg(_to):
+            return args, {**kwargs, _to: value}
+        else:
+            raise RuntimeError('Houston, we have a problem...')
+
+    def __call__(self, *args, **kwargs):
+        # print('factory call on', args, kwargs)
+        for operation in self.operations:
+            args, kwargs = operation.apply(*args, **kwargs)
+            # print(' ... after', operation, 'got', args, kwargs)
+        return Bag(*args, **kwargs)
+
+    def __getitem__(self, item):
+        return CURSOR_TYPES[self.default_cursor_type](self, item)
+
+
+if __name__ == '__main__':
+    f = Factory()
+
+    f[0].dict().map_keys({'foo': 'F00'})
+    f['foo'].str().upper()
+
+    print('operations:', f.operations)
+    print(f({'foo': 'bisou'}, foo='blah'))
+
+'''
+specs:
+
+- rename keys of an input dict (in args, or kwargs) using a translation map.
+
+
+f = Factory()
+
+f[0]
+f['xxx'] = 
+    
+f[0].dict().get('foo.bar').move_to('foo.baz').apply(str.upper)
+f[0].get('foo.*').items().map(str.lower)
+
+f['foo'].keys_map({
+    'a': 'b'
+})
+
+'''
diff --git a/bonobo/nodes/io/json.py b/bonobo/nodes/io/json.py
index fdb49b8..91c8f34 100644
--- a/bonobo/nodes/io/json.py
+++ b/bonobo/nodes/io/json.py
@@ -1,5 +1,7 @@
 import json
+from itertools import starmap
 
+from bonobo.structs.bags import Bag
 from bonobo.config.processors import ContextProcessor
 from .file import FileWriter, FileReader
 
@@ -21,6 +23,13 @@ class JsonReader(JsonHandler, FileReader):
             yield line
 
 
+class JsonDictReader(JsonReader):
+    """ not api, don't use or expect breakage. """
+
+    def read(self, fs, file):
+        yield from starmap(Bag, self.loader(file).items())
+
+
 class JsonWriter(JsonHandler, FileWriter):
     @ContextProcessor
     def envelope(self, context, fs, file, lineno):

From be844c3ed788a19b04aeb2a8fec79ad36854556a Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Thu, 25 May 2017 16:41:01 +0200
Subject: [PATCH 02/16] WIP GRAPHVIZ

---
 Makefile                 |   2 +-
 Projectfile              |   1 +
 bin/imgcat               | 112 +++++++++++++++++++++++++++++++++++++++
 bin/test_graph           |   1 +
 bonobo/commands/graph.py |  22 ++++++++
 bonobo/commands/run.py   |  27 +++++-----
 bonobo/constants.py      |   2 +
 bonobo/structs/graphs.py |   2 +
 bonobo/util/graphviz.py  |   9 ++++
 setup.py                 |   1 +
 10 files changed, 165 insertions(+), 14 deletions(-)
 create mode 100755 bin/imgcat
 create mode 100644 bin/test_graph
 create mode 100644 bonobo/commands/graph.py
 create mode 100644 bonobo/util/graphviz.py

diff --git a/Makefile b/Makefile
index 6db0f7c..6c5443d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 # This file has been auto-generated.
 # All changes will be lost, see Projectfile.
 #
-# Updated at 2017-05-03 18:02:59.359160
+# Updated at 2017-05-08 11:34:30.472553
 
 PACKAGE ?= bonobo
 PYTHON ?= $(shell which python)
diff --git a/Projectfile b/Projectfile
index 47844ee..95e5afa 100644
--- a/Projectfile
+++ b/Projectfile
@@ -60,6 +60,7 @@ entry_points = {
     ],
     'bonobo.commands': [
         'init = bonobo.commands.init:register',
+        'graph = bonobo.commands.graph:register',
         'run = bonobo.commands.run:register',
         'version = bonobo.commands.version:register',
     ],
diff --git a/bin/imgcat b/bin/imgcat
new file mode 100755
index 0000000..001d2b8
--- /dev/null
+++ b/bin/imgcat
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# tmux requires unrecognized OSC sequences to be wrapped with DCS tmux;
+# <sequence> ST, and for all ESCs in <sequence> to be replaced with ESC ESC. It
+# only accepts ESC backslash for ST.
+function print_osc() {
+    if [[ $TERM == screen* ]] ; then
+        printf "\033Ptmux;\033\033]"
+    else
+        printf "\033]"
+    fi
+}
+
+# More of the tmux workaround described above.
+function print_st() {
+    if [[ $TERM == screen* ]] ; then
+        printf "\a\033\\"
+    else
+        printf "\a"
+    fi
+}
+
+# print_image filename inline base64contents print_filename
+#   filename: Filename to convey to client
+#   inline: 0 or 1
+#   base64contents: Base64-encoded contents
+#   print_filename: If non-empty, print the filename 
+#                   before outputting the image
+function print_image() {
+    print_osc
+    printf '1337;File='
+    if [[ -n "$1" ]]; then
+      printf 'name='`printf "%s" "$1" | base64`";"
+    fi
+
+    VERSION=$(base64 --version 2>&1)
+    if [[ "$VERSION" =~ fourmilab ]]; then
+      BASE64ARG=-d
+    elif [[ "$VERSION" =~ GNU ]]; then
+      BASE64ARG=-di
+    else
+      BASE64ARG=-D
+    fi
+
+    printf "%s" "$3" | base64 $BASE64ARG | wc -c | awk '{printf "size=%d",$1}'
+    printf ";inline=$2"
+    printf ":"
+    printf "%s" "$3"
+    print_st
+    printf '\n'
+    if [[ -n "$4" ]]; then
+      echo $1
+    fi
+}
+
+function error() {
+    echo "ERROR: $*" 1>&2
+}
+
+function show_help() {
+    echo "Usage: imgcat [-p] filename ..." 1>& 2
+    echo "   or: cat filename | imgcat" 1>& 2
+}
+
+## Main
+
+if [ -t 0 ]; then
+    has_stdin=f
+else
+    has_stdin=t
+fi
+
+# Show help if no arguments and no stdin.
+if [ $has_stdin = f -a $# -eq 0 ]; then
+    show_help
+    exit
+fi
+
+# Look for command line flags.
+while [ $# -gt 0 ]; do
+    case "$1" in
+    -h|--h|--help)
+        show_help
+        exit
+        ;;
+    -p|--p|--print)
+        print_filename=1
+        ;;
+    -*)
+        error "Unknown option flag: $1"
+        show_help
+        exit 1
+      ;;
+    *)
+        if [ -r "$1" ] ; then
+            has_stdin=f
+            print_image "$1" 1 "$(base64 < "$1")" "$print_filename"
+        else
+            error "imgcat: $1: No such file or directory"
+            exit 2
+        fi
+        ;;
+    esac
+    shift
+done
+
+# Read and print stdin
+if [ $has_stdin = t ]; then
+    print_image "" 1 "$(cat | base64)" ""
+fi
+
+exit 0
diff --git a/bin/test_graph b/bin/test_graph
new file mode 100644
index 0000000..1e5fd85
--- /dev/null
+++ b/bin/test_graph
@@ -0,0 +1 @@
+bonobo graph bonobo/examples/tutorials/tut02_03_writeasmap.py | dot -otest.png -Tpng && bin/imgcat test.png 
diff --git a/bonobo/commands/graph.py b/bonobo/commands/graph.py
new file mode 100644
index 0000000..b8bbdf9
--- /dev/null
+++ b/bonobo/commands/graph.py
@@ -0,0 +1,22 @@
+import json
+
+from bonobo.util.objects import get_name
+from bonobo.commands.run import read_file
+from bonobo.constants import BEGIN
+
+
+def execute(file):
+    graph, plugins, services = read_file(file)
+
+    print('digraph {')
+    print('  rankdir = LR;')
+    print('  "BEGIN" [shape="point"];')
+    for i in graph.outputs_of(BEGIN):
+        print('  "BEGIN" -> ' + json.dumps(get_name(graph.nodes[i])) + ';')
+    print('}')
+
+
+def register(parser):
+    import argparse
+    parser.add_argument('file', type=argparse.FileType())
+    return execute
diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py
index b7872e2..60123c8 100644
--- a/bonobo/commands/run.py
+++ b/bonobo/commands/run.py
@@ -1,11 +1,7 @@
-import argparse
-
 import os
 
 import bonobo
-
-DEFAULT_SERVICES_FILENAME = '_services.py'
-DEFAULT_SERVICES_ATTR = 'get_services'
+from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME
 
 
 def get_default_services(filename, services=None):
@@ -29,7 +25,7 @@ def get_default_services(filename, services=None):
     return services or {}
 
 
-def execute(file, quiet=False):
+def read_file(file):
     with file:
         code = compile(file.read(), file.name, 'exec')
 
@@ -56,19 +52,24 @@ def execute(file, quiet=False):
     ).format(len(graphs))
 
     graph = list(graphs.values())[0]
+    plugins = []
+    services = get_default_services(
+        file.name, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
+    )
+
+    return graph, plugins, services
+
+
+def execute(file, quiet=False):
+    graph, plugins, services = read_file(file)
 
     # todo if console and not quiet, then add the console plugin
     # todo when better console plugin, add it if console and just disable display
-    return bonobo.run(
-        graph,
-        plugins=[],
-        services=get_default_services(
-            file.name, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
-        )
-    )
+    return bonobo.run(graph, plugins=plugins, services=services)
 
 
 def register(parser):
+    import argparse
     parser.add_argument('file', type=argparse.FileType())
     parser.add_argument('--quiet', action='store_true')
     return execute
diff --git a/bonobo/constants.py b/bonobo/constants.py
index d567229..4187197 100644
--- a/bonobo/constants.py
+++ b/bonobo/constants.py
@@ -4,3 +4,5 @@ BEGIN = Token('Begin')
 END = Token('End')
 INHERIT_INPUT = Token('InheritInput')
 NOT_MODIFIED = Token('NotModified')
+DEFAULT_SERVICES_FILENAME = '_services.py'
+DEFAULT_SERVICES_ATTR = 'get_services'
\ No newline at end of file
diff --git a/bonobo/structs/graphs.py b/bonobo/structs/graphs.py
index d2d755e..c68186b 100644
--- a/bonobo/structs/graphs.py
+++ b/bonobo/structs/graphs.py
@@ -29,3 +29,5 @@ class Graph:
 
     def __len__(self):
         return len(self.nodes)
+
+
diff --git a/bonobo/util/graphviz.py b/bonobo/util/graphviz.py
new file mode 100644
index 0000000..fa88974
--- /dev/null
+++ b/bonobo/util/graphviz.py
@@ -0,0 +1,9 @@
+
+def render_as_dot(graph):
+    """
+    
+    :param bonobo.Graph graph: 
+    :return: str
+    """
+
+    pass
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 4cd8d82..1153f35 100644
--- a/setup.py
+++ b/setup.py
@@ -65,6 +65,7 @@ setup(
     entry_points={
         'bonobo.commands': [
             'init = bonobo.commands.init:register',
+            'graph = bonobo.commands.graph:register',
             'run = bonobo.commands.run:register',
             'version = bonobo.commands.version:register'
         ],

From 3e961776e3b8a7b0b4c0fe1c9be27c1cfb24efbf Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sun, 16 Jul 2017 18:45:49 +0200
Subject: [PATCH 03/16] [cli] First draft implementation of "convert" command,
 which builds a simple graph of reader+writer and executes it.

---
 Makefile                   |  2 +-
 Projectfile                |  1 +
 bonobo/commands/convert.py | 61 ++++++++++++++++++++++++++++++++++++++
 setup.py                   |  4 +--
 4 files changed, 65 insertions(+), 3 deletions(-)
 create mode 100644 bonobo/commands/convert.py

diff --git a/Makefile b/Makefile
index 10094af..fcdc7e2 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 # This file has been auto-generated.
 # All changes will be lost, see Projectfile.
 #
-# Updated at 2017-07-04 10:50:55.775681
+# Updated at 2017-07-16 18:44:56.169119
 
 PACKAGE ?= bonobo
 PYTHON ?= $(shell which python)
diff --git a/Projectfile b/Projectfile
index 6fe6c2b..117dd76 100644
--- a/Projectfile
+++ b/Projectfile
@@ -29,6 +29,7 @@ python.setup(
             'bonobo = bonobo.commands:entrypoint',
         ],
         'bonobo.commands': [
+            'convert = bonobo.commands.convert:register',
             'init = bonobo.commands.init:register',
             'run = bonobo.commands.run:register',
             'version = bonobo.commands.version:register',
diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py
new file mode 100644
index 0000000..c29f31c
--- /dev/null
+++ b/bonobo/commands/convert.py
@@ -0,0 +1,61 @@
+import mimetypes
+
+import bonobo
+
+SHORTCUTS = {
+    'plain': 'text/plain',
+    'txt': 'text/plain',
+    'text': 'text/plain',
+    'csv': 'text/csv',
+    'json': 'application/json',
+}
+
+REGISTRY = {
+    'text/plain': (bonobo.FileReader, bonobo.FileWriter),
+    'text/csv': (bonobo.CsvReader, bonobo.CsvWriter),
+    'application/json': (bonobo.JsonReader, bonobo.JsonWriter),
+}
+
+
+def resolve_factory(name, filename, factory_type):
+    """
+    Try to resolve which transformation factory to use for this filename. User eventually provided a name, which has
+    priority, otherwise we try to detect it using the mimetype detection on filename.
+
+    """
+    if name is None:
+        name = mimetypes.guess_type(filename)[0]
+
+    if name in SHORTCUTS:
+        name = SHORTCUTS[name]
+
+    if not name in REGISTRY:
+        raise RuntimeError('Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} <format>.'.format(name=name, filename=filename, factory_type=factory_type, opt=factory_type[0]))
+
+    if factory_type == 'reader':
+        return REGISTRY[name][0]
+    elif factory_type == 'writer':
+        return REGISTRY[name][1]
+    else:
+        raise ValueError('Invalid factory type.')
+
+def execute(input, output, reader=None, reader_options=None, writer=None, writer_options=None, options=None):
+    reader = resolve_factory(reader, input, 'reader')(input)
+    writer = resolve_factory(writer, output, 'writer')(output)
+
+    graph = bonobo.Graph()
+    graph.add_chain(reader, writer)
+
+    return bonobo.run(graph, services={
+            'fs': bonobo.open_fs(),
+        })
+
+def register(parser):
+    parser.add_argument('input')
+    parser.add_argument('output')
+    parser.add_argument('--reader', '-r')
+    parser.add_argument('--writer', '-w')
+    parser.add_argument('--reader-option', '-ro', dest='reader_options')
+    parser.add_argument('--writer-option', '-wo', dest='writer_options')
+    parser.add_argument('--option', '-o', dest='options')
+    return execute
diff --git a/setup.py b/setup.py
index 89c9ccd..a925811 100644
--- a/setup.py
+++ b/setup.py
@@ -67,8 +67,8 @@ setup(
     },
     entry_points={
         'bonobo.commands': [
-            'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register',
-            'version = bonobo.commands.version:register'
+            'convert = bonobo.commands.convert:register', 'init = bonobo.commands.init:register',
+            'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register'
         ],
         'console_scripts': ['bonobo = bonobo.commands:entrypoint']
     },

From c881ca106155a8777af6bd69aaf007680c7df376 Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sun, 16 Jul 2017 18:51:17 +0200
Subject: [PATCH 04/16] [cli] Convert: adds pickle format, comment out unused
 arguments, for now.

---
 bonobo/commands/convert.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py
index c29f31c..90bd57b 100644
--- a/bonobo/commands/convert.py
+++ b/bonobo/commands/convert.py
@@ -11,9 +11,10 @@ SHORTCUTS = {
 }
 
 REGISTRY = {
-    'text/plain': (bonobo.FileReader, bonobo.FileWriter),
-    'text/csv': (bonobo.CsvReader, bonobo.CsvWriter),
     'application/json': (bonobo.JsonReader, bonobo.JsonWriter),
+    'pickle': (bonobo.PickleReader, bonobo.PickleWriter),
+    'text/csv': (bonobo.CsvReader, bonobo.CsvWriter),
+    'text/plain': (bonobo.FileReader, bonobo.FileWriter),
 }
 
 
@@ -55,7 +56,7 @@ def register(parser):
     parser.add_argument('output')
     parser.add_argument('--reader', '-r')
     parser.add_argument('--writer', '-w')
-    parser.add_argument('--reader-option', '-ro', dest='reader_options')
-    parser.add_argument('--writer-option', '-wo', dest='writer_options')
-    parser.add_argument('--option', '-o', dest='options')
+    # parser.add_argument('--reader-option', '-ro', dest='reader_options')
+    # parser.add_argument('--writer-option', '-wo', dest='writer_options')
+    # parser.add_argument('--option', '-o', dest='options')
     return execute

From fcc24f4badba3843a27971ade82cbea136d50478 Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sun, 16 Jul 2017 18:58:56 +0200
Subject: [PATCH 05/16] Restore removed examples that were used in tests.

---
 bonobo/examples/types/__init__.py |  7 +++++
 bonobo/examples/types/bags.py     | 41 +++++++++++++++++++++++++++++
 bonobo/examples/types/dicts.py    | 43 +++++++++++++++++++++++++++++++
 bonobo/examples/types/strings.py  | 39 ++++++++++++++++++++++++++++
 4 files changed, 130 insertions(+)
 create mode 100644 bonobo/examples/types/__init__.py
 create mode 100644 bonobo/examples/types/bags.py
 create mode 100644 bonobo/examples/types/dicts.py
 create mode 100644 bonobo/examples/types/strings.py

diff --git a/bonobo/examples/types/__init__.py b/bonobo/examples/types/__init__.py
new file mode 100644
index 0000000..a2c0ceb
--- /dev/null
+++ b/bonobo/examples/types/__init__.py
@@ -0,0 +1,7 @@
+from . import bags, dicts, strings
+
+__all__ = [
+    'bags',
+    'dicts',
+    'strings',
+]
\ No newline at end of file
diff --git a/bonobo/examples/types/bags.py b/bonobo/examples/types/bags.py
new file mode 100644
index 0000000..2bfe5de
--- /dev/null
+++ b/bonobo/examples/types/bags.py
@@ -0,0 +1,41 @@
+"""
+Example on how to use :class:`bonobo.Bag` instances to pass flexible args/kwargs to the next callable.
+
+.. graphviz::
+
+    digraph {
+        rankdir = LR;
+        stylesheet = "../_static/graphs.css";
+
+        BEGIN [shape="point"];
+        BEGIN -> "extract()" -> "transform(...)" -> "load(...)";
+    }
+
+"""
+
+from random import randint
+
+from bonobo import Bag, Graph
+
+
+def extract():
+    yield Bag(topic='foo')
+    yield Bag(topic='bar')
+    yield Bag(topic='baz')
+
+
+def transform(topic: str):
+    return Bag.inherit(title=topic.title(), rand=randint(10, 99))
+
+
+def load(topic: str, title: str, rand: int):
+    print('{} ({}) wait={}'.format(title, topic, rand))
+
+
+graph = Graph()
+graph.add_chain(extract, transform, load)
+
+if __name__ == '__main__':
+    from bonobo import run
+
+    run(graph)
diff --git a/bonobo/examples/types/dicts.py b/bonobo/examples/types/dicts.py
new file mode 100644
index 0000000..fde4b08
--- /dev/null
+++ b/bonobo/examples/types/dicts.py
@@ -0,0 +1,43 @@
+"""
+Example on how to use symple python dictionaries to communicate between transformations.
+
+.. graphviz::
+
+    digraph {
+        rankdir = LR;
+        stylesheet = "../_static/graphs.css";
+
+        BEGIN [shape="point"];
+        BEGIN -> "extract()" -> "transform(row: dict)" -> "load(row: dict)";
+    }
+
+"""
+
+from random import randint
+
+from bonobo import Graph
+
+
+def extract():
+    yield {'topic': 'foo'}
+    yield {'topic': 'bar'}
+    yield {'topic': 'baz'}
+
+
+def transform(row: dict):
+    return {
+        'topic': row['topic'].title(),
+        'randint': randint(10, 99),
+    }
+
+
+def load(row: dict):
+    print(row)
+
+
+graph = Graph(extract, transform, load)
+
+if __name__ == '__main__':
+    from bonobo import run
+
+    run(graph)
diff --git a/bonobo/examples/types/strings.py b/bonobo/examples/types/strings.py
new file mode 100644
index 0000000..1903151
--- /dev/null
+++ b/bonobo/examples/types/strings.py
@@ -0,0 +1,39 @@
+"""
+Example on how to use symple python strings to communicate between transformations.
+
+.. graphviz::
+
+    digraph {
+        rankdir = LR;
+        stylesheet = "../_static/graphs.css";
+
+        BEGIN [shape="point"];
+        BEGIN -> "extract()" -> "transform(s: str)" -> "load(s: str)";
+    }
+
+"""
+from random import randint
+
+from bonobo import Graph
+
+
+def extract():
+    yield 'foo'
+    yield 'bar'
+    yield 'baz'
+
+
+def transform(s: str):
+    return '{} ({})'.format(s.title(), randint(10, 99))
+
+
+def load(s: str):
+    print(s)
+
+
+graph = Graph(extract, transform, load)
+
+if __name__ == '__main__':
+    from bonobo import run
+
+    run(graph)

From 966628e15697d20205e1f5b615b48e5e1257df73 Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Mon, 18 Sep 2017 17:16:18 +0200
Subject: [PATCH 06/16] Implements graphviz output after a graph inspection.

---
 bonobo/commands/graph.py | 22 ++++++++++++++++------
 bonobo/commands/run.py   |  8 +++++++-
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/bonobo/commands/graph.py b/bonobo/commands/graph.py
index b8bbdf9..7afa8de 100644
--- a/bonobo/commands/graph.py
+++ b/bonobo/commands/graph.py
@@ -1,22 +1,32 @@
 import json
 
+import itertools
+
 from bonobo.util.objects import get_name
-from bonobo.commands.run import read_file
+from bonobo.commands.run import read, register_generic_run_arguments
 from bonobo.constants import BEGIN
 
 
-def execute(file):
-    graph, plugins, services = read_file(file)
+def execute(filename, module, install=False, quiet=False, verbose=False):
+    graph, plugins, services = read(filename, module, install, quiet, verbose)
 
     print('digraph {')
     print('  rankdir = LR;')
     print('  "BEGIN" [shape="point"];')
+
     for i in graph.outputs_of(BEGIN):
-        print('  "BEGIN" -> ' + json.dumps(get_name(graph.nodes[i])) + ';')
+        print('  "BEGIN" -> ' + json.dumps(get_name(graph[i])) + ';')
+
+    for ix in graph.topologically_sorted_indexes:
+        for iy in graph.outputs_of(ix):
+            print('  {} -> {};'.format(
+                json.dumps(get_name(graph[ix])),
+                json.dumps(get_name(graph[iy]))
+            ))
+
     print('}')
 
 
 def register(parser):
-    import argparse
-    parser.add_argument('file', type=argparse.FileType())
+    register_generic_run_arguments(parser)
     return execute
diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py
index 58cd82d..604fc39 100644
--- a/bonobo/commands/run.py
+++ b/bonobo/commands/run.py
@@ -40,7 +40,7 @@ def _install_requirements(requirements):
     importlib.reload(site)
 
 
-def execute(filename, module, install=False, quiet=False, verbose=False):
+def read(filename, module, install=False, quiet=False, verbose=False):
     import runpy
     from bonobo import Graph, settings
 
@@ -86,6 +86,12 @@ def execute(filename, module, install=False, quiet=False, verbose=False):
         filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
     )
 
+    return graph, plugins, services
+
+
+def execute(filename, module, install=False, quiet=False, verbose=False):
+    graph, plugins, services = read(filename, module, install, quiet, verbose)
+
     return bonobo.run(
         graph,
         plugins=plugins,

From 8b9dac50eca8d2737e377c225fdd5e4c4dc948c8 Mon Sep 17 00:00:00 2001
From: cwandrews <cwandrews.oh@gmail.com>
Date: Mon, 18 Sep 2017 15:24:27 -0400
Subject: [PATCH 07/16] Added optional passing of one or multiple environment
 variables via --env flag to the bonobo cli.

---
 bonobo/commands/run.py       | 12 +++++++++++-
 tests/test_commands.py       | 27 +++++++++++++++++++++++++++
 tests/util/get_passed_env.py | 22 ++++++++++++++++++++++
 3 files changed, 60 insertions(+), 1 deletion(-)
 create mode 100644 tests/util/get_passed_env.py

diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py
index fb93e77..c9bb39e 100644
--- a/bonobo/commands/run.py
+++ b/bonobo/commands/run.py
@@ -40,7 +40,10 @@ def _install_requirements(requirements):
     importlib.reload(site)
 
 
-def execute(filename, module, install=False, quiet=False, verbose=False):
+def execute(filename, module, install=False, quiet=False, verbose=False,
+            env=None):
+    import re
+
     import runpy
     from bonobo import Graph, run, settings
 
@@ -50,6 +53,12 @@ def execute(filename, module, install=False, quiet=False, verbose=False):
     if verbose:
         settings.DEBUG.set(True)
 
+    if env:
+        quote_killer = re.compile('["\']')
+        for e in env:
+            var_name, var_value = e.split('=')
+            os.environ[var_name] = quote_killer.sub('', var_value)
+
     if filename:
         if os.path.isdir(filename):
             if install:
@@ -106,4 +115,5 @@ def register(parser):
     verbosity_group.add_argument('--quiet', '-q', action='store_true')
     verbosity_group.add_argument('--verbose', '-v', action='store_true')
     parser.add_argument('--install', '-I', action='store_true')
+    parser.add_argument('--env', '-e', action='append')
     return execute
diff --git a/tests/test_commands.py b/tests/test_commands.py
index daf245f..cff9e38 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -3,6 +3,7 @@ import runpy
 import sys
 from unittest.mock import patch
 
+import pathlib
 import pkg_resources
 import pytest
 
@@ -96,3 +97,29 @@ def test_version(runner, capsys):
     out = out.strip()
     assert out.startswith('bonobo ')
     assert __version__ in out
+
+
+@all_runners
+def test_run_with_env(runner, capsys):
+    runner('run', '--quiet',
+           str(pathlib.Path(os.path.dirname(__file__),
+                            'util', 'get_passed_env.py')),
+           '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
+           '--env', "ENV_TEST_STRING='my_test_string'")
+    out, err = capsys.readouterr()
+    out = out.split('\n')
+    assert out[0] == 'cwandrews'
+    assert out[1] == '123'
+    assert out[2] == 'my_test_string'
+
+
+@all_runners
+def test_run_module_with_env(runner, capsys):
+    runner('run', '--quiet', '-m', 'tests.util.get_passed_env',
+           '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
+           '--env', "ENV_TEST_STRING='my_test_string'")
+    out, err = capsys.readouterr()
+    out = out.split('\n')
+    assert out[0] == 'cwandrews'
+    assert out[1] == '123'
+    assert out[2] == 'my_test_string'
diff --git a/tests/util/get_passed_env.py b/tests/util/get_passed_env.py
new file mode 100644
index 0000000..d9c4ba6
--- /dev/null
+++ b/tests/util/get_passed_env.py
@@ -0,0 +1,22 @@
+import os
+
+from bonobo import Graph
+
+
+def extract():
+    env_test_user = os.getenv('ENV_TEST_USER')
+    env_test_number = os.getenv('ENV_TEST_NUMBER')
+    env_test_string = os.getenv('ENV_TEST_STRING')
+    return env_test_user, env_test_number, env_test_string
+
+
+def load(s: str):
+    print(s)
+
+
+graph = Graph(extract, load)
+
+if __name__ == '__main__':
+    from bonobo import run
+
+    run(graph)

From 7ca3369f7141246823e0fe2dabb0d98e5d91af5a Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sat, 30 Sep 2017 11:01:53 +0200
Subject: [PATCH 08/16] Rename "bonobo graph" to "bonobo inspect". For now,
 graphviz is default but there will probably be a humand default in the
 future, with graphviz source generation set if --graph (or -g) flag is
 passed.

---
 Makefile                   |  2 +-
 Projectfile                |  2 +-
 bonobo/commands/graph.py   | 32 --------------------------------
 bonobo/commands/inspect.py | 33 +++++++++++++++++++++++++++++++++
 bonobo/commands/run.py     |  8 ++------
 bonobo/util/graphviz.py    |  3 +--
 setup.py                   |  2 +-
 tests/test_commands.py     | 17 +++++++++--------
 8 files changed, 48 insertions(+), 51 deletions(-)
 delete mode 100644 bonobo/commands/graph.py
 create mode 100644 bonobo/commands/inspect.py

diff --git a/Makefile b/Makefile
index 1e617cb..aac0c92 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 # This file has been auto-generated.
 # All changes will be lost, see Projectfile.
 #
-# Updated at 2017-09-30 10:24:51.699716
+# Updated at 2017-09-30 10:57:00.855477
 
 PACKAGE ?= bonobo
 PYTHON ?= $(shell which python)
diff --git a/Projectfile b/Projectfile
index eea8cc5..8e848b3 100644
--- a/Projectfile
+++ b/Projectfile
@@ -30,7 +30,7 @@ python.setup(
         ],
         'bonobo.commands': [
             'init = bonobo.commands.init:register',
-            'graph = bonobo.commands.graph:register',
+            'inspect = bonobo.commands.inspect:register',
             'run = bonobo.commands.run:register',
             'version = bonobo.commands.version:register',
         ],
diff --git a/bonobo/commands/graph.py b/bonobo/commands/graph.py
deleted file mode 100644
index 7afa8de..0000000
--- a/bonobo/commands/graph.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import json
-
-import itertools
-
-from bonobo.util.objects import get_name
-from bonobo.commands.run import read, register_generic_run_arguments
-from bonobo.constants import BEGIN
-
-
-def execute(filename, module, install=False, quiet=False, verbose=False):
-    graph, plugins, services = read(filename, module, install, quiet, verbose)
-
-    print('digraph {')
-    print('  rankdir = LR;')
-    print('  "BEGIN" [shape="point"];')
-
-    for i in graph.outputs_of(BEGIN):
-        print('  "BEGIN" -> ' + json.dumps(get_name(graph[i])) + ';')
-
-    for ix in graph.topologically_sorted_indexes:
-        for iy in graph.outputs_of(ix):
-            print('  {} -> {};'.format(
-                json.dumps(get_name(graph[ix])),
-                json.dumps(get_name(graph[iy]))
-            ))
-
-    print('}')
-
-
-def register(parser):
-    register_generic_run_arguments(parser)
-    return execute
diff --git a/bonobo/commands/inspect.py b/bonobo/commands/inspect.py
new file mode 100644
index 0000000..83b770e
--- /dev/null
+++ b/bonobo/commands/inspect.py
@@ -0,0 +1,33 @@
+import json
+
+from bonobo.commands.run import read, register_generic_run_arguments
+from bonobo.constants import BEGIN
+from bonobo.util.objects import get_name
+
+OUTPUT_GRAPHVIZ = 'graphviz'
+
+def execute(*, output, **kwargs):
+    graph, plugins, services = read(**kwargs)
+
+    if output == OUTPUT_GRAPHVIZ:
+        print('digraph {')
+        print('  rankdir = LR;')
+        print('  "BEGIN" [shape="point"];')
+
+        for i in graph.outputs_of(BEGIN):
+            print('  "BEGIN" -> ' + json.dumps(get_name(graph[i])) + ';')
+
+        for ix in graph.topologically_sorted_indexes:
+            for iy in graph.outputs_of(ix):
+                print('  {} -> {};'.format(json.dumps(get_name(graph[ix])), json.dumps(get_name(graph[iy]))))
+
+        print('}')
+    else:
+        raise NotImplementedError('Output type not implemented.')
+
+
+def register(parser):
+    register_generic_run_arguments(parser)
+    parser.add_argument('--graph', '-g', dest='output', action='store_const', const=OUTPUT_GRAPHVIZ)
+    parser.set_defaults(output=OUTPUT_GRAPHVIZ)
+    return execute
diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py
index 27a2329..2204a3b 100644
--- a/bonobo/commands/run.py
+++ b/bonobo/commands/run.py
@@ -3,7 +3,7 @@ import os
 import bonobo
 from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME
 
-DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py',)
+DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', )
 DEFAULT_GRAPH_ATTR = 'get_graph'
 
 
@@ -99,11 +99,7 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None):
 def execute(filename, module, install=False, quiet=False, verbose=False, env=None):
     graph, plugins, services = read(filename, module, install, quiet, verbose, env)
 
-    return bonobo.run(
-        graph,
-        plugins=plugins,
-        services=services
-    )
+    return bonobo.run(graph, plugins=plugins, services=services)
 
 
 def register_generic_run_arguments(parser, required=True):
diff --git a/bonobo/util/graphviz.py b/bonobo/util/graphviz.py
index fa88974..588e374 100644
--- a/bonobo/util/graphviz.py
+++ b/bonobo/util/graphviz.py
@@ -1,4 +1,3 @@
-
 def render_as_dot(graph):
     """
     
@@ -6,4 +5,4 @@ def render_as_dot(graph):
     :return: str
     """
 
-    pass
\ No newline at end of file
+    pass
diff --git a/setup.py b/setup.py
index 0abee00..08b84e0 100644
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,7 @@ setup(
     },
     entry_points={
         'bonobo.commands': [
-            'init = bonobo.commands.init:register', 'graph = bonobo.commands.graph:register',
+            'init = bonobo.commands.init:register', 'inspect = bonobo.commands.inspect:register',
             'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register'
         ],
         'console_scripts': ['bonobo = bonobo.commands:entrypoint']
diff --git a/tests/test_commands.py b/tests/test_commands.py
index cff9e38..730bc0b 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -101,11 +101,11 @@ def test_version(runner, capsys):
 
 @all_runners
 def test_run_with_env(runner, capsys):
-    runner('run', '--quiet',
-           str(pathlib.Path(os.path.dirname(__file__),
-                            'util', 'get_passed_env.py')),
-           '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
-           '--env', "ENV_TEST_STRING='my_test_string'")
+    runner(
+        'run', '--quiet',
+        str(pathlib.Path(os.path.dirname(__file__), 'util', 'get_passed_env.py')), '--env', 'ENV_TEST_NUMBER=123',
+        '--env', 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
+    )
     out, err = capsys.readouterr()
     out = out.split('\n')
     assert out[0] == 'cwandrews'
@@ -115,9 +115,10 @@ def test_run_with_env(runner, capsys):
 
 @all_runners
 def test_run_module_with_env(runner, capsys):
-    runner('run', '--quiet', '-m', 'tests.util.get_passed_env',
-           '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
-           '--env', "ENV_TEST_STRING='my_test_string'")
+    runner(
+        'run', '--quiet', '-m', 'tests.util.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env',
+        'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
+    )
     out, err = capsys.readouterr()
     out = out.split('\n')
     assert out[0] == 'cwandrews'

From b49ccaa7a77aad78a4560c8469c1e09d37a74abb Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sat, 30 Sep 2017 11:26:22 +0200
Subject: [PATCH 09/16] Formating.

---
 bonobo/commands/inspect.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bonobo/commands/inspect.py b/bonobo/commands/inspect.py
index 83b770e..bb82704 100644
--- a/bonobo/commands/inspect.py
+++ b/bonobo/commands/inspect.py
@@ -6,6 +6,7 @@ from bonobo.util.objects import get_name
 
 OUTPUT_GRAPHVIZ = 'graphviz'
 
+
 def execute(*, output, **kwargs):
     graph, plugins, services = read(**kwargs)
 

From c6cde93f4ee8afe0256b8b46e6f331375551921b Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sat, 30 Sep 2017 11:38:36 +0200
Subject: [PATCH 10/16] [cli] small refactoring in bonobo convert to use
 shortcuts as extensions if nothing else matched.

---
 bonobo/commands/convert.py | 42 ++++++++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py
index 90bd57b..3c26724 100644
--- a/bonobo/commands/convert.py
+++ b/bonobo/commands/convert.py
@@ -1,13 +1,15 @@
 import mimetypes
+import os
 
 import bonobo
 
 SHORTCUTS = {
-    'plain': 'text/plain',
-    'txt': 'text/plain',
-    'text': 'text/plain',
     'csv': 'text/csv',
     'json': 'application/json',
+    'pickle': 'pickle',
+    'plain': 'text/plain',
+    'text': 'text/plain',
+    'txt': 'text/plain',
 }
 
 REGISTRY = {
@@ -17,6 +19,9 @@ REGISTRY = {
     'text/plain': (bonobo.FileReader, bonobo.FileWriter),
 }
 
+READER = 'reader'
+WRITER = 'writer'
+
 
 def resolve_factory(name, filename, factory_type):
     """
@@ -30,32 +35,43 @@ def resolve_factory(name, filename, factory_type):
     if name in SHORTCUTS:
         name = SHORTCUTS[name]
 
-    if not name in REGISTRY:
-        raise RuntimeError('Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} <format>.'.format(name=name, filename=filename, factory_type=factory_type, opt=factory_type[0]))
+    if name is None:
+        _, _ext = os.path.splitext(filename)
+        if _ext:
+            _ext = _ext[1:]
+        if _ext in SHORTCUTS:
+            name = SHORTCUTS[_ext]
 
-    if factory_type == 'reader':
+    if not name in REGISTRY:
+        raise RuntimeError(
+            'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} <format>.'.format(
+                name=name, filename=filename, factory_type=factory_type, opt=factory_type[0]))
+
+    if factory_type == READER:
         return REGISTRY[name][0]
-    elif factory_type == 'writer':
+    elif factory_type == WRITER:
         return REGISTRY[name][1]
     else:
         raise ValueError('Invalid factory type.')
 
+
 def execute(input, output, reader=None, reader_options=None, writer=None, writer_options=None, options=None):
-    reader = resolve_factory(reader, input, 'reader')(input)
-    writer = resolve_factory(writer, output, 'writer')(output)
+    reader = resolve_factory(reader, input, READER)(input)
+    writer = resolve_factory(writer, output, WRITER)(output)
 
     graph = bonobo.Graph()
     graph.add_chain(reader, writer)
 
     return bonobo.run(graph, services={
-            'fs': bonobo.open_fs(),
-        })
+        'fs': bonobo.open_fs(),
+    })
+
 
 def register(parser):
     parser.add_argument('input')
     parser.add_argument('output')
-    parser.add_argument('--reader', '-r')
-    parser.add_argument('--writer', '-w')
+    parser.add_argument('--' + READER, '-r')
+    parser.add_argument('--' + WRITER, '-w')
     # parser.add_argument('--reader-option', '-ro', dest='reader_options')
     # parser.add_argument('--writer-option', '-wo', dest='writer_options')
     # parser.add_argument('--option', '-o', dest='options')

From 7fb572ec90548050dff47ae72a1cc1df70487cd7 Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sat, 30 Sep 2017 11:45:13 +0200
Subject: [PATCH 11/16] [doc] new commands.

---
 docs/reference/commands.rst | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/docs/reference/commands.rst b/docs/reference/commands.rst
index dcd054a..674d549 100644
--- a/docs/reference/commands.rst
+++ b/docs/reference/commands.rst
@@ -1,6 +1,21 @@
 Command-line
 ============
 
+
+Bonobo Convert
+::::::::::::::
+
+Build a simple bonobo graph with one reader and one writer, then execute it, allowing to use bonobo in "no code" mode
+for simple file format conversions.
+
+Syntax: `bonobo convert [-r reader] input_filename [-w writer] output_filename`
+
+.. todo::
+
+    add a way to override default options of reader/writers, add a way to add "filters", for example this could be used
+    to read from csv and write to csv too (or other format) but adding a geocoder filter that would add some fields.
+
+
 Bonobo Init
 :::::::::::
 
@@ -8,7 +23,17 @@ Create an empty project, ready to use bonobo.
 
 Syntax: `bonobo init`
 
-Requires `edgy.project`.
+Requires `cookiecutter`.
+
+
+Bonobo Inspect
+::::::::::::::
+
+Inspects a bonobo graph source files. For now, only support graphviz output.
+
+Syntax: `bonobo inspect [--graph|-g] filename`
+
+Requires graphviz if you want to generate an actual graph picture, although the command itself depends on nothing.
 
 
 Bonobo Run
@@ -20,6 +45,7 @@ Syntax: `bonobo run [-c cmd | -m mod | file | -] [arg]`
 
 .. todo:: implement -m, check if -c is of any use and if yes, implement it too. Implement args, too.
 
+
 Bonobo RunC
 :::::::::::
 

From 9b3777f88d6bbf44372684e6681b813f4cc48b1d Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sat, 30 Sep 2017 11:48:17 +0200
Subject: [PATCH 12/16] [minor] update test script for graphs.

---
 bin/test_graph | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/test_graph b/bin/test_graph
index 1e5fd85..29841f5 100644
--- a/bin/test_graph
+++ b/bin/test_graph
@@ -1 +1 @@
-bonobo graph bonobo/examples/tutorials/tut02_03_writeasmap.py | dot -otest.png -Tpng && bin/imgcat test.png 
+bonobo inspect --graph bonobo/examples/tutorials/tut02e03_writeasmap.py | dot -o test_output.png -T png && bin/imgcat test_output.png

From b26dbc83cbfd1b35903920b6bdaaa7f93152e2d9 Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sat, 30 Sep 2017 11:50:34 +0200
Subject: [PATCH 13/16] minor cleanups

---
 bonobo/nodes/throttle.py | 3 ---
 bonobo/util/graphviz.py  | 8 --------
 2 files changed, 11 deletions(-)
 delete mode 100644 bonobo/util/graphviz.py

diff --git a/bonobo/nodes/throttle.py b/bonobo/nodes/throttle.py
index 2f08cd3..58f5c09 100644
--- a/bonobo/nodes/throttle.py
+++ b/bonobo/nodes/throttle.py
@@ -41,15 +41,12 @@ class RateLimited(Configurable):
 
     @ContextProcessor
     def bucket(self, context):
-        print(context)
         bucket = RateLimitBucket(self.initial, self.amount, self.period)
         bucket.start()
-        print(bucket)
         yield bucket
         bucket.stop()
         bucket.join()
 
     def call(self, bucket, *args, **kwargs):
-        print(bucket, args, kwargs)
         bucket.wait()
         return self.handler(*args, **kwargs)
diff --git a/bonobo/util/graphviz.py b/bonobo/util/graphviz.py
deleted file mode 100644
index 588e374..0000000
--- a/bonobo/util/graphviz.py
+++ /dev/null
@@ -1,8 +0,0 @@
-def render_as_dot(graph):
-    """
-    
-    :param bonobo.Graph graph: 
-    :return: str
-    """
-
-    pass

From ef938c99707168a28ed6b98945831de7fc30e86d Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sat, 30 Sep 2017 11:58:37 +0200
Subject: [PATCH 14/16] Adding warnings to transformation factory, and code
 formating.

---
 bonobo/commands/convert.py          | 13 ++++++++-----
 bonobo/examples/datasets/fablabs.py | 11 ++++++-----
 bonobo/examples/nodes/factory.py    |  7 ++++---
 bonobo/nodes/factory.py             | 18 ++++++++++++------
 4 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py
index 3c26724..17b98c2 100644
--- a/bonobo/commands/convert.py
+++ b/bonobo/commands/convert.py
@@ -44,8 +44,9 @@ def resolve_factory(name, filename, factory_type):
 
     if not name in REGISTRY:
         raise RuntimeError(
-            'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} <format>.'.format(
-                name=name, filename=filename, factory_type=factory_type, opt=factory_type[0]))
+            'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} <format>.'.
+            format(name=name, filename=filename, factory_type=factory_type, opt=factory_type[0])
+        )
 
     if factory_type == READER:
         return REGISTRY[name][0]
@@ -62,9 +63,11 @@ def execute(input, output, reader=None, reader_options=None, writer=None, writer
     graph = bonobo.Graph()
     graph.add_chain(reader, writer)
 
-    return bonobo.run(graph, services={
-        'fs': bonobo.open_fs(),
-    })
+    return bonobo.run(
+        graph, services={
+            'fs': bonobo.open_fs(),
+        }
+    )
 
 
 def register(parser):
diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py
index 1a21671..b87019f 100644
--- a/bonobo/examples/datasets/fablabs.py
+++ b/bonobo/examples/datasets/fablabs.py
@@ -48,7 +48,6 @@ def normalize(row):
     return result
 
 
-
 def display(row):
     print(Style.BRIGHT, row.get('name'), Style.RESET_ALL, sep='')
 
@@ -69,15 +68,15 @@ def display(row):
 
     print(
         '  - {}address{}: {address}'.
-            format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address))
+        format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address))
     )
     print(
         '  - {}links{}: {links}'.
-            format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links']))
+        format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links']))
     )
     print(
         '  - {}geometry{}: {geometry}'.
-            format(Fore.BLUE, Style.RESET_ALL, **row)
+        format(Fore.BLUE, Style.RESET_ALL, **row)
     )
     print(
         '  - {}source{}: {source}'.format(
@@ -87,7 +86,9 @@ def display(row):
 
 
 graph = bonobo.Graph(
-    OpenDataSoftAPI(dataset=API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris'),
+    OpenDataSoftAPI(
+        dataset=API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris'
+    ),
     normalize,
     bonobo.Filter(filter=lambda row: row.get('country') == 'France'),
     bonobo.JsonWriter(path='fablabs.txt', ioformat='arg0'),
diff --git a/bonobo/examples/nodes/factory.py b/bonobo/examples/nodes/factory.py
index d4702c6..d1aac89 100644
--- a/bonobo/examples/nodes/factory.py
+++ b/bonobo/examples/nodes/factory.py
@@ -8,6 +8,7 @@ from bonobo.config import Configurable
 from bonobo.nodes.factory import Factory
 from bonobo.nodes.io.json import JsonDictReader
 
+
 @Factory
 def Normalize(self):
     self[0].str().title()
@@ -15,11 +16,11 @@ def Normalize(self):
     self.move(0, 'address')
 
 
-
-
 class PrettyPrinter(Configurable):
     def call(self, *args, **kwargs):
-        for i, (item, value) in enumerate(itertools.chain(enumerate(args), kwargs.items())):
+        for i, (
+            item, value
+        ) in enumerate(itertools.chain(enumerate(args), kwargs.items())):
             print('  ' if i else '• ', item, '=', value)
 
 
diff --git a/bonobo/nodes/factory.py b/bonobo/nodes/factory.py
index af3f778..736ca2b 100644
--- a/bonobo/nodes/factory.py
+++ b/bonobo/nodes/factory.py
@@ -1,8 +1,9 @@
 import functools
+import warnings
 from functools import partial
 
 from bonobo import Bag
-from bonobo.config import Configurable, Method
+from bonobo.config import Configurable
 
 _isarg = lambda item: type(item) is int
 _iskwarg = lambda item: type(item) is str
@@ -110,7 +111,10 @@ class Cursor():
             setattr(self, item, partial(_operation, self))
             return getattr(self, item)
 
-        raise AttributeError('Unknown operation {}.{}().'.format(type(self).__name__, item, ))
+        raise AttributeError('Unknown operation {}.{}().'.format(
+            type(self).__name__,
+            item,
+        ))
 
 
 CURSOR_TYPES['default'] = Cursor
@@ -139,12 +143,15 @@ CURSOR_TYPES['str'] = StringCursor
 
 
 class Factory(Configurable):
-    setup = Method()
-
     def __init__(self):
+        warnings.warn(
+            __file__ +
+            ' is experimental, API may change in the future, use it as a preview only and knowing the risks.',
+            FutureWarning
+        )
+        super(Factory, self).__init__()
         self.default_cursor_type = 'default'
         self.operations = []
-        self.setup()
 
     @factory_operation
     def move(self, _from, _to, *args, **kwargs):
@@ -186,7 +193,6 @@ if __name__ == '__main__':
 
     print('operations:', f.operations)
     print(f({'foo': 'bisou'}, foo='blah'))
-
 '''
 specs:
 

From 0d53e135813d83aee73cc0e92eb51054aee66c93 Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Sat, 30 Sep 2017 12:23:14 +0200
Subject: [PATCH 15/16] Making factory example work. Still work in progress.

---
 bonobo/examples/datasets/coffeeshops.txt | 214 +++++++++++------------
 bonobo/examples/nodes/factory.py         |  32 +---
 bonobo/nodes/factory.py                  |  15 +-
 bonobo/nodes/io/json.py                  |   9 +-
 4 files changed, 132 insertions(+), 138 deletions(-)

diff --git a/bonobo/examples/datasets/coffeeshops.txt b/bonobo/examples/datasets/coffeeshops.txt
index b87eacb..9e3c181 100644
--- a/bonobo/examples/datasets/coffeeshops.txt
+++ b/bonobo/examples/datasets/coffeeshops.txt
@@ -1,36 +1,37 @@
-les montparnos, 65 boulevard Pasteur, 75015 Paris, France
-Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France
-Café Lea, 5 rue Claude Bernard, 75005 Paris, France
-Le Bellerive, 71 quai de Seine, 75019 Paris, France
-Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
+Extérieur Quai, 5, rue d'Alsace, 75010 Paris, France
+Le Sully, 6 Bd henri IV, 75004 Paris, France
 O q de poule, 53 rue du ruisseau, 75018 Paris, France
-Le café des amis, 125 rue Blomet, 75015 Paris, France
+Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France
+La Renaissance, 112 Rue Championnet, 75018 Paris, France
+La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France
 Le chantereine, 51 Rue Victoire, 75009 Paris, France
 Le Müller, 11 rue Feutrier, 75018 Paris, France
-Extérieur Quai, 5, rue d'Alsace, 75010 Paris, France
-La Bauloise, 36 rue du hameau, 75015 Paris, France
-Le Dellac, 14 rue Rougemont, 75009 Paris, France
-Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
-Le Sully, 6 Bd henri IV, 75004 Paris, France
-Le Felteu, 1 rue Pecquay, 75004 Paris, France
-Le bistrot de Maëlle et Augustin, 42 rue coquillère, 75001 Paris, France
-Dédé la frite, 52 rue Notre-Dame des Victoires, 75002 Paris, France
-Cardinal Saint-Germain, 11 boulevard Saint-Germain, 75005 Paris, France
-Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
-Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France
-Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
-La Renaissance, 112 Rue Championnet, 75018 Paris, France
-Le Square, 31 rue Saint-Dominique, 75007 Paris, France
-Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
-Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
-Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
-Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
-Café antoine, 17 rue Jean de la Fontaine, 75016 Paris, France
-Au cerceau d'or, 129 boulevard sebastopol, 75002 Paris, France
-La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France
-Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France
+Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
+Le café des amis, 125 rue Blomet, 75015 Paris, France
 Le Café Livres, 10 rue Saint Martin, 75004 Paris, France
+Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
 Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France
+Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
+Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
+Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
+Le Square, 31 rue Saint-Dominique, 75007 Paris, France
+Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
+Au cerceau d'or, 129 boulevard sebastopol, 75002 Paris, France
+Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France
+Café antoine, 17 rue Jean de la Fontaine, 75016 Paris, France
+Café de la Mairie (du VIII), rue de Lisbonne, 75008 Paris, France
+Café Lea, 5 rue Claude Bernard, 75005 Paris, France
+Cardinal Saint-Germain, 11 boulevard Saint-Germain, 75005 Paris, France
+Dédé la frite, 52 rue Notre-Dame des Victoires, 75002 Paris, France
+La Bauloise, 36 rue du hameau, 75015 Paris, France
+Le Bellerive, 71 quai de Seine, 75019 Paris, France
+Le bistrot de Maëlle et Augustin, 42 rue coquillère, 75001 Paris, France
+Le Dellac, 14 rue Rougemont, 75009 Paris, France
+Le Felteu, 1 rue Pecquay, 75004 Paris, France
+Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
+Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
+les montparnos, 65 boulevard Pasteur, 75015 Paris, France
+L'antre d'eux, 16 rue DE MEZIERES, 75006 Paris, France
 Drole d'endroit pour une rencontre, 58 rue de Montorgueil, 75002 Paris, France
 Le pari's café, 104 rue caulaincourt, 75018 Paris, France
 Le Poulailler, 60 rue saint-sabin, 75011 Paris, France
@@ -62,7 +63,6 @@ Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France
 Le Café frappé, 95 rue Montmartre, 75002 Paris, France
 La Perle, 78 rue vieille du temple, 75003 Paris, France
 Le Descartes, 1 rue Thouin, 75005 Paris, France
-Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
 Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France
 Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France
 Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France
@@ -75,7 +75,6 @@ Extra old café, 307 fg saint Antoine, 75011 Paris, France
 Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France
 En attendant l'or, 3 rue Faidherbe, 75011 Paris, France
 Brûlerie San José, 30 rue des Petits-Champs, 75002 Paris, France
-Café de la Mairie (du VIII), rue de Lisbonne, 75008 Paris, France
 Café Martin, 2 place Martin Nadaud, 75001 Paris, France
 Etienne, 14 rue Turbigo, Paris, 75001 Paris, France
 L'ingénu, 184 bd Voltaire, 75011 Paris, France
@@ -87,96 +86,97 @@ Le Germinal, 95 avenue Emile Zola, 75015 Paris, France
 Le Ragueneau, 202 rue Saint-Honoré, 75001 Paris, France
 Le refuge, 72 rue lamarck, 75018 Paris, France
 Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France
+Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France
+Le bal du pirate, 60 rue des bergers, 75015 Paris, France
+zic zinc, 95 rue claude decaen, 75012 Paris, France
+l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
+Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
+L'Inévitable, 22 rue Linné, 75005 Paris, France
 Le Dunois, 77 rue Dunois, 75013 Paris, France
-La Montagne Sans Geneviève, 13 Rue du Pot de Fer, 75005 Paris, France
+Ragueneau, 202 rue Saint Honoré, 75001 Paris, France
 Le Caminito, 48 rue du Dessous des Berges, 75013 Paris, France
+Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France
 Le petit Bretonneau, Le petit Bretonneau - à l'intérieur de l'Hôpital, 75018 Paris, France
+Le Centenaire, 104 rue amelot, 75011 Paris, France
+La Montagne Sans Geneviève, 13 Rue du Pot de Fer, 75005 Paris, France
+Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France
+Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
+Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
 La chaumière gourmande, Route de la Muette à Neuilly
 Club hippique du Jardin d’Acclimatation, 75016 Paris, France
-Le bal du pirate, 60 rue des bergers, 75015 Paris, France
-Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
-L'antre d'eux, 16 rue DE MEZIERES, 75006 Paris, France
-l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
-zic zinc, 95 rue claude decaen, 75012 Paris, France
-Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France
-Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France
-Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
-Le Centenaire, 104 rue amelot, 75011 Paris, France
-Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
-Ragueneau, 202 rue Saint Honoré, 75001 Paris, France
+Le Brio, 216, rue Marcadet, 75018 Paris, France
+Caves populaires, 22 rue des Dames, 75017 Paris, France
+Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
+Tamm Bara, 7 rue Clisson, 75013 Paris, France
+L'anjou, 1 rue de Montholon, 75009 Paris, France
+Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France
+Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
+Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
+bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
+Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
 Café Pistache, 9 rue des petits champs, 75001 Paris, France
 La Cagnotte, 13 Rue Jean-Baptiste Dumay, 75020 Paris, France
-Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
-Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
 le 1 cinq, 172 rue de vaugirard, 75015 Paris, France
+Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
 Les Artisans, 106 rue Lecourbe, 75015 Paris, France
 Peperoni, 83 avenue de Wagram, 75001 Paris, France
-Le Brio, 216, rue Marcadet, 75018 Paris, France
-Tamm Bara, 7 rue Clisson, 75013 Paris, France
-Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France
-bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
-Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
-Caves populaires, 22 rue des Dames, 75017 Paris, France
-Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
-Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
-L'Inévitable, 22 rue Linné, 75005 Paris, France
-L'anjou, 1 rue de Montholon, 75009 Paris, France
-Botak cafe, 1 rue Paul albert, 75018 Paris, France
-Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
-Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
-Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
-L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
-le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
-Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
-maison du vin, 52 rue des plantes, 75014 Paris, France
-Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
-Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
-Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France
-La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France
-Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France
-Les caves populaires, 22 rue des Dames, 75017 Paris, France
-Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
-Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
-La Brocante, 10 rue Rossini, 75009 Paris, France
-Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
-Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
-Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
-La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France
-La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
-Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
-Café Clochette, 16 avenue Richerand, 75010 Paris, France
+le lutece, 380 rue de vaugirard, 75015 Paris, France
+Brasiloja, 16 rue Ganneron, 75018 Paris, France
+Rivolux, 16 rue de Rivoli, 75004 Paris, France
 L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France
 NoMa, 39 rue Notre Dame de Nazareth, 75003 Paris, France
-le lutece, 380 rue de vaugirard, 75015 Paris, France
 O'Paris, 1 Rue des Envierges, 75020 Paris, France
-Rivolux, 16 rue de Rivoli, 75004 Paris, France
-Brasiloja, 16 rue Ganneron, 75018 Paris, France
-Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
-Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
-Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
-Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France
-L'Angle, 28 rue de Ponthieu, 75008 Paris, France
-Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France
-Café Dupont, 198 rue de la Convention, 75015 Paris, France
-Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France
-L'Entracte, place de l'opera, 75002 Paris, France
-Panem, 18 rue de Crussol, 75011 Paris, France
-Au pays de Vannes, 34 bis rue de Wattignies, 75012 Paris, France
-l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France
-L'âge d'or, 26 rue du Docteur Magnan, 75013 Paris, France
-Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
-L'horizon, 93, rue de la Roquette, 75011 Paris, France
-L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
-Café Victor, 10 boulevard Victor, 75015 Paris, France
-Café Varenne, 36 rue de Varenne, 75007 Paris, France
-Le Brigadier, 12 rue Blanche, 75009 Paris, France
-Waikiki, 10 rue d"Ulm, 75005 Paris, France
-Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France
-Pari's Café, 174 avenue de Clichy, 75017 Paris, France
-Melting Pot, 3 rue de Lagny, 75020 Paris, France
-le Zango, 58 rue Daguerre, 75014 Paris, France
-Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
+Café Clochette, 16 avenue Richerand, 75010 Paris, France
+La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
+Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
+La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France
+Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France
+Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France
+La Brocante, 10 rue Rossini, 75009 Paris, France
+Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
+Les caves populaires, 22 rue des Dames, 75017 Paris, France
+Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
+Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
+Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
+Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
+La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France
+Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
+L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
+Botak cafe, 1 rue Paul albert, 75018 Paris, France
+le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
+Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
+Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
+Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
+Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
+Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
+maison du vin, 52 rue des plantes, 75014 Paris, France
+L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France
 Le café Monde et Médias, Place de la République, 75003 Paris, France
 Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France
 Brasserie le Morvan, 61 rue du château d'eau, 75010 Paris, France
-L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France
\ No newline at end of file
+Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
+Panem, 18 rue de Crussol, 75011 Paris, France
+Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
+Café Dupont, 198 rue de la Convention, 75015 Paris, France
+L'Angle, 28 rue de Ponthieu, 75008 Paris, France
+Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
+Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
+L'Entracte, place de l'opera, 75002 Paris, France
+Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France
+Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France
+Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France
+Le Brigadier, 12 rue Blanche, 75009 Paris, France
+L'âge d'or, 26 rue du Docteur Magnan, 75013 Paris, France
+Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
+Café Victor, 10 boulevard Victor, 75015 Paris, France
+L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
+L'horizon, 93, rue de la Roquette, 75011 Paris, France
+Waikiki, 10 rue d"Ulm, 75005 Paris, France
+Au pays de Vannes, 34 bis rue de Wattignies, 75012 Paris, France
+Café Varenne, 36 rue de Varenne, 75007 Paris, France
+l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France
+Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
+Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France
+le Zango, 58 rue Daguerre, 75014 Paris, France
+Melting Pot, 3 rue de Lagny, 75020 Paris, France
+Pari's Café, 174 avenue de Clichy, 75017 Paris, France
\ No newline at end of file
diff --git a/bonobo/examples/nodes/factory.py b/bonobo/examples/nodes/factory.py
index d1aac89..c1f3818 100644
--- a/bonobo/examples/nodes/factory.py
+++ b/bonobo/examples/nodes/factory.py
@@ -1,33 +1,17 @@
-from functools import partial
-
-import itertools
-
 import bonobo
 from bonobo.commands.run import get_default_services
-from bonobo.config import Configurable
 from bonobo.nodes.factory import Factory
-from bonobo.nodes.io.json import JsonDictReader
-
-
-@Factory
-def Normalize(self):
-    self[0].str().title()
-    self.move(0, 'title')
-    self.move(0, 'address')
-
-
-class PrettyPrinter(Configurable):
-    def call(self, *args, **kwargs):
-        for i, (
-            item, value
-        ) in enumerate(itertools.chain(enumerate(args), kwargs.items())):
-            print('  ' if i else '• ', item, '=', value)
+from bonobo.nodes.io.json import JsonDictItemsReader
 
+normalize = Factory()
+normalize[0].str().title()
+normalize.move(0, 'title')
+normalize.move(0, 'address')
 
 graph = bonobo.Graph(
-    JsonDictReader('datasets/coffeeshops.json'),
-    Normalize(),
-    PrettyPrinter(),
+    JsonDictItemsReader('datasets/coffeeshops.json'),
+    normalize,
+    bonobo.PrettyPrinter(),
 )
 
 if __name__ == '__main__':
diff --git a/bonobo/nodes/factory.py b/bonobo/nodes/factory.py
index 736ca2b..2a1c30b 100644
--- a/bonobo/nodes/factory.py
+++ b/bonobo/nodes/factory.py
@@ -3,7 +3,7 @@ import warnings
 from functools import partial
 
 from bonobo import Bag
-from bonobo.config import Configurable
+from bonobo.config import Configurable, Method
 
 _isarg = lambda item: type(item) is int
 _iskwarg = lambda item: type(item) is str
@@ -143,16 +143,21 @@ CURSOR_TYPES['str'] = StringCursor
 
 
 class Factory(Configurable):
-    def __init__(self):
+    initialize = Method(required=False)
+
+    def __init__(self, *args, **kwargs):
         warnings.warn(
             __file__ +
             ' is experimental, API may change in the future, use it as a preview only and knowing the risks.',
             FutureWarning
         )
-        super(Factory, self).__init__()
+        super(Factory, self).__init__(*args, **kwargs)
         self.default_cursor_type = 'default'
         self.operations = []
 
+        if self.initialize is not None:
+            self.initialize(self)
+
     @factory_operation
     def move(self, _from, _to, *args, **kwargs):
         if _from == _to:
@@ -175,10 +180,10 @@ class Factory(Configurable):
             raise RuntimeError('Houston, we have a problem...')
 
     def __call__(self, *args, **kwargs):
-        # print('factory call on', args, kwargs)
+        print('factory call on', args, kwargs)
         for operation in self.operations:
             args, kwargs = operation.apply(*args, **kwargs)
-            # print(' ... after', operation, 'got', args, kwargs)
+            print(' ... after', operation, 'got', args, kwargs)
         return Bag(*args, **kwargs)
 
     def __getitem__(self, item):
diff --git a/bonobo/nodes/io/json.py b/bonobo/nodes/io/json.py
index 81b6870..f1c6df0 100644
--- a/bonobo/nodes/io/json.py
+++ b/bonobo/nodes/io/json.py
@@ -1,11 +1,10 @@
 import json
-from itertools import starmap
 
-from bonobo.structs.bags import Bag
 from bonobo.config.processors import ContextProcessor
 from bonobo.constants import NOT_MODIFIED
 from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
 from bonobo.nodes.io.file import FileReader, FileWriter
+from bonobo.structs.bags import Bag
 
 
 class JsonHandler(FileHandler):
@@ -21,6 +20,12 @@ class JsonReader(IOFormatEnabled, FileReader, JsonHandler):
             yield self.get_output(line)
 
 
+class JsonDictItemsReader(JsonReader):
+    def read(self, fs, file):
+        for line in self.loader(file).items():
+            yield Bag(*line)
+
+
 class JsonWriter(IOFormatEnabled, FileWriter, JsonHandler):
     @ContextProcessor
     def envelope(self, context, fs, file, lineno):

From 0eb5fd8a150c95e984321a30e7f7748c78416acd Mon Sep 17 00:00:00 2001
From: spagoc <spagoc@users.noreply.github.com>
Date: Thu, 28 Sep 2017 08:42:35 +0200
Subject: [PATCH 16/16] More readable statistics on Ubuntu workstation standard
 terminal

statistics are hard to read on the Ubuntu standard terminal issue #165
---
 bonobo/ext/console.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py
index 6679092..4d8cb6f 100644
--- a/bonobo/ext/console.py
+++ b/bonobo/ext/console.py
@@ -81,7 +81,7 @@ class ConsoleOutputPlugin(Plugin):
                 print(line + CLEAR_EOL, file=sys.stderr)
 
         alive_color = Style.BRIGHT
-        dead_color = (Style.BRIGHT + Fore.BLACK) if self.iswindows else Fore.BLACK
+        dead_color = Style.BRIGHT + Fore.BLACK
 
         for i in context.graph.topologically_sorted_indexes:
             node = context[i]