Minor fixes and update documentation. Preparing the upcoming 0.2 release.

bonobo/__init__.py

@@ -1,30 +1,31 @@
-""" Bonobo data-processing toolkit.
+# Bonobo data-processing toolkit.
+#
+# Bonobo is a line-by-line data-processing toolkit for python 3.5+ emphasizing simplicity and atomicity of data
+# transformations using a simple directed graph of python callables.
+#
+# Licensed under Apache License 2.0, read the LICENSE file in the root of the source tree.
 
-    Bonobo is a line-by-line data-processing toolkit for python 3.5+ emphasizing simplicity and atomicity of data
-    transformations using a simple directed graph of python callables.
+"""Bonobo data-processing toolkit main module."""
 
-    Read more at http://docs.bonobo-project.org/
-
-    Copyright 2012-2014 Romain Dorgueil
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-        http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-"""
 import sys
 import warnings
 
 assert (sys.version_info >= (3, 5)), 'Python 3.5+ is required to use Bonobo.'
 
 from ._version import __version__
+from .config import __all__ as __all_config__
+from .context import __all__ as __all_context__
+from .core import __all__ as __all_core__
+from .io import __all__ as __all_io__
+from .util import __all__ as __all_util__
+
+__all__ = __all_config__ + __all_context__ + __all_core__ + __all_io__ + __all_util__ + [
+    '__version__',
+    'create_strategy',
+    'get_examples_path',
+    'run',
+]
+
 from .config import *
 from .context import *
 from .core import *
@@ -40,56 +41,43 @@ STRATEGIES = {
 }
 
 
-def run(graph, *chain, strategy=None, plugins=None):
+def get_examples_path(*pathsegments):
+    import os
+    import pathlib
+    return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments))
+
+
+def create_strategy(name=None):
     from bonobo.core.strategies.base import Strategy
+    import logging
+
+    if isinstance(name, Strategy):
+        return name
+
+    if name is None:
+        name = DEFAULT_STRATEGY
+
+    logging.debug('Creating strategy {}...'.format(name))
+
+    try:
+        factory = STRATEGIES[name]
+    except KeyError as exc:
+        raise RuntimeError('Invalid strategy {}. Available choices: {}.'.format(repr(name), ', '.join(
+            sorted(STRATEGIES.keys())))) from exc
+
+    return factory()
+
+
+def run(graph, *chain, strategy=None, plugins=None):
+    strategy = create_strategy(strategy)
 
     if len(chain):
         warnings.warn('DEPRECATED. You should pass a Graph instance instead of a chain.')
         from bonobo import Graph
         graph = Graph(graph, *chain)
 
-    if not isinstance(strategy, Strategy):
-        if strategy is None:
-            strategy = DEFAULT_STRATEGY
-
-        try:
-            strategy = STRATEGIES[strategy]
-        except KeyError as exc:
-            raise RuntimeError('Invalid strategy {}.'.format(repr(strategy))) from exc
-
-        strategy = strategy()
-
     return strategy.execute(graph, plugins=plugins)
 
 
-__all__ = [
-    'Bag',
-    'Configurable',
-    'ContextProcessor',
-    'contextual',
-    'CsvReader',
-    'CsvWriter',
-    'FileReader',
-    'FileWriter',
-    'Graph',
-    'JsonReader',
-    'JsonWriter',
-    'NOT_MODIFIED',
-    'NaiveStrategy',
-    'Option',
-    'ProcessPoolExecutorStrategy',
-    'ThreadPoolExecutorStrategy',
-    '__version__',
-    'console_run',
-    'inject',
-    'jupyter_run',
-    'limit',
-    'log',
-    'noop',
-    'pprint',
-    'service',
-    'tee',
-]
-
-del warnings
 del sys
+del warnings

bonobo/commands/__init__.py

@@ -1,22 +1,25 @@
 import argparse
 
+import logging
 from stevedore import ExtensionManager
 
 
-def entrypoint():
+def entrypoint(args=None):
     parser = argparse.ArgumentParser()
 
     subparsers = parser.add_subparsers(dest='command')
     subparsers.required = True
 
-    def register_extension(ext):
-        parser = subparsers.add_parser(ext.name)
-        command = ext.plugin(parser)
-        parser.set_defaults(command=command)
+    commands = {}
+    def register_extension(ext, commands=commands):
+        try:
+            parser = subparsers.add_parser(ext.name)
+            commands[ext.name] = ext.plugin(parser)
+        except Exception:
+            logging.exception('Error while loading command {}.'.format(ext.name))
 
     mgr = ExtensionManager(namespace='bonobo.commands', )
     mgr.map(register_extension)
 
-    args = parser.parse_args().__dict__
-    command = args.pop('command')
-    command(**args)
+    args = parser.parse_args(args).__dict__
+    commands[args.pop('command')](**args)

bonobo/commands/run.py

@@ -1,13 +1,21 @@
 import argparse
 
-from bonobo import Graph, console_run
+from bonobo import Graph, run
 
 
-def execute(file):
+def execute(file, quiet=False):
     with file:
         code = compile(file.read(), file.name, 'exec')
 
-    context = {}
+    # TODO: A few special variables should be set before running the file:
+    #
+    # See:
+    #  - https://docs.python.org/3/reference/import.html#import-mod-attrs
+    #  - https://docs.python.org/3/library/runpy.html#runpy.run_module
+    context = {
+        '__name__': '__bonobo__',
+        '__file__': file.name,
+    }
 
     try:
         exec(code, context)
@@ -16,14 +24,19 @@ def execute(file):
 
     graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph))
 
-    assert len(graphs) == 1, 'Having more than one graph definition in one file is unsupported for now, but it is ' \
-                             'something that will be implemented in the future. '
+    assert len(graphs) == 1, ('Having zero or more than one graph definition in one file is unsupported for now, '
+                              'but it is something that will be implemented in the future.\n\nExpected: 1, got: {}.').format(
+        len(graphs))
 
     name, graph = list(graphs.items())[0]
 
-    return console_run(graph)
+    # todo if console and not quiet, then add the console plugin
+    # todo when better console plugin, add it if console and just disable display
+
+    return run(graph)
 
 
 def register(parser):
     parser.add_argument('file', type=argparse.FileType())
+    parser.add_argument('--quiet', action='store_true')
     return execute

bonobo/config.py

@@ -1,3 +1,8 @@
+__all__ = [
+    'Configurable',
+    'Option',
+]
+
 class Option:
     def __init__(self, type=None, *, required=False, default=None):
         self.name = None

bonobo/context/execution.py

@@ -23,7 +23,7 @@ class GraphExecutionContext:
 
     @property
     def alive(self):
-        return self.started and not self.stopped
+        return any(node.alive for node in self.nodes)
 
     def __init__(self, graph, plugins=None):
         self.graph = graph

bonobo/core/graphs.py

@@ -26,3 +26,6 @@ class Graph:
             _next = self.add_node(node)
             self.outputs_of(_input, create=True).add(_next)
             _input = _next
+
+    def __len__(self):
+        return len(self.nodes)

bonobo/core/strategies/executor.py

@@ -39,11 +39,9 @@ class ExecutorStrategy(Strategy):
             futures.append(executor.submit(_runner))
 
         for node_context in context.nodes:
-
             def _runner(node_context=node_context):
                 node_context.start()
                 node_context.loop()
-
             futures.append(executor.submit(_runner))
 
         while context.alive:

bonobo/examples/datasets/coffeeshops.py

@@ -0,0 +1,17 @@
+from os.path import dirname, realpath, join
+
+from bonobo import console_run
+from bonobo.ext.opendatasoft import from_opendatasoft_api
+from bonobo.io.file import FileWriter
+
+OUTPUT_FILENAME = realpath(join(dirname(__file__), 'coffeeshops.txt'))
+
+console_run(
+    from_opendatasoft_api(
+        'liste-des-cafes-a-un-euro', netloc='opendata.paris.fr'
+    ),
+    lambda row: '{nom_du_cafe}, {adresse}, {arrondissement} Paris, France'.format(**row),
+    FileWriter(OUTPUT_FILENAME),
+)
+
+print('Import done, read {} for results.'.format(OUTPUT_FILENAME))

bonobo/examples/datasets/coffeeshops.txt

@@ -0,0 +1,182 @@
+Extérieur Quai, 5, rue d'Alsace, 75010 Paris, France
+Le Sully, 6 Bd henri IV, 75004 Paris, France
+O q de poule, 53 rue du ruisseau, 75018 Paris, France
+Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France
+Le Dunois, 77 rue Dunois, 75013 Paris, France
+La Renaissance, 112 Rue Championnet, 75018 Paris, France
+Le chantereine, 51 Rue Victoire, 75009 Paris, France
+Le Müller, 11 rue Feutrier, 75018 Paris, France
+Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
+Le café des amis, 125 rue Blomet, 75015 Paris, France
+Le Café Livres, 10 rue Saint Martin, 75004 Paris, France
+Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
+Le Brio, 216, rue Marcadet, 75018 Paris, France
+Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
+Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
+Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
+Le Square, 31 rue Saint-Dominique, 75007 Paris, France
+Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
+Au cerceau d'or, 129 boulevard sebastopol, 75002 Paris, France
+Café antoine, 17 rue Jean de la Fontaine, 75016 Paris, France
+Café Lea, 5 rue Claude Bernard, 75005 Paris, France
+Cardinal Saint-Germain, 11 boulevard Saint-Germain, 75005 Paris, France
+Dédé la frite, 52 rue Notre-Dame des Victoires, 75002 Paris, France
+La Bauloise, 36 rue du hameau, 75015 Paris, France
+Le Bellerive, 71 quai de Seine, 75019 Paris, France
+Le bistrot de Maëlle et Augustin, 42 rue coquillère, 75001 Paris, France
+Le Dellac, 14 rue Rougemont, 75009 Paris, France
+Le Felteu, 1 rue Pecquay, 75004 Paris, France
+Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
+les montparnos, 65 boulevard Pasteur, 75015 Paris, France
+Drole d'endroit pour une rencontre, 58 rue de Montorgueil, 75002 Paris, France
+Le pari's café, 104 rue caulaincourt, 75018 Paris, France
+Le Poulailler, 60 rue saint-sabin, 75011 Paris, France
+L'Assassin, 99 rue Jean-Pierre Timbaud, 75011 Paris, France
+l'Usine, 1 rue d'Avron, 75020 Paris, France
+La Bricole, 52 rue Liebniz, 75018 Paris, France
+le ronsard, place maubert, 75005 Paris, France
+Face Bar, 82 rue des archives, 75003 Paris, France
+American Kitchen, 49 rue bichat, 75010 Paris, France
+La Marine, 55 bis quai de valmy, 75010 Paris, France
+Le Bloc, 21 avenue Brochant, 75017 Paris, France
+La Recoleta au Manoir, 229 avenue Gambetta, 75020 Paris, France
+Le Pareloup, 80 Rue Saint-Charles, 75015 Paris, France
+La Brasserie Gaité, 3 rue de la Gaité, 75014 Paris, France
+Café Zen, 46 rue Victoire, 75009 Paris, France
+O'Breizh, 27 rue de Penthièvre, 75008 Paris, France
+Le Petit Choiseul, 23 rue saint augustin, 75002 Paris, France
+Invitez vous chez nous, 7 rue Epée de Bois, 75005 Paris, France
+La Cordonnerie, 142 Rue Saint-Denis 75002 Paris, 75002 Paris, France
+Le Supercoin, 3, rue Baudelique, 75018 Paris, France
+Populettes, 86 bis rue Riquet, 75018 Paris, France
+Au bon coin, 49 rue des Cloys, 75018 Paris, France
+Le Couvent, 69 rue Broca, 75013 Paris, France
+La Brûlerie des Ternes, 111 rue mouffetard, 75005 Paris, France
+L'Écir, 59 Boulevard Saint-Jacques, 75014 Paris, France
+Le Chat bossu, 126, rue du Faubourg Saint Antoine, 75012 Paris, France
+Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France
+Le Café frappé, 95 rue Montmartre, 75002 Paris, France
+La Perle, 78 rue vieille du temple, 75003 Paris, France
+Le Descartes, 1 rue Thouin, 75005 Paris, France
+Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
+Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France
+Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France
+Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France
+Le Malar, 88 rue Saint-Dominique, 75007 Paris, France
+Au panini de la place, 47 rue Belgrand, 75020 Paris, France
+Le Village, 182 rue de Courcelles, 75017 Paris, France
+Pause Café, 41 rue de Charonne, 75011 Paris, France
+Le Pure café, 14 rue Jean Macé, 75011 Paris, France
+Extra old café, 307 fg saint Antoine, 75011 Paris, France
+Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France
+En attendant l'or, 3 rue Faidherbe, 75011 Paris, France
+Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France
+Brûlerie San José, 30 rue des Petits-Champs, 75002 Paris, France
+Etienne, 14 rue Turbigo, Paris, 75001 Paris, France
+L'ingénu, 184 bd Voltaire, 75011 Paris, France
+L'Olive, 8 rue L'Olive, 75018 Paris, France
+Le Biz, 18 rue Favart, 75002 Paris, France
+Le Cap Bourbon, 1 rue Louis le Grand, 75002 Paris, France
+Le General Beuret, 9 Place du General Beuret, 75015 Paris, France
+Le Germinal, 95 avenue Emile Zola, 75015 Paris, France
+Le Ragueneau, 202 rue Saint-Honoré, 75001 Paris, France
+Le refuge, 72 rue lamarck, 75018 Paris, France
+Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France
+L'antre d'eux, 16 rue DE MEZIERES, 75006 Paris, France
+Le bal du pirate, 60 rue des bergers, 75015 Paris, France
+zic zinc, 95 rue claude decaen, 75012 Paris, France
+l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
+Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
+L'Inévitable, 22 rue Linné, 75005 Paris, France
+Ragueneau, 202 rue Saint Honoré, 75001 Paris, France
+Le Caminito, 48 rue du Dessous des Berges, 75013 Paris, France
+Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France
+Le petit Bretonneau, Le petit Bretonneau - à l'intérieur de l'Hôpital, 75018 Paris, France
+Le Centenaire, 104 rue amelot, 75011 Paris, France
+La Montagne Sans Geneviève, 13 Rue du Pot de Fer, 75005 Paris, France
+Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France
+Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
+Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
+La chaumière gourmande, Route de la Muette à Neuilly
+Club hippique du Jardin d’Acclimatation, 75016 Paris, France
+Caves populaires, 22 rue des Dames, 75017 Paris, France
+Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
+Tamm Bara, 7 rue Clisson, 75013 Paris, France
+L'anjou, 1 rue de Montholon, 75009 Paris, France
+Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France
+Waikiki, 10 rue d"Ulm, 75005 Paris, France
+Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
+Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
+bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
+Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
+Café de la Mairie (du VIII), rue de Lisbonne, 75008 Paris, France
+Café Pistache, 9 rue des petits champs, 75001 Paris, France
+La Cagnotte, 13 Rue Jean-Baptiste Dumay, 75020 Paris, France
+le 1 cinq, 172 rue de vaugirard, 75015 Paris, France
+Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
+Les Artisans, 106 rue Lecourbe, 75015 Paris, France
+Peperoni, 83 avenue de Wagram, 75001 Paris, France
+le lutece, 380 rue de vaugirard, 75015 Paris, France
+Brasiloja, 16 rue Ganneron, 75018 Paris, France
+Rivolux, 16 rue de Rivoli, 75004 Paris, France
+Chai 33, 33 Cour Saint Emilion, 75012 Paris, France
+L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France
+NoMa, 39 rue Notre Dame de Nazareth, 75003 Paris, France
+O'Paris, 1 Rue des Envierges, 75020 Paris, France
+Café Clochette, 16 avenue Richerand, 75010 Paris, France
+La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
+Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
+La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France
+Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France
+Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France
+La Brocante, 10 rue Rossini, 75009 Paris, France
+Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
+Les caves populaires, 22 rue des Dames, 75017 Paris, France
+Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
+Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
+Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France
+Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
+Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
+La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France
+Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
+L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
+Botak cafe, 1 rue Paul albert, 75018 Paris, France
+le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
+Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
+Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
+Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
+Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
+Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
+maison du vin, 52 rue des plantes, 75014 Paris, France
+Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France
+L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France
+Le café Monde et Médias, Place de la République, 75003 Paris, France
+Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France
+Brasserie le Morvan, 61 rue du château d'eau, 75010 Paris, France
+Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
+La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France
+Panem, 18 rue de Crussol, 75011 Paris, France
+Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
+Café Dupont, 198 rue de la Convention, 75015 Paris, France
+L'Angle, 28 rue de Ponthieu, 75008 Paris, France
+Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
+Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
+L'Entracte, place de l'opera, 75002 Paris, France
+Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France
+Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France
+Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France
+Le Brigadier, 12 rue Blanche, 75009 Paris, France
+L'âge d'or, 26 rue du Docteur Magnan, 75013 Paris, France
+Café Victor, 10 boulevard Victor, 75015 Paris, France
+L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
+L'horizon, 93, rue de la Roquette, 75011 Paris, France
+Au pays de Vannes, 34 bis rue de Wattignies, 75012 Paris, France
+Café Martin, 2 place Martin Nadaud, 75001 Paris, France
+Café Varenne, 36 rue de Varenne, 75007 Paris, France
+l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France
+Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
+Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France
+Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
+le Zango, 58 rue Daguerre, 75014 Paris, France
+Melting Pot, 3 rue de Lagny, 75020 Paris, France
+Pari's Café, 174 avenue de Clichy, 75017 Paris, France

bonobo/examples/datasets/fablabs.py

@@ -0,0 +1,73 @@
+import json
+import os
+
+from blessings import Terminal
+
+from bonobo import Tee, JsonWriter, Graph, get_examples_path
+from bonobo.ext.opendatasoft import from_opendatasoft_api
+
+try:
+    import pycountry
+except ImportError as exc:
+    raise ImportError('You must install package "pycountry" to run this example.') from exc
+
+API_DATASET = 'fablabs-in-the-world'
+API_NETLOC = 'datanova.laposte.fr'
+ROWS = 100
+
+t = Terminal()
+__path__ = os.path.dirname(__file__)
+
+
+def _getlink(x):
+    return x.get('url', None)
+
+
+def normalize(row):
+    result = {
+        **
+        row,
+        'links': list(filter(None, map(_getlink, json.loads(row.get('links'))))),
+        'country': pycountry.countries.get(alpha_2=row.get('country_code', '').upper()).name,
+    }
+    return result
+
+
+def filter_france(row):
+    if row.get('country') == 'France':
+        yield row
+
+
+def display(row):
+    print(t.bold(row.get('name')))
+
+    address = list(
+        filter(
+            None, (
+                ' '.join(filter(None, (row.get('postal_code', None), row.get('city', None)))),
+                row.get('county', None),
+                row.get('country'),
+            )
+        )
+    )
+
+    print('  - {}: {address}'.format(t.blue('address'), address=', '.join(address)))
+    print('  - {}: {links}'.format(t.blue('links'), links=', '.join(row['links'])))
+    print('  - {}: {geometry}'.format(t.blue('geometry'), **row))
+    print('  - {}: {source}'.format(t.blue('source'), source='datanova/' + API_DATASET))
+
+
+graph = Graph(
+    from_opendatasoft_api(
+        API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris'
+    ),
+    normalize,
+    filter_france,
+    Tee(display),
+    JsonWriter(path=get_examples_path('datasets/fablabs.txt')),
+)
+
+if __name__ == '__main__':
+    from bonobo import run
+
+    run(graph)

bonobo/examples/files/csv.py

@@ -0,0 +1,11 @@
+from bonobo import CsvReader, Graph, get_examples_path
+
+graph = Graph(
+    CsvReader(path=get_examples_path('datasets/coffeeshops.txt')),
+    print,
+)
+
+if __name__ == '__main__':
+    import bonobo
+
+    bonobo.run(graph)

bonobo/examples/files/text.py

@@ -0,0 +1,20 @@
+from bonobo import FileReader, Graph
+
+
+def skip_comments(line):
+    if not line.startswith('#'):
+        yield line
+
+
+graph = Graph(
+    FileReader(path='/etc/passwd'),
+    skip_comments,
+    lambda s: s.split(':'),
+    lambda l: l[0],
+    print,
+)
+
+if __name__ == '__main__':
+    import bonobo
+
+    bonobo.run(graph)

bonobo/examples/tutorials/tut02_01_read.py

@@ -0,0 +1,14 @@
+import os
+import pathlib
+
+import bonobo
+
+workdir = pathlib.Path(os.path.dirname(__file__))
+
+graph = bonobo.Graph(
+    bonobo.FileReader(path=workdir.joinpath('datasets/coffeeshops.txt')),
+    print,
+)
+
+if __name__ == '__main__':
+    bonobo.run(graph)

bonobo/examples/tutorials/tutorial_basics_firststeps.py

@@ -0,0 +1,18 @@
+from bonobo import run
+
+
+def generate_data():
+    yield 'foo'
+    yield 'bar'
+    yield 'baz'
+
+
+def uppercase(x: str):
+    return x.upper()
+
+
+def output(x: str):
+    print(x)
+
+
+run(generate_data, uppercase, output)

bonobo/examples/tutorials/tutorial_basics_summary.py

@@ -0,0 +1,19 @@
+from bonobo import Graph, ThreadPoolExecutorStrategy
+
+
+def yield_from(*args):
+    yield from args
+
+
+# Represent our data processor as a simple directed graph of callables.
+graph = Graph(
+    lambda: (x for x in ('foo', 'bar', 'baz')),
+    str.upper,
+    print,
+)
+
+# Use a thread pool.
+executor = ThreadPoolExecutorStrategy()
+
+# Run the thing.
+executor.execute(graph)

bonobo/examples/types/__init__.py

@@ -0,0 +1,7 @@
+from . import bags, dicts, strings
+
+__all__ = [
+    'bags',
+    'dicts',
+    'strings',
+]

bonobo/examples/types/bags.py

@@ -0,0 +1,45 @@
+"""
+Example on how to use :class:`bonobo.Bag` instances to pass flexible args/kwargs to the next callable.
+
+.. graphviz::
+
+    digraph {
+        rankdir = LR;
+        stylesheet = "../_static/graphs.css";
+
+        BEGIN [shape="point"];
+        BEGIN -> "extract()" -> "transform(...)" -> "load(...)";
+    }
+
+"""
+
+
+from random import randint
+
+from bonobo import Bag, Graph
+
+
+def extract():
+    yield Bag(topic='foo')
+    yield Bag(topic='bar')
+    yield Bag(topic='baz')
+
+
+def transform(topic: str):
+    return Bag.inherit(
+        title=topic.title(),
+        rand=randint(10, 99)
+    )
+
+
+def load(topic: str, title: str, rand: int):
+    print('{} ({}) wait={}'.format(title, topic, rand))
+
+
+graph = Graph()
+graph.add_chain(extract, transform, load)
+
+if __name__ == '__main__':
+    from bonobo import run
+
+    run(graph)

bonobo/examples/types/dicts.py

@@ -0,0 +1,47 @@
+"""
+Example on how to use symple python dictionaries to communicate between transformations.
+
+.. graphviz::
+
+    digraph {
+        rankdir = LR;
+        stylesheet = "../_static/graphs.css";
+
+        BEGIN [shape="point"];
+        BEGIN -> "extract()" -> "transform(row: dict)" -> "load(row: dict)";
+    }
+
+"""
+
+from random import randint
+
+from bonobo import Graph
+
+
+def extract():
+    yield {'topic': 'foo'}
+    yield {'topic': 'bar'}
+    yield {'topic': 'baz'}
+
+
+def transform(row: dict):
+    return {
+        'topic': row['topic'].title(),
+        'randint': randint(10, 99),
+    }
+
+
+def load(row: dict):
+    print(row)
+
+
+graph = Graph(
+    extract,
+    transform,
+    load
+)
+
+if __name__ == '__main__':
+    from bonobo import run
+
+    run(graph)

bonobo/examples/types/strings.py

@@ -0,0 +1,43 @@
+"""
+Example on how to use symple python strings to communicate between transformations.
+
+.. graphviz::
+
+    digraph {
+        rankdir = LR;
+        stylesheet = "../_static/graphs.css";
+
+        BEGIN [shape="point"];
+        BEGIN -> "extract()" -> "transform(s: str)" -> "load(s: str)";
+    }
+
+"""
+from random import randint
+
+from bonobo import Graph
+
+
+def extract():
+    yield 'foo'
+    yield 'bar'
+    yield 'baz'
+
+
+def transform(s: str):
+    return '{} ({})'.format(s.title(), randint(10, 99))
+
+
+def load(s: str):
+    print(s)
+
+
+graph = Graph(
+    extract,
+    transform,
+    load
+)
+
+if __name__ == '__main__':
+    from bonobo import run
+
+    run(graph)

bonobo/ext/edgy/project/feature.py

@@ -0,0 +1,26 @@
+try:
+    import edgy.project
+except ImportError as e:
+    import logging
+
+    logging.exception('You must install edgy.project to use this.')
+
+import os
+
+from edgy.project.events import subscribe
+from edgy.project.feature import Feature, SUPPORT_PRIORITY
+
+
+class BonoboFeature(Feature):
+    requires = {'python'}
+
+    @subscribe('edgy.project.feature.make.on_generate', priority=SUPPORT_PRIORITY)
+    def on_make_generate(self, event):
+        makefile = event.makefile
+
+    @subscribe('edgy.project.on_start', priority=SUPPORT_PRIORITY)
+    def on_start(self, event):
+        package_path = event.setup['name'].replace('.', os.sep)
+
+        for file in ('example_graph'):
+            self.render_file(os.path.join(package_path, file + '.py'), os.path.join('tornado', file + '.py.j2'))

bonobo/io/csv.py

@@ -1,6 +1,7 @@
 import csv
 
-from bonobo import Option, ContextProcessor, contextual
+from bonobo.config import Option
+from bonobo.context import ContextProcessor, contextual
 from bonobo.util.objects import ValueHolder
 from .file import FileReader, FileWriter, FileHandler
 

bonobo/io/json.py

@@ -1,6 +1,6 @@
 import json
 
-from bonobo import ContextProcessor, contextual
+from bonobo.context import ContextProcessor, contextual
 from .file import FileWriter, FileReader
 
 __all__ = ['JsonWriter', ]

bonobo/util/__init__.py

@@ -9,14 +9,14 @@ from .helpers import console_run, jupyter_run
 from .tokens import NOT_MODIFIED
 
 __all__ = [
+    'Limit',
     'NOT_MODIFIED',
+    'PrettyPrint',
+    'Tee',
     'console_run',
     'jupyter_run',
-    'limit',
-    'log',
     'noop',
     'pprint',
-    'tee',
 ]
 
 
@@ -24,7 +24,7 @@ def identity(x):
     return x
 
 
-def limit(n=10):
+def Limit(n=10):
     i = 0
 
     def _limit(*args, **kwargs):
@@ -37,7 +37,7 @@ def limit(n=10):
     return _limit
 
 
-def tee(f):
+def Tee(f):
     @functools.wraps(f)
     def wrapped(*args, **kwargs):
         nonlocal f
@@ -47,10 +47,10 @@ def tee(f):
     return wrapped
 
 
-log = tee(_pprint)
+pprint = Tee(_pprint)
 
 
-def pprint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):
+def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):
     term = blessings.Terminal()
 
     def _pprint(*args, **kwargs):
@@ -78,6 +78,7 @@ def pprint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):
 
 
 '''
+    Old code from rdc.etl
 
     def writehr(self, label=None):
         width = t.width or 80
@@ -113,4 +114,4 @@ def pprint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):
 
 
 def noop(*args, **kwargs):  # pylint: disable=unused-argument
-    pass
+    return NOT_MODIFIED