paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #173 from hartym/develop

Browse Source 
New features and PRs.
This commit is contained in:
Romain Dorgueil
2017-09-30 12:32:10 +02:00
committed by GitHub
parent 213b471237 0eb5fd8a15
commit 532ad11c4c
22 changed files with 813 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -1,7 +1,7 @@
# This file has been auto-generated.
# All changes will be lost, see Projectfile.
#
# Updated at 2017-09-30 09:50:47.806007
# Updated at 2017-09-30 11:26:44.075878
PACKAGE ?= bonobo
PYTHON ?= $(shell which python)

4
Projectfile
View File

@ -29,7 +29,9 @@ python.setup(
'bonobo = bonobo.commands:entrypoint',
],
'bonobo.commands': [
'convert = bonobo.commands.convert:register',
'init = bonobo.commands.init:register',
'inspect = bonobo.commands.inspect:register',
'run = bonobo.commands.run:register',
'version = bonobo.commands.version:register',
],
@ -56,3 +58,5 @@ python.add_requirements(
'ipywidgets >=6.0.0,<7',
]
)
# vim: ft=python:

112
bin/imgcat Executable file
View File

@ -0,0 +1,112 @@
#!/bin/bash
# tmux requires unrecognized OSC sequences to be wrapped with DCS tmux;
# <sequence> ST, and for all ESCs in <sequence> to be replaced with ESC ESC. It
# only accepts ESC backslash for ST.
function print_osc() {
if [[ $TERM == screen* ]] ; then
printf "\033Ptmux;\033\033]"
else
printf "\033]"
fi
}
# More of the tmux workaround described above.
function print_st() {
if [[ $TERM == screen* ]] ; then
printf "\a\033\\"
else
printf "\a"
fi
}
# print_image filename inline base64contents print_filename
# filename: Filename to convey to client
# inline: 0 or 1
# base64contents: Base64-encoded contents
# print_filename: If non-empty, print the filename
# before outputting the image
function print_image() {
print_osc
printf '1337;File='
if [[ -n "$1" ]]; then
printf 'name='`printf "%s" "$1" | base64`";"
fi
VERSION=$(base64 --version 2>&1)
if [[ "$VERSION" =~ fourmilab ]]; then
BASE64ARG=-d
elif [[ "$VERSION" =~ GNU ]]; then
BASE64ARG=-di
else
BASE64ARG=-D
fi
printf "%s" "$3" | base64 $BASE64ARG | wc -c | awk '{printf "size=%d",$1}'
printf ";inline=$2"
printf ":"
printf "%s" "$3"
print_st
printf '\n'
if [[ -n "$4" ]]; then
echo $1
fi
}
function error() {
echo "ERROR: $*" 1>&2
}
function show_help() {
echo "Usage: imgcat [-p] filename ..." 1>& 2
echo " or: cat filename | imgcat" 1>& 2
}
## Main
if [ -t 0 ]; then
has_stdin=f
else
has_stdin=t
fi
# Show help if no arguments and no stdin.
if [ $has_stdin = f -a $# -eq 0 ]; then
show_help
exit
fi
# Look for command line flags.
while [ $# -gt 0 ]; do
case "$1" in
-h|--h|--help)
show_help
exit
;;
-p|--p|--print)
print_filename=1
;;
-*)
error "Unknown option flag: $1"
show_help
exit 1
;;
*)
if [ -r "$1" ] ; then
has_stdin=f
print_image "$1" 1 "$(base64 < "$1")" "$print_filename"
else
error "imgcat: $1: No such file or directory"
exit 2
fi
;;
esac
shift
done
# Read and print stdin
if [ $has_stdin = t ]; then
print_image "" 1 "$(cat | base64)" ""
fi
exit 0

1
bin/test_graph Normal file
View File

@ -0,0 +1 @@
bonobo inspect --graph bonobo/examples/tutorials/tut02e03_writeasmap.py | dot -o test_output.png -T png && bin/imgcat test_output.png

81
bonobo/commands/convert.py Normal file
View File

@ -0,0 +1,81 @@
import mimetypes
import os
import bonobo
SHORTCUTS = {
'csv': 'text/csv',
'json': 'application/json',
'pickle': 'pickle',
'plain': 'text/plain',
'text': 'text/plain',
'txt': 'text/plain',
}
REGISTRY = {
'application/json': (bonobo.JsonReader, bonobo.JsonWriter),
'pickle': (bonobo.PickleReader, bonobo.PickleWriter),
'text/csv': (bonobo.CsvReader, bonobo.CsvWriter),
'text/plain': (bonobo.FileReader, bonobo.FileWriter),
}
READER = 'reader'
WRITER = 'writer'
def resolve_factory(name, filename, factory_type):
"""
Try to resolve which transformation factory to use for this filename. User eventually provided a name, which has
priority, otherwise we try to detect it using the mimetype detection on filename.
"""
if name is None:
name = mimetypes.guess_type(filename)[0]
if name in SHORTCUTS:
name = SHORTCUTS[name]
if name is None:
_, _ext = os.path.splitext(filename)
if _ext:
_ext = _ext[1:]
if _ext in SHORTCUTS:
name = SHORTCUTS[_ext]
if not name in REGISTRY:
raise RuntimeError(
'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} <format>.'.
format(name=name, filename=filename, factory_type=factory_type, opt=factory_type[0])
)
if factory_type == READER:
return REGISTRY[name][0]
elif factory_type == WRITER:
return REGISTRY[name][1]
else:
raise ValueError('Invalid factory type.')
def execute(input, output, reader=None, reader_options=None, writer=None, writer_options=None, options=None):
reader = resolve_factory(reader, input, READER)(input)
writer = resolve_factory(writer, output, WRITER)(output)
graph = bonobo.Graph()
graph.add_chain(reader, writer)
return bonobo.run(
graph, services={
'fs': bonobo.open_fs(),
}
)
def register(parser):
parser.add_argument('input')
parser.add_argument('output')
parser.add_argument('--' + READER, '-r')
parser.add_argument('--' + WRITER, '-w')
# parser.add_argument('--reader-option', '-ro', dest='reader_options')
# parser.add_argument('--writer-option', '-wo', dest='writer_options')
# parser.add_argument('--option', '-o', dest='options')
return execute

34
bonobo/commands/inspect.py Normal file
View File

@ -0,0 +1,34 @@
import json
from bonobo.commands.run import read, register_generic_run_arguments
from bonobo.constants import BEGIN
from bonobo.util.objects import get_name
OUTPUT_GRAPHVIZ = 'graphviz'
def execute(*, output, **kwargs):
graph, plugins, services = read(**kwargs)
if output == OUTPUT_GRAPHVIZ:
print('digraph {')
print(' rankdir = LR;')
print(' "BEGIN" [shape="point"];')
for i in graph.outputs_of(BEGIN):
print(' "BEGIN" -> ' + json.dumps(get_name(graph[i])) + ';')
for ix in graph.topologically_sorted_indexes:
for iy in graph.outputs_of(ix):
print(' {} -> {};'.format(json.dumps(get_name(graph[ix])), json.dumps(get_name(graph[iy]))))
print('}')
else:
raise NotImplementedError('Output type not implemented.')
def register(parser):
register_generic_run_arguments(parser)
parser.add_argument('--graph', '-g', dest='output', action='store_const', const=OUTPUT_GRAPHVIZ)
parser.set_defaults(output=OUTPUT_GRAPHVIZ)
return execute

36
bonobo/commands/run.py
View File

@ -1,7 +1,7 @@
import os
DEFAULT_SERVICES_FILENAME = '_services.py'
DEFAULT_SERVICES_ATTR = 'get_services'
import bonobo
from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME
DEFAULT_GRAPH_FILENAMES = ('__main__.py', 'main.py', )
DEFAULT_GRAPH_ATTR = 'get_graph'
@ -40,9 +40,10 @@ def _install_requirements(requirements):
importlib.reload(site)
def execute(filename, module, install=False, quiet=False, verbose=False):
def read(filename, module, install=False, quiet=False, verbose=False, env=None):
import re
import runpy
from bonobo import Graph, run, settings
from bonobo import Graph, settings
if quiet:
settings.QUIET.set(True)
@ -50,6 +51,12 @@ def execute(filename, module, install=False, quiet=False, verbose=False):
if verbose:
settings.DEBUG.set(True)
if env:
quote_killer = re.compile('["\']')
for e in env:
var_name, var_value = e.split('=')
os.environ[var_name] = quote_killer.sub('', var_value)
if filename:
if os.path.isdir(filename):
if install:
@ -81,17 +88,19 @@ def execute(filename, module, install=False, quiet=False, verbose=False):
).format(len(graphs))
graph = list(graphs.values())[0]
# todo if console and not quiet, then add the console plugin
# todo when better console plugin, add it if console and just disable display
return run(
graph,
plugins=[],
services=get_default_services(
filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
)
plugins = []
services = get_default_services(
filename, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None
)
return graph, plugins, services
def execute(filename, module, install=False, quiet=False, verbose=False, env=None):
graph, plugins, services = read(filename, module, install, quiet, verbose, env)
return bonobo.run(graph, plugins=plugins, services=services)
def register_generic_run_arguments(parser, required=True):
source_group = parser.add_mutually_exclusive_group(required=required)
@ -106,4 +115,5 @@ def register(parser):
verbosity_group.add_argument('--quiet', '-q', action='store_true')
verbosity_group.add_argument('--verbose', '-v', action='store_true')
parser.add_argument('--install', '-I', action='store_true')
parser.add_argument('--env', '-e', action='append')
return execute

2
bonobo/constants.py
View File

@ -4,3 +4,5 @@ BEGIN = Token('Begin')
END = Token('End')
INHERIT_INPUT = Token('InheritInput')
NOT_MODIFIED = Token('NotModified')
DEFAULT_SERVICES_FILENAME = '_services.py'
DEFAULT_SERVICES_ATTR = 'get_services'

214
bonobo/examples/datasets/coffeeshops.txt
View File

@ -1,36 +1,37 @@
les montparnos, 65 boulevard Pasteur, 75015 Paris, France
Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France
Café Lea, 5 rue Claude Bernard, 75005 Paris, France
Le Bellerive, 71 quai de Seine, 75019 Paris, France
Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
Extérieur Quai, 5, rue d'Alsace, 75010 Paris, France
Le Sully, 6 Bd henri IV, 75004 Paris, France
O q de poule, 53 rue du ruisseau, 75018 Paris, France
Le café des amis, 125 rue Blomet, 75015 Paris, France
Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France
La Renaissance, 112 Rue Championnet, 75018 Paris, France
La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France
Le chantereine, 51 Rue Victoire, 75009 Paris, France
Le Müller, 11 rue Feutrier, 75018 Paris, France
Extérieur Quai, 5, rue d'Alsace, 75010 Paris, France
La Bauloise, 36 rue du hameau, 75015 Paris, France
Le Dellac, 14 rue Rougemont, 75009 Paris, France
Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
Le Sully, 6 Bd henri IV, 75004 Paris, France
Le Felteu, 1 rue Pecquay, 75004 Paris, France
Le bistrot de Maëlle et Augustin, 42 rue coquillère, 75001 Paris, France
Dédé la frite, 52 rue Notre-Dame des Victoires, 75002 Paris, France
Cardinal Saint-Germain, 11 boulevard Saint-Germain, 75005 Paris, France
Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France
Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
La Renaissance, 112 Rue Championnet, 75018 Paris, France
Le Square, 31 rue Saint-Dominique, 75007 Paris, France
Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
Café antoine, 17 rue Jean de la Fontaine, 75016 Paris, France
Au cerceau d'or, 129 boulevard sebastopol, 75002 Paris, France
La Caravane, Rue de la Fontaine au Roi, 75011 Paris, France
Le Pas Sage, 1 Passage du Grand Cerf, 75002 Paris, France
Le drapeau de la fidelité, 21 rue Copreaux, 75015 Paris, France
Le café des amis, 125 rue Blomet, 75015 Paris, France
Le Café Livres, 10 rue Saint Martin, 75004 Paris, France
Le Bosquet, 46 avenue Bosquet, 75007 Paris, France
Le Chaumontois, 12 rue Armand Carrel, 75018 Paris, France
Le Kleemend's, 34 avenue Pierre Mendès-France, 75013 Paris, France
Café Pierre, 202 rue du faubourg st antoine, 75012 Paris, France
Les Arcades, 61 rue de Ponthieu, 75008 Paris, France
Le Square, 31 rue Saint-Dominique, 75007 Paris, France
Assaporare Dix sur Dix, 75, avenue Ledru-Rollin, 75012 Paris, France
Au cerceau d'or, 129 boulevard sebastopol, 75002 Paris, France
Aux cadrans, 21 ter boulevard Diderot, 75012 Paris, France
Café antoine, 17 rue Jean de la Fontaine, 75016 Paris, France
Café de la Mairie (du VIII), rue de Lisbonne, 75008 Paris, France
Café Lea, 5 rue Claude Bernard, 75005 Paris, France
Cardinal Saint-Germain, 11 boulevard Saint-Germain, 75005 Paris, France
Dédé la frite, 52 rue Notre-Dame des Victoires, 75002 Paris, France
La Bauloise, 36 rue du hameau, 75015 Paris, France
Le Bellerive, 71 quai de Seine, 75019 Paris, France
Le bistrot de Maëlle et Augustin, 42 rue coquillère, 75001 Paris, France
Le Dellac, 14 rue Rougemont, 75009 Paris, France
Le Felteu, 1 rue Pecquay, 75004 Paris, France
Le Reynou, 2 bis quai de la mégisserie, 75001 Paris, France
Le Saint Jean, 23 rue des abbesses, 75018 Paris, France
les montparnos, 65 boulevard Pasteur, 75015 Paris, France
L'antre d'eux, 16 rue DE MEZIERES, 75006 Paris, France
Drole d'endroit pour une rencontre, 58 rue de Montorgueil, 75002 Paris, France
Le pari's café, 104 rue caulaincourt, 75018 Paris, France
Le Poulailler, 60 rue saint-sabin, 75011 Paris, France
@ -62,7 +63,6 @@ Denfert café, 58 boulvevard Saint Jacques, 75014 Paris, France
Le Café frappé, 95 rue Montmartre, 75002 Paris, France
La Perle, 78 rue vieille du temple, 75003 Paris, France
Le Descartes, 1 rue Thouin, 75005 Paris, France
Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
Le petit club, 55 rue de la tombe Issoire, 75014 Paris, France
Le Plein soleil, 90 avenue Parmentier, 75011 Paris, France
Le Relais Haussmann, 146, boulevard Haussmann, 75008 Paris, France
@ -75,7 +75,6 @@ Extra old café, 307 fg saint Antoine, 75011 Paris, France
Chez Fafa, 44 rue Vinaigriers, 75010 Paris, France
En attendant l'or, 3 rue Faidherbe, 75011 Paris, France
Brûlerie San José, 30 rue des Petits-Champs, 75002 Paris, France
Café de la Mairie (du VIII), rue de Lisbonne, 75008 Paris, France
Café Martin, 2 place Martin Nadaud, 75001 Paris, France
Etienne, 14 rue Turbigo, Paris, 75001 Paris, France
L'ingénu, 184 bd Voltaire, 75011 Paris, France
@ -87,96 +86,97 @@ Le Germinal, 95 avenue Emile Zola, 75015 Paris, France
Le Ragueneau, 202 rue Saint-Honoré, 75001 Paris, France
Le refuge, 72 rue lamarck, 75018 Paris, France
Le sully, 13 rue du Faubourg Saint Denis, 75010 Paris, France
Coffee Chope, 344Vrue Vaugirard, 75015 Paris, France
Le bal du pirate, 60 rue des bergers, 75015 Paris, France
zic zinc, 95 rue claude decaen, 75012 Paris, France
l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
L'Inévitable, 22 rue Linné, 75005 Paris, France
Le Dunois, 77 rue Dunois, 75013 Paris, France
La Montagne Sans Geneviève, 13 Rue du Pot de Fer, 75005 Paris, France
Ragueneau, 202 rue Saint Honoré, 75001 Paris, France
Le Caminito, 48 rue du Dessous des Berges, 75013 Paris, France
Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France
Le petit Bretonneau, Le petit Bretonneau - à l'intérieur de l'Hôpital, 75018 Paris, France
Le Centenaire, 104 rue amelot, 75011 Paris, France
La Montagne Sans Geneviève, 13 Rue du Pot de Fer, 75005 Paris, France
Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France
Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
La chaumière gourmande, Route de la Muette à Neuilly
Club hippique du Jardin d’Acclimatation, 75016 Paris, France
Le bal du pirate, 60 rue des bergers, 75015 Paris, France
Le Zazabar, 116 Rue de Ménilmontant, 75020 Paris, France
L'antre d'eux, 16 rue DE MEZIERES, 75006 Paris, France
l'orillon bar, 35 rue de l'orillon, 75011 Paris, France
zic zinc, 95 rue claude decaen, 75012 Paris, France
Les Pères Populaires, 46 rue de Buzenval, 75020 Paris, France
Epicerie Musicale, 55bis quai de Valmy, 75010 Paris, France
Le relais de la victoire, 73 rue de la Victoire, 75009 Paris, France
Le Centenaire, 104 rue amelot, 75011 Paris, France
Cafe de grenelle, 188 rue de Grenelle, 75007 Paris, France
Ragueneau, 202 rue Saint Honoré, 75001 Paris, France
Le Brio, 216, rue Marcadet, 75018 Paris, France
Caves populaires, 22 rue des Dames, 75017 Paris, France
Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
Tamm Bara, 7 rue Clisson, 75013 Paris, France
L'anjou, 1 rue de Montholon, 75009 Paris, France
Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France
Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
Café Pistache, 9 rue des petits champs, 75001 Paris, France
La Cagnotte, 13 Rue Jean-Baptiste Dumay, 75020 Paris, France
Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
Café beauveau, 9 rue de Miromesnil, 75008 Paris, France
le 1 cinq, 172 rue de vaugirard, 75015 Paris, France
Le Killy Jen, 28 bis boulevard Diderot, 75012 Paris, France
Les Artisans, 106 rue Lecourbe, 75015 Paris, France
Peperoni, 83 avenue de Wagram, 75001 Paris, France
Le Brio, 216, rue Marcadet, 75018 Paris, France
Tamm Bara, 7 rue Clisson, 75013 Paris, France
Café dans l'aerogare Air France Invalides, 2 rue Robert Esnault Pelterie, 75007 Paris, France
bistrot les timbrés, 14 rue d'alleray, 75015 Paris, France
Caprice café, 12 avenue Jean Moulin, 75014 Paris, France
Caves populaires, 22 rue des Dames, 75017 Paris, France
Au Vin Des Rues, 21 rue Boulard, 75014 Paris, France
Chez Prune, 36 rue Beaurepaire, 75010 Paris, France
L'Inévitable, 22 rue Linné, 75005 Paris, France
L'anjou, 1 rue de Montholon, 75009 Paris, France
Botak cafe, 1 rue Paul albert, 75018 Paris, France
Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
maison du vin, 52 rue des plantes, 75014 Paris, France
Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France
La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France
Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France
Les caves populaires, 22 rue des Dames, 75017 Paris, France
Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
La Brocante, 10 rue Rossini, 75009 Paris, France
Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France
La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
Café Clochette, 16 avenue Richerand, 75010 Paris, France
le lutece, 380 rue de vaugirard, 75015 Paris, France
Brasiloja, 16 rue Ganneron, 75018 Paris, France
Rivolux, 16 rue de Rivoli, 75004 Paris, France
L'européen, 21 Bis Boulevard Diderot, 75012 Paris, France
NoMa, 39 rue Notre Dame de Nazareth, 75003 Paris, France
le lutece, 380 rue de vaugirard, 75015 Paris, France
O'Paris, 1 Rue des Envierges, 75020 Paris, France
Rivolux, 16 rue de Rivoli, 75004 Paris, France
Brasiloja, 16 rue Ganneron, 75018 Paris, France
Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France
L'Angle, 28 rue de Ponthieu, 75008 Paris, France
Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France
Café Dupont, 198 rue de la Convention, 75015 Paris, France
Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France
L'Entracte, place de l'opera, 75002 Paris, France
Panem, 18 rue de Crussol, 75011 Paris, France
Au pays de Vannes, 34 bis rue de Wattignies, 75012 Paris, France
l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France
L'âge d'or, 26 rue du Docteur Magnan, 75013 Paris, France
Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
L'horizon, 93, rue de la Roquette, 75011 Paris, France
L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
Café Victor, 10 boulevard Victor, 75015 Paris, France
Café Varenne, 36 rue de Varenne, 75007 Paris, France
Le Brigadier, 12 rue Blanche, 75009 Paris, France
Waikiki, 10 rue d"Ulm, 75005 Paris, France
Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France
Pari's Café, 174 avenue de Clichy, 75017 Paris, France
Melting Pot, 3 rue de Lagny, 75020 Paris, France
le Zango, 58 rue Daguerre, 75014 Paris, France
Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
Café Clochette, 16 avenue Richerand, 75010 Paris, France
La cantoche de Paname, 40 Boulevard Beaumarchais, 75011 Paris, France
Le Saint René, 148 Boulevard de Charonne, 75020 Paris, France
La Liberté, 196 rue du faubourg saint-antoine, 75012 Paris, France
Chez Rutabaga, 16 rue des Petits Champs, 75002 Paris, France
Le BB (Bouchon des Batignolles), 2 rue Lemercier, 75017 Paris, France
La Brocante, 10 rue Rossini, 75009 Paris, France
Le Plomb du cantal, 3 rue Gaîté, 75014 Paris, France
Les caves populaires, 22 rue des Dames, 75017 Paris, France
Chez Luna, 108 rue de Ménilmontant, 75020 Paris, France
Le bar Fleuri, 1 rue du Plateau, 75019 Paris, France
Trois pièces cuisine, 101 rue des dames, 75017 Paris, France
Le Zinc, 61 avenue de la Motte Picquet, 75015 Paris, France
La cantine de Zoé, 136 rue du Faubourg poissonnière, 75010 Paris, France
Les Vendangeurs, 6/8 rue Stanislas, 75006 Paris, France
L'avant comptoir, 3 carrefour de l'Odéon, 75006 Paris, France
Botak cafe, 1 rue Paul albert, 75018 Paris, France
le chateau d'eau, 67 rue du Château d'eau, 75010 Paris, France
Bistrot Saint-Antoine, 58 rue du Fbg Saint-Antoine, 75012 Paris, France
Chez Oscar, 11/13 boulevard Beaumarchais, 75004 Paris, France
Le Fronton, 63 rue de Ponthieu, 75008 Paris, France
Le Piquet, 48 avenue de la Motte Picquet, 75015 Paris, France
Le Tournebride, 104 rue Mouffetard, 75005 Paris, France
maison du vin, 52 rue des plantes, 75014 Paris, France
L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France
Le café Monde et Médias, Place de la République, 75003 Paris, France
Café rallye tournelles, 11 Quai de la Tournelle, 75005 Paris, France
Brasserie le Morvan, 61 rue du château d'eau, 75010 Paris, France
L'entrepôt, 157 rue Bercy 75012 Paris, 75012 Paris, France
Chez Miamophile, 6 rue Mélingue, 75019 Paris, France
Panem, 18 rue de Crussol, 75011 Paris, France
Petits Freres des Pauvres, 47 rue de Batignolles, 75017 Paris, France
Café Dupont, 198 rue de la Convention, 75015 Paris, France
L'Angle, 28 rue de Ponthieu, 75008 Paris, France
Institut des Cultures d'Islam, 19-23 rue Léon, 75018 Paris, France
Canopy Café associatif, 19 rue Pajol, 75018 Paris, France
L'Entracte, place de l'opera, 75002 Paris, France
Le Sévigné, 15 rue du Parc Royal, 75003 Paris, France
Le Café d'avant, 35 rue Claude Bernard, 75005 Paris, France
Le Lucernaire, 53 rue Notre-Dame des Champs, 75006 Paris, France
Le Brigadier, 12 rue Blanche, 75009 Paris, France
L'âge d'or, 26 rue du Docteur Magnan, 75013 Paris, France
Bagels & Coffee Corner, Place de Clichy, 75017 Paris, France
Café Victor, 10 boulevard Victor, 75015 Paris, France
L'empreinte, 54, avenue Daumesnil, 75012 Paris, France
L'horizon, 93, rue de la Roquette, 75011 Paris, France
Waikiki, 10 rue d"Ulm, 75005 Paris, France
Au pays de Vannes, 34 bis rue de Wattignies, 75012 Paris, France
Café Varenne, 36 rue de Varenne, 75007 Paris, France
l'Eléphant du nil, 125 Rue Saint-Antoine, 75004 Paris, France
Le Comptoir, 354 bis rue Vaugirard, 75015 Paris, France
Le Parc Vaugirard, 358 rue de Vaugirard, 75015 Paris, France
le Zango, 58 rue Daguerre, 75014 Paris, France
Melting Pot, 3 rue de Lagny, 75020 Paris, France
Pari's Café, 174 avenue de Clichy, 75017 Paris, France

7
bonobo/examples/datasets/fablabs.py
View File

@ -48,11 +48,6 @@ def normalize(row):
return result
def filter_france(row):
if row.get('country') == 'France':
yield row
def display(row):
print(Style.BRIGHT, row.get('name'), Style.RESET_ALL, sep='')
@ -95,7 +90,7 @@ graph = bonobo.Graph(
dataset=API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris'
),
normalize,
filter_france,
bonobo.Filter(filter=lambda row: row.get('country') == 'France'),
bonobo.JsonWriter(path='fablabs.txt', ioformat='arg0'),
bonobo.Tee(display),
)

41
bonobo/examples/nodes/bags.py Normal file
View File

@ -0,0 +1,41 @@
"""
Example on how to use :class:`bonobo.Bag` instances to pass flexible args/kwargs to the next callable.
.. graphviz::
digraph {
rankdir = LR;
stylesheet = "../_static/graphs.css";
BEGIN [shape="point"];
BEGIN -> "extract()" -> "transform(...)" -> "load(...)";
}
"""
from random import randint
from bonobo import Bag, Graph
def extract():
yield Bag(topic='foo')
yield Bag(topic='bar')
yield Bag(topic='baz')
def transform(topic: str):
return Bag.inherit(title=topic.title(), rand=randint(10, 99))
def load(topic: str, title: str, rand: int):
print('{} ({}) wait={}'.format(title, topic, rand))
graph = Graph()
graph.add_chain(extract, transform, load)
if __name__ == '__main__':
from bonobo import run
run(graph)

43
bonobo/examples/nodes/dicts.py Normal file
View File

@ -0,0 +1,43 @@
"""
Example on how to use symple python dictionaries to communicate between transformations.
.. graphviz::
digraph {
rankdir = LR;
stylesheet = "../_static/graphs.css";
BEGIN [shape="point"];
BEGIN -> "extract()" -> "transform(row: dict)" -> "load(row: dict)";
}
"""
from random import randint
from bonobo import Graph
def extract():
yield {'topic': 'foo'}
yield {'topic': 'bar'}
yield {'topic': 'baz'}
def transform(row: dict):
return {
'topic': row['topic'].title(),
'randint': randint(10, 99),
}
def load(row: dict):
print(row)
graph = Graph(extract, transform, load)
if __name__ == '__main__':
from bonobo import run
run(graph)

18
bonobo/examples/nodes/factory.py Normal file
View File

@ -0,0 +1,18 @@
import bonobo
from bonobo.commands.run import get_default_services
from bonobo.nodes.factory import Factory
from bonobo.nodes.io.json import JsonDictItemsReader
normalize = Factory()
normalize[0].str().title()
normalize.move(0, 'title')
normalize.move(0, 'address')
graph = bonobo.Graph(
JsonDictItemsReader('datasets/coffeeshops.json'),
normalize,
bonobo.PrettyPrinter(),
)
if __name__ == '__main__':
bonobo.run(graph, services=get_default_services(__file__))

39
bonobo/examples/nodes/strings.py Normal file
View File

@ -0,0 +1,39 @@
"""
Example on how to use symple python strings to communicate between transformations.
.. graphviz::
digraph {
rankdir = LR;
stylesheet = "../_static/graphs.css";
BEGIN [shape="point"];
BEGIN -> "extract()" -> "transform(s: str)" -> "load(s: str)";
}
"""
from random import randint
from bonobo import Graph
def extract():
yield 'foo'
yield 'bar'
yield 'baz'
def transform(s: str):
return '{} ({})'.format(s.title(), randint(10, 99))
def load(s: str):
print(s)
graph = Graph(extract, transform, load)
if __name__ == '__main__':
from bonobo import run
run(graph)

2
bonobo/ext/console.py
View File

@ -81,7 +81,7 @@ class ConsoleOutputPlugin(Plugin):
print(line + CLEAR_EOL, file=sys.stderr)
alive_color = Style.BRIGHT
dead_color = (Style.BRIGHT + Fore.BLACK) if self.iswindows else Fore.BLACK
dead_color = Style.BRIGHT + Fore.BLACK
for i in context.graph.topologically_sorted_indexes:
node = context[i]

219
bonobo/nodes/factory.py Normal file
View File

@ -0,0 +1,219 @@
import functools
import warnings
from functools import partial
from bonobo import Bag
from bonobo.config import Configurable, Method
_isarg = lambda item: type(item) is int
_iskwarg = lambda item: type(item) is str
class Operation():
def __init__(self, item, callable):
self.item = item
self.callable = callable
def __repr__(self):
return '<operation {} on {}>'.format(self.callable.__name__, self.item)
def apply(self, *args, **kwargs):
if _isarg(self.item):
return (*args[0:self.item], self.callable(args[self.item]), *args[self.item + 1:]), kwargs
if _iskwarg(self.item):
return args, {**kwargs, self.item: self.callable(kwargs.get(self.item))}
raise RuntimeError('Houston, we have a problem...')
class FactoryOperation():
def __init__(self, factory, callable):
self.factory = factory
self.callable = callable
def __repr__(self):
return '<factory operation {}>'.format(self.callable.__name__)
def apply(self, *args, **kwargs):
return self.callable(*args, **kwargs)
CURSOR_TYPES = {}
def operation(mixed):
def decorator(m, ctype=mixed):
def lazy_operation(self, *args, **kwargs):
@functools.wraps(m)
def actual_operation(x):
return m(self, x, *args, **kwargs)
self.factory.operations.append(Operation(self.item, actual_operation))
return CURSOR_TYPES[ctype](self.factory, self.item) if ctype else self
return lazy_operation
return decorator if isinstance(mixed, str) else decorator(mixed, ctype=None)
def factory_operation(m):
def lazy_operation(self, *config):
@functools.wraps(m)
def actual_operation(*args, **kwargs):
return m(self, *config, *args, **kwargs)
self.operations.append(FactoryOperation(self, actual_operation))
return self
return lazy_operation
class Cursor():
_type = None
def __init__(self, factory, item):
self.factory = factory
self.item = item
@operation('dict')
def dict(self, x):
return x if isinstance(x, dict) else dict(x)
@operation('int')
def int(self):
pass
@operation('str')
def str(self, x):
return x if isinstance(x, str) else str(x)
@operation('list')
def list(self):
pass
@operation('tuple')
def tuple(self):
pass
def __getattr__(self, item):
"""
Fallback to type methods if they exist, for example StrCursor.upper will use str.upper if not overriden, etc.
:param item:
"""
if self._type and item in self._type.__dict__:
method = self._type.__dict__[item]
@operation
@functools.wraps(method)
def _operation(self, x, *args, **kwargs):
return method(x, *args, **kwargs)
setattr(self, item, partial(_operation, self))
return getattr(self, item)
raise AttributeError('Unknown operation {}.{}().'.format(
type(self).__name__,
item,
))
CURSOR_TYPES['default'] = Cursor
class DictCursor(Cursor):
_type = dict
@operation('default')
def get(self, x, path):
return x.get(path)
@operation
def map_keys(self, x, mapping):
return {mapping.get(k): v for k, v in x.items()}
CURSOR_TYPES['dict'] = DictCursor
class StringCursor(Cursor):
_type = str
CURSOR_TYPES['str'] = StringCursor
class Factory(Configurable):
initialize = Method(required=False)
def __init__(self, *args, **kwargs):
warnings.warn(
__file__ +
' is experimental, API may change in the future, use it as a preview only and knowing the risks.',
FutureWarning
)
super(Factory, self).__init__(*args, **kwargs)
self.default_cursor_type = 'default'
self.operations = []
if self.initialize is not None:
self.initialize(self)
@factory_operation
def move(self, _from, _to, *args, **kwargs):
if _from == _to:
return args, kwargs
if _isarg(_from):
value = args[_from]
args = args[:_from] + args[_from + 1:]
elif _iskwarg(_from):
value = kwargs[_from]
kwargs = {k: v for k, v in kwargs if k != _from}
else:
raise RuntimeError('Houston, we have a problem...')
if _isarg(_to):
return (*args[:_to], value, *args[_to + 1:]), kwargs
elif _iskwarg(_to):
return args, {**kwargs, _to: value}
else:
raise RuntimeError('Houston, we have a problem...')
def __call__(self, *args, **kwargs):
print('factory call on', args, kwargs)
for operation in self.operations:
args, kwargs = operation.apply(*args, **kwargs)
print(' ... after', operation, 'got', args, kwargs)
return Bag(*args, **kwargs)
def __getitem__(self, item):
return CURSOR_TYPES[self.default_cursor_type](self, item)
if __name__ == '__main__':
f = Factory()
f[0].dict().map_keys({'foo': 'F00'})
f['foo'].str().upper()
print('operations:', f.operations)
print(f({'foo': 'bisou'}, foo='blah'))
'''
specs:
- rename keys of an input dict (in args, or kwargs) using a translation map.
f = Factory()
f[0]
f['xxx'] =
f[0].dict().get('foo.bar').move_to('foo.baz').apply(str.upper)
f[0].get('foo.*').items().map(str.lower)
f['foo'].keys_map({
'a': 'b'
})
'''

7
bonobo/nodes/io/json.py
View File

@ -4,6 +4,7 @@ from bonobo.config.processors import ContextProcessor
from bonobo.constants import NOT_MODIFIED
from bonobo.nodes.io.base import FileHandler, IOFormatEnabled
from bonobo.nodes.io.file import FileReader, FileWriter
from bonobo.structs.bags import Bag
class JsonHandler(FileHandler):
@ -19,6 +20,12 @@ class JsonReader(IOFormatEnabled, FileReader, JsonHandler):
yield self.get_output(line)
class JsonDictItemsReader(JsonReader):
def read(self, fs, file):
for line in self.loader(file).items():
yield Bag(*line)
class JsonWriter(IOFormatEnabled, FileWriter, JsonHandler):
@ContextProcessor
def envelope(self, context, fs, file, lineno):

3
bonobo/nodes/throttle.py
View File

@ -41,15 +41,12 @@ class RateLimited(Configurable):
@ContextProcessor
def bucket(self, context):
print(context)
bucket = RateLimitBucket(self.initial, self.amount, self.period)
bucket.start()
print(bucket)
yield bucket
bucket.stop()
bucket.join()
def call(self, bucket, *args, **kwargs):
print(bucket, args, kwargs)
bucket.wait()
return self.handler(*args, **kwargs)

28
docs/reference/commands.rst
View File

@ -1,6 +1,21 @@
Command-line
============
Bonobo Convert
::::::::::::::
Build a simple bonobo graph with one reader and one writer, then execute it, allowing to use bonobo in "no code" mode
for simple file format conversions.
Syntax: `bonobo convert [-r reader] input_filename [-w writer] output_filename`
.. todo::
add a way to override default options of reader/writers, add a way to add "filters", for example this could be used
to read from csv and write to csv too (or other format) but adding a geocoder filter that would add some fields.
Bonobo Init
:::::::::::
@ -8,7 +23,17 @@ Create an empty project, ready to use bonobo.
Syntax: `bonobo init`
Requires `edgy.project`.
Requires `cookiecutter`.
Bonobo Inspect
::::::::::::::
Inspects a bonobo graph source files. For now, only support graphviz output.
Syntax: `bonobo inspect [--graph|-g] filename`
Requires graphviz if you want to generate an actual graph picture, although the command itself depends on nothing.
Bonobo Run
@ -20,6 +45,7 @@ Syntax: `bonobo run [-c cmd | -m mod | file | -] [arg]`
.. todo:: implement -m, check if -c is of any use and if yes, implement it too. Implement args, too.
Bonobo RunC
:::::::::::

3
setup.py
View File

@ -67,7 +67,8 @@ setup(
},
entry_points={
'bonobo.commands': [
'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register',
'convert = bonobo.commands.convert:register', 'init = bonobo.commands.init:register',
'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register',
'version = bonobo.commands.version:register'
],
'console_scripts': ['bonobo = bonobo.commands:entrypoint']

28
tests/test_commands.py
View File

@ -3,6 +3,7 @@ import runpy
import sys
from unittest.mock import patch
import pathlib
import pkg_resources
import pytest
@ -96,3 +97,30 @@ def test_version(runner, capsys):
out = out.strip()
assert out.startswith('bonobo ')
assert __version__ in out
@all_runners
def test_run_with_env(runner, capsys):
runner(
'run', '--quiet',
str(pathlib.Path(os.path.dirname(__file__), 'util', 'get_passed_env.py')), '--env', 'ENV_TEST_NUMBER=123',
'--env', 'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
@all_runners
def test_run_module_with_env(runner, capsys):
runner(
'run', '--quiet', '-m', 'tests.util.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env',
'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
)
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'

22
tests/util/get_passed_env.py Normal file
View File

@ -0,0 +1,22 @@
import os
from bonobo import Graph
def extract():
env_test_user = os.getenv('ENV_TEST_USER')
env_test_number = os.getenv('ENV_TEST_NUMBER')
env_test_string = os.getenv('ENV_TEST_STRING')
return env_test_user, env_test_number, env_test_string
def load(s: str):
print(s)
graph = Graph(extract, load)
if __name__ == '__main__':
from bonobo import run
run(graph)