From 3d0855ed23f882766a63b2de5a5eb9050afc2fc4 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 08:35:50 +0200 Subject: [PATCH 01/15] Update project management artifacts --- Makefile | 7 +++-- setup.py | 95 +++++++++++++++++++++++++++++++++----------------------- 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index a9c2888..887a67e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-04-30 10:12:39.793759 +# Updated at 2017-05-01 08:35:15.162008 PYTHON ?= $(shell which python) PYTHON_BASENAME ?= $(shell basename $(PYTHON)) @@ -10,6 +10,7 @@ PYTHON_REQUIREMENTS_DEV_FILE ?= requirements-dev.txt QUICK ?= VIRTUAL_ENV ?= .virtualenv-$(PYTHON_BASENAME) PIP ?= $(VIRTUAL_ENV)/bin/pip +PIP_INSTALL_OPTIONS ?= PYTEST ?= $(VIRTUAL_ENV)/bin/pytest PYTEST_OPTIONS ?= --capture=no --cov=bonobo --cov-report html SPHINX_OPTS ?= @@ -24,13 +25,13 @@ YAPF_OPTIONS ?= -rip # Installs the local project dependencies. install: $(VIRTUAL_ENV) if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel -r $(PYTHON_REQUIREMENTS_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \ fi # Installs the local project dependencies, including development-only libraries. install-dev: $(VIRTUAL_ENV) if [ -z "$(QUICK)" ]; then \ - $(PIP) install -U pip wheel -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ + $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \ fi # Cleans up the local mess. diff --git a/setup.py b/setup.py index 0f2d0b8..83bd632 100644 --- a/setup.py +++ b/setup.py @@ -8,59 +8,76 @@ root_dir = os.path.dirname(os.path.abspath(__file__)) tolines = lambda c: list(filter(None, map(lambda s: s.strip(), c.split('\n')))) + def read(filename, flt=None): try: - with open(filename) as f: + with open(filename, 'rt') as f: content = f.read().strip() return flt(content) if callable(flt) else content - except FileNotFoundError: + except EnvironmentError: + # File not found? Let's say it's empty. return '' + except UnicodeError: + # Problem decoding the file? Let's not stop on this (but it's a temp fix). + return '' + # Py3 compatibility hacks, borrowed from IPython. try: execfile except NameError: + def execfile(fname, globs, locs=None): locs = locs or globs exec(compile(open(fname).read(), fname, "exec"), globs, locs) + version_ns = {} -execfile(os.path.join(root_dir, 'bonobo/_version.py'), version_ns) -version = version_ns.get('__version__', 'dev') +try: + execfile(os.path.join(root_dir, 'bonobo/_version.py'), version_ns) +except EnvironmentError: + version = 'dev' +else: + version = version_ns.get('__version__', 'dev') setup( - name = 'bonobo', - description = 'Bonobo', - license = 'Apache License, Version 2.0', - install_requires = ['colorama >=0.3,<1.0', - 'fs >=2.0,<3.0', - 'psutil >=5.2,<6.0', - 'requests >=2.0,<3.0', - 'stevedore >=1.21,<2.0'], - version = version, - long_description = read('README.rst'), - classifiers = read('classifiers.txt', tolines), - packages = find_packages(exclude=['ez_setup', 'example', 'test']), - include_package_data = True, - data_files = [('share/jupyter/nbextensions/bonobo-jupyter', - ['bonobo/ext/jupyter/static/extension.js', - 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map'])], - extras_require = {'dev': ['coverage >=4,<5', - 'pylint >=1,<2', - 'pytest >=3,<4', - 'pytest-cov >=2,<3', - 'pytest-timeout >=1,<2', - 'sphinx', - 'sphinx_rtd_theme', - 'yapf'], - 'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5']}, - entry_points = {'bonobo.commands': ['init = bonobo.commands.init:register', - 'run = bonobo.commands.run:register', - 'version = bonobo.commands.version:register'], - 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], - 'edgy.project.features': ['bonobo = ' - 'bonobo.ext.edgy.project.feature:BonoboFeature']}, - url = 'https://www.bonobo-project.org/', - download_url = 'https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), -) + name='bonobo', + description='Bonobo', + license='Apache License, Version 2.0', + install_requires=[ + 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', + 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0' + ], + version=version, + long_description=read('README.rst'), + classifiers=read('classifiers.txt', tolines), + packages=find_packages(exclude=['ez_setup', 'example', 'test']), + include_package_data=True, + data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/ext/jupyter/static/extension.js', + 'bonobo/ext/jupyter/static/index.js', + 'bonobo/ext/jupyter/static/index.js.map' + ])], + extras_require={ + 'dev': [ + 'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', + 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx', + 'sphinx_rtd_theme', 'yapf' + ], + 'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5'] + }, + entry_points={ + 'bonobo.commands': [ + 'init = bonobo.commands.init:register', + 'run = bonobo.commands.run:register', + 'version = bonobo.commands.version:register' + ], + 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], + 'edgy.project.features': + ['bonobo = ' + 'bonobo.ext.edgy.project.feature:BonoboFeature'] + }, + url='https://www.bonobo-project.org/', + download_url= + 'https://github.com/python-bonobo/bonobo/tarball/{version}'.format( + version=version), ) From d79ca9e20ea3261b1b7a2191bac499abad4d61d8 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 09:00:07 +0200 Subject: [PATCH 02/15] Cleaner api exposition. --- bonobo/__init__.py | 3 +- bonobo/_api.py | 96 ++++++++++++++++------------ bonobo/basics.py | 6 +- bonobo/examples/datasets/fablabs.txt | 6 +- 4 files changed, 64 insertions(+), 47 deletions(-) diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 6123b9a..11e17a1 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -11,6 +11,7 @@ import sys assert (sys.version_info >= (3, 5)), 'Python 3.5+ is required to use Bonobo.' from bonobo._api import * from bonobo._api import __all__ +from bonobo._version import __version__ -__all__ = __all__ +__all__ = ['__version__'] + __all__ del sys diff --git a/bonobo/_api.py b/bonobo/_api.py index 70d033f..26ff253 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -1,49 +1,51 @@ -from bonobo._version import __version__ - -__all__ = [ - '__version__', -] - -from bonobo.structs import Bag, Graph - -__all__ += ['Bag', 'Graph'] - -# Filesystem. This is a shortcut from the excellent filesystem2 library, that we make available there for convenience. -from fs import open_fs as _open_fs - -open_fs = lambda url, *args, **kwargs: _open_fs(str(url), *args, **kwargs) -__all__ += ['open_fs'] - -# Basic transformations. -from bonobo.basics import * -from bonobo.basics import __all__ as _all_basics - -__all__ += _all_basics - -# Execution strategies. +from bonobo.basics import Limit, PrettyPrint, Tee, count, identity, noop, pprint from bonobo.strategies import create_strategy +from bonobo.structs import Bag, Graph +from bonobo.util.objects import get_name +from bonobo.io import CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter -__all__ += ['create_strategy'] - -# Extract and loads from stdlib. -from bonobo.io import * -from bonobo.io import __all__ as _all_io - -__all__ += _all_io +__all__ = [] -# XXX This may be belonging to the bonobo.examples package. -def get_examples_path(*pathsegments): - import os - import pathlib - return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments)) +def register_api(x, __all__=__all__): + __all__.append(get_name(x)) + return x + +def register_api_group(*args): + for attr in args: + register_api(attr) -def open_examples_fs(*pathsegments): - return open_fs(get_examples_path(*pathsegments)) +# bonobo.basics +register_api_group(Limit, PrettyPrint, Tee, count, identity, noop, pprint, ) + +# bonobo.io +register_api_group(CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter) + +# bonobo.strategies +register_api(create_strategy) + +# bonobo.structs +register_api_group(Bag, Graph) + +# Shortcut to filesystem2's open_fs, that we make available there for convenience. +@register_api +def open_fs(fs_url, *args, **kwargs): + """ + Wraps :func:`fs.open_fs` function with a few candies. + + :param str fs_url: A filesystem URL + :param parse_result: A parsed filesystem URL. + :type parse_result: :class:`ParseResult` + :param bool writeable: True if the filesystem must be writeable. + :param bool create: True if the filesystem should be created if it does not exist. + :param str cwd: The current working directory (generally only relevant for OS filesystems). + :param str default_protocol: The protocol to use if one is not supplied in the FS URL (defaults to ``"osfs"``). + :returns: :class:`~fs.base.FS` object + """ + from fs import open_fs as _open_fs + return _open_fs(str(fs_url), *args, **kwargs) -__all__.append(get_examples_path.__name__) -__all__.append(open_examples_fs.__name__) def _is_interactive_console(): @@ -58,7 +60,7 @@ def _is_jupyter_notebook(): return False -# @api +@register_api def run(graph, *chain, strategy=None, plugins=None, services=None): if len(chain): warnings.warn('DEPRECATED. You should pass a Graph instance instead of a chain.') @@ -81,4 +83,16 @@ def run(graph, *chain, strategy=None, plugins=None, services=None): return strategy.execute(graph, plugins=plugins, services=services) -__all__.append(run.__name__) +@register_api +def get_examples_path(*pathsegments): + import os + import pathlib + return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments)) + + +@register_api +def open_examples_fs(*pathsegments): + return open_fs(get_examples_path(*pathsegments)) + + +print(__all__) diff --git a/bonobo/basics.py b/bonobo/basics.py index 9709976..6fd8c3d 100644 --- a/bonobo/basics.py +++ b/bonobo/basics.py @@ -4,7 +4,6 @@ from pprint import pprint as _pprint from colorama import Fore, Style from bonobo.config.processors import contextual -from bonobo.constants import NOT_MODIFIED from bonobo.structs.bags import Bag from bonobo.util.objects import ValueHolder from bonobo.util.term import CLEAR_EOL @@ -25,6 +24,7 @@ def identity(x): def Limit(n=10): + from bonobo.constants import NOT_MODIFIED i = 0 def _limit(*args, **kwargs): @@ -38,6 +38,8 @@ def Limit(n=10): def Tee(f): + from bonobo.constants import NOT_MODIFIED + @functools.wraps(f) def wrapped(*args, **kwargs): nonlocal f @@ -63,6 +65,7 @@ pprint = Tee(_pprint) def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True): + from bonobo.constants import NOT_MODIFIED def _pprint(*args, **kwargs): nonlocal title_keys, sort, print_values @@ -98,4 +101,5 @@ def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True def noop(*args, **kwargs): # pylint: disable=unused-argument + from bonobo.constants import NOT_MODIFIED return NOT_MODIFIED diff --git a/bonobo/examples/datasets/fablabs.txt b/bonobo/examples/datasets/fablabs.txt index 00ee95e..9333578 100644 --- a/bonobo/examples/datasets/fablabs.txt +++ b/bonobo/examples/datasets/fablabs.txt @@ -1,5 +1,4 @@ -[ -{"city": "Lannion", "kind_name": "fab_lab", "links": ["http://fablab-lannion.org"], "capabilities": "three_d_printing;cnc_milling;circuit_production", "url": "https://www.fablabs.io/labs/fablablannion", "coordinates": [48.7317261, -3.4509764], "name": "Fablab Lannion - KerNEL", "phone": "+33 2 96 37 84 46", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/13/27/c6c015ba-26c6-4620-833f-8441123a4afc/Fablab Lannion - KerNEL.jpg", "postal_code": "22300", "longitude": -3.45097639999994, "country_code": "fr", "latitude": 48.7317261, "address_notes": "Use the small portal", "email": "contact@fablab-lannion.org", "address_1": "14 Rue de Beauchamp", "geometry": {"type": "Point", "coordinates": [-3.4509764, 48.7317261]}, "country": "France"}, +[{"city": "Lannion", "kind_name": "fab_lab", "links": ["http://fablab-lannion.org"], "capabilities": "three_d_printing;cnc_milling;circuit_production", "url": "https://www.fablabs.io/labs/fablablannion", "coordinates": [48.7317261, -3.4509764], "name": "Fablab Lannion - KerNEL", "phone": "+33 2 96 37 84 46", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/13/27/c6c015ba-26c6-4620-833f-8441123a4afc/Fablab Lannion - KerNEL.jpg", "postal_code": "22300", "longitude": -3.45097639999994, "country_code": "fr", "latitude": 48.7317261, "address_notes": "Use the small portal", "email": "contact@fablab-lannion.org", "address_1": "14 Rue de Beauchamp", "geometry": {"type": "Point", "coordinates": [-3.4509764, 48.7317261]}, "country": "France"}, {"city": "Villeneuve-d'Ascq", "kind_name": "fab_lab", "links": ["http://www.flickr.com/photos/fablablille/", "https://twitter.com/FabLab_Lille", "http://www.fablablille.fr"], "url": "https://www.fablabs.io/labs/fablablille", "coordinates": [50.642869867, 3.1386641], "county": "Nord-Pas-de-Calais", "phone": "+33 9 72 29 47 65", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/10/34/147c88ca-2acd-42a4-aeb0-17b2dc830903/FabLab Lille.jpg", "postal_code": "59650", "longitude": 3.13866410000003, "country_code": "fr", "latitude": 50.6428698670218, "address_1": "2 All\u00e9e Lakanal", "name": "FabLab Lille", "geometry": {"type": "Point", "coordinates": [3.1386641, 50.642869867]}, "country": "France"}, {"city": "Dijon", "name": "L'abscisse", "links": ["http://fablab.coagul.org"], "url": "https://www.fablabs.io/labs/lab6", "longitude": 5.04147999999998, "county": "France", "parent_id": 545, "kind_name": "mini_fab_lab", "postal_code": "2100", "coordinates": [47.322047, 5.04148], "address_2": "6, impasse Quentin", "latitude": 47.322047, "country_code": "fr", "email": "c-bureau@outils.coagul.org", "address_1": "Dijon", "geometry": {"type": "Point", "coordinates": [5.04148, 47.322047]}, "country": "France"}, {"city": "Montreuil", "kind_name": "fab_lab", "links": ["http://www.apedec.org ", "http://webtv.montreuil.fr/festival-m.u.s.i.c-et-fablab-video-415-12.html", "http://www.wedemain.fr/A-Montreuil-un-fab-lab-circulaire-dans-une-usine-verticale_a421.html"], "capabilities": "three_d_printing", "url": "https://www.fablabs.io/labs/ecodesignfablab", "name": "ECODESIGN FAB LAB", "email": "contact@apedec.org", "coordinates": [48.8693157, 2.4564764], "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/51/53/74898eb4-e94d-49fc-9e57-18246d1901c8/ECODESIGN FAB LAB.jpg", "phone": "+33 1 (0)9.81.29.17.31", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/15/13/33b98c6f-b6c1-4cfd-8b63-401c4441f964/ECODESIGN FAB LAB.jpg", "postal_code": "93106", "longitude": 2.45647640000004, "country_code": "fr", "latitude": 48.8693157, "address_1": "Montreuil", "address_notes": "lot 38 D", "address_2": "2 \u00e0 20 avenue Allende, MOZINOR", "blurb": "FAB LAB specialized in upcycling and ecodesign with furniture production based on diverted source of industrial waste, located in a industrial zone, in the heart of a popular city.", "description": "Based on the roof of an industrial zone of 50 SMEs (and 500 workers), Ecodesign Fab Lab is now open to address upcycling and eco-innovation, thanks waste collection, designers and classical wood equipment, but also 3D printers (and CNC equipment in the next weeks).", "geometry": {"type": "Point", "coordinates": [2.4564764, 48.8693157]}, "country": "France"}, @@ -133,5 +132,4 @@ {"city": "Bron", "kind_name": "fab_lab", "links": ["http://fablab-lyon.fr"], "capabilities": "three_d_printing;cnc_milling;laser;vinyl_cutting", "url": "https://www.fablabs.io/labs/fabriquedobjetslibres", "name": "Fabrique d'Objets Libres", "email": "contact@fabriquedobjetslibres.fr", "coordinates": [45.7429334, 4.9082135], "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/50/01/0190e790-aaec-4f2f-9985-11156655145d/Fabrique d'Objets Libres.jpg", "county": "Rh\u00f4ne", "phone": "+33 7 68 01 40 26 (Tue-Sat 2pm-6pm)", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/13/49/73ea9f2d-0216-4f52-a6bf-2ff97ee474b2/Fabrique d'Objets Libres.jpg", "postal_code": "69500", "longitude": 4.90821349999999, "country_code": "fr", "latitude": 45.7429334, "address_1": "All\u00e9e Gaillard Romanet", "address_notes": "Au sous-sol de la MJC. Downstairs inside the MJC.", "address_2": "MJC Louis Aragon", "blurb": "Le fablab lyonnais, install\u00e9 \u00e0 la MJC Louis Aragon de Bron, ouvert tous les mercredis et formation hebdomadaire de fabrication num\u00e9rique. Projets autour du handicap, des arts et du recyclage.", "description": "La Fabrique d'Objets Libres est un fablab associatif sur Lyon et sa r\u00e9gion. Install\u00e9 \u00e0 la MJC Louis Aragon de Bron depuis janvier 2013, c'est un espace de cr\u00e9ation et de fabrication num\u00e9rique ouvert \u00e0 tous, qui permet \u00e0 chacun de d\u00e9couvrir, d'inventer et de fabriquer tout type d'objet.\r\n \r\nV\u00e9ritable laboratoire citoyen de fabrication, la Fabrique d\u2019Objets Libres met \u00e0 disposition de ses adh\u00e9rents des outils \u00e0 commande num\u00e9rique et des mati\u00e8res premi\u00e8res secondaires permettant de concevoir et de fabriquer localement des objets libres.\r\nC\u2019est une plate-forme pluridisciplinaire collaborative qui m\u00eale les profils (techniciens, informaticiens, ing\u00e9nieurs, scientifiques, bricoleurs, cr\u00e9ateurs...) et les g\u00e9n\u00e9rations afin de r\u00e9unir tous types de comp\u00e9tences.\r\n\r\nLe fablab est ouvert tous les mercredis pour les \"temps libres\", durant lesquels les adh\u00e9rents utilisent les machines librement. Par ailleurs, il propose un atelier hebdomadaire aux adh\u00e9rents de la MJC, \"De l'id\u00e9e \u00e0 l'objet\": en une dizaine de s\u00e9ances sur un trimestre, les participants apprennent \u00e0 utiliser toutes les machines du fablab pour r\u00e9aliser leurs objets, et r\u00e9fl\u00e9chissent autour d'une th\u00e9matique sociale comme le handicap, la musique, ou la ville.\r\n\r\nL'association organise \u00e9galement des \u00e9v\u00e9nements et ateliers th\u00e9matiques utilisant la fabrication num\u00e9rique autour de sujet plus vastes, comme l'art, avec les machines \u00e0 dessiner, ou le handicap, dans le cadre du projet Handilab, ou encore la fin de vie des objets, avec le Laboratoire de l'Obsolescence D\u00e9programm\u00e9e. Enfin, le fablab s'associe \u00e0 d'autres associations et des entreprises pour des projets communs.", "geometry": {"type": "Point", "coordinates": [4.9082135, 45.7429334]}, "country": "France"}, {"city": "N\u00e9ons-sur-Creuse", "kind_name": "fab_lab", "links": ["http://www.rurallab.org"], "url": "https://www.fablabs.io/labs/rurallab", "coordinates": [46.744746, 0.931698], "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/49/00/95c7b9f2-a034-4b2b-931d-43ced33ddfb1/RuralLab.jpg", "phone": "+33603318810", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/12/49/ec5f7c54-e6ce-40fd-b5c5-c4142d208e6b/RuralLab.jpg", "postal_code": "36220", "longitude": 0.931697999999983, "country_code": "fr", "latitude": 46.744746, "address_1": "Rue de l'\u00c9cole", "email": "rurallab36@gmail.com", "blurb": "A FabLab in the countryside in Neons sur Creuse, France", "name": "RuralLab", "geometry": {"type": "Point", "coordinates": [0.931698, 46.744746]}, "country": "France"}, {"city": "Gif-sur-Yvette", "kind_name": "supernode", "links": ["http://fablab.digiscope.fr/#!/", "http://fablabdigiscope.wordpress.com"], "url": "https://www.fablabs.io/labs/fablabdigiscope", "name": "(Fab)Lab Digiscope", "longitude": 2.16830979999997, "header_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/13/52/18/8d63351d-c2fb-4a90-8e58-bb45422202a6/(Fab)Lab Digiscope.jpg", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/15/46/51553da4-b295-426c-837f-934c311933ba/(Fab)Lab Digiscope.jpg", "postal_code": "91190", "coordinates": [48.7117632, 2.1683098], "country_code": "fr", "latitude": 48.7117632, "address_1": "660 Rue Noetzlin", "capabilities": "three_d_printing;cnc_milling;circuit_production;laser;precision_milling;vinyl_cutting", "email": "fablabdigiscope@gmail.com", "blurb": "(FAB)LAB DIGISCOPE is a fabrication laboratory dedicated to research in sciences | design | education | art | engineering and what ever field of research you come from. Open to Everyone. Book now!", "description": "(FAB)LAB DIGISCOPE is a fabrication laboratory dedicated to research in sciences | design | education | arts | engineering and what ever field of research you come from. We host Fab Academy and Bio Academy. We host Digital Fabrication Classes for EITC Master. Open to Everyone since the beginning.\r\n\r\nFablab Digiscope started in 2013 when Aviz-INRIA research team director Jean-Daniel Fekete and colleague researcher Pierre Dragicevic hired Romain Di Vozzo as a R&D Engineer to be the fablab manager of what would later become an attractive place on the new Campus Paris-Saclay. Fablab Digiscope is part of the Digiscope Project, a network of 10 high-performance platforms for interactive visualization of large datasets and complex computation for which Michel Beaudouin-Lafon is the scientific Director. Fablab Digiscope is mutualised between 10 institutions involved in research and education.\r\n\r\nRomain Di Vozzo runs and develops Fablab Digiscope everyday, trains publics, designs objects, shares creative thoughts, gives advices on designs, etc. Romain also actively collaborates to the globally distributed fablab network and with the Fab Foundation by operating as Fab Academy SuperNode, as Instructor for Fab Academy and Bio Academy, by giving conferences and workshops in France and abroad and by performing very small tasks that make the fablab network grow.", "geometry": {"type": "Point", "coordinates": [2.1683098, 48.7117632]}, "country": "France"}, -{"city": "Metz", "kind_name": "fab_lab", "links": ["http://graoulab.org/wiki", "http://graoulab.org"], "url": "https://www.fablabs.io/labs/graoulab", "coordinates": [49.1262692, 6.182086], "name": "GraouLab", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/18/24/af4709d8-1f60-48a7-ba35-4c42ef40a195/GraouLab.jpg", "postal_code": "57000", "longitude": 6.18208600000003, "country_code": "fr", "latitude": 49.1262692, "capabilities": "three_d_printing;laser", "email": "contact@graoulab.org", "blurb": "The FabLab of Metz. A place for folks innovation.", "address_1": "7 Avenue de Blida", "geometry": {"type": "Point", "coordinates": [6.182086, 49.1262692]}, "country": "France"} -] \ No newline at end of file +{"city": "Metz", "kind_name": "fab_lab", "links": ["http://graoulab.org/wiki", "http://graoulab.org"], "url": "https://www.fablabs.io/labs/graoulab", "coordinates": [49.1262692, 6.182086], "name": "GraouLab", "avatar_url": "http://fablabs.io.s3.amazonaws.com/2017/01/28/11/18/24/af4709d8-1f60-48a7-ba35-4c42ef40a195/GraouLab.jpg", "postal_code": "57000", "longitude": 6.18208600000003, "country_code": "fr", "latitude": 49.1262692, "capabilities": "three_d_printing;laser", "email": "contact@graoulab.org", "blurb": "The FabLab of Metz. A place for folks innovation.", "address_1": "7 Avenue de Blida", "geometry": {"type": "Point", "coordinates": [6.182086, 49.1262692]}, "country": "France"}] \ No newline at end of file From 71a32f623f55a1870c3496e36a039d5a662cf4ef Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 09:56:10 +0200 Subject: [PATCH 03/15] Sweeping the docs. --- bonobo/_api.py | 83 ++++++++++++++++++++++------------- bonobo/structs/bags.py | 24 ++++++++++ docs/contribute/index.rst | 15 +++++++ docs/guide/index.rst | 1 + docs/reference/api.rst | 53 +++------------------- docs/reference/api_config.rst | 10 +++++ docs/reference/commands.rst | 4 +- docs/reference/index.rst | 5 +-- docs/tutorial/index.rst | 2 +- docs/tutorial/python.rst | 14 +----- 10 files changed, 115 insertions(+), 96 deletions(-) create mode 100644 docs/reference/api_config.rst diff --git a/bonobo/_api.py b/bonobo/_api.py index 26ff253..7d0f15e 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -1,3 +1,5 @@ +import warnings + from bonobo.basics import Limit, PrettyPrint, Tee, count, identity, noop, pprint from bonobo.strategies import create_strategy from bonobo.structs import Bag, Graph @@ -11,22 +13,60 @@ def register_api(x, __all__=__all__): __all__.append(get_name(x)) return x + def register_api_group(*args): for attr in args: register_api(attr) -# bonobo.basics -register_api_group(Limit, PrettyPrint, Tee, count, identity, noop, pprint, ) +@register_api +def run(graph, *chain, strategy=None, plugins=None, services=None): + """ + Main entry point of bonobo. It takes a graph and creates all the necessary plumbery around to execute it. + + The only necessary argument is a :class:`Graph` instance, containing the logic you actually want to execute. + + By default, this graph will be executed using the "threadpool" strategy: each graph node will be wrapped in a + thread, and executed in a loop until there is no more input to this node. + + You can provide plugins factory objects in the plugins list, this function will add the necessary plugins for + interactive console execution and jupyter notebook execution if it detects correctly that it runs in this context. + + You'll probably want to provide a services dictionary mapping service names to service instances. + + :param Graph graph: The :class:`Graph` to execute. + :param str strategy: The :class:`bonobo.strategies.base.Strategy` to use. + :param list plugins: The list of plugins to enhance execution. + :param dict services: The implementations of services this graph will use. + :return bonobo.execution.graph.GraphExecutionContext: + """ + if len(chain): + warnings.warn('DEPRECATED. You should pass a Graph instance instead of a chain.') + from bonobo import Graph + graph = Graph(graph, *chain) -# bonobo.io -register_api_group(CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter) + strategy = create_strategy(strategy) + plugins = plugins or [] + + if _is_interactive_console(): + from bonobo.ext.console import ConsoleOutputPlugin + if ConsoleOutputPlugin not in plugins: + plugins.append(ConsoleOutputPlugin) + + if _is_jupyter_notebook(): + from bonobo.ext.jupyter import JupyterOutputPlugin + if JupyterOutputPlugin not in plugins: + plugins.append(JupyterOutputPlugin) + + return strategy.execute(graph, plugins=plugins, services=services) + + +# bonobo.structs +register_api_group(Bag, Graph) # bonobo.strategies register_api(create_strategy) -# bonobo.structs -register_api_group(Bag, Graph) # Shortcut to filesystem2's open_fs, that we make available there for convenience. @register_api @@ -47,6 +87,12 @@ def open_fs(fs_url, *args, **kwargs): return _open_fs(str(fs_url), *args, **kwargs) +# bonobo.basics +register_api_group(Limit, PrettyPrint, Tee, count, identity, noop, pprint, ) + +# bonobo.io +register_api_group(CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter) + def _is_interactive_console(): import sys @@ -60,29 +106,6 @@ def _is_jupyter_notebook(): return False -@register_api -def run(graph, *chain, strategy=None, plugins=None, services=None): - if len(chain): - warnings.warn('DEPRECATED. You should pass a Graph instance instead of a chain.') - from bonobo import Graph - graph = Graph(graph, *chain) - - strategy = create_strategy(strategy) - plugins = [] - - if _is_interactive_console(): - from bonobo.ext.console import ConsoleOutputPlugin - if ConsoleOutputPlugin not in plugins: - plugins.append(ConsoleOutputPlugin) - - if _is_jupyter_notebook(): - from bonobo.ext.jupyter import JupyterOutputPlugin - if JupyterOutputPlugin not in plugins: - plugins.append(JupyterOutputPlugin) - - return strategy.execute(graph, plugins=plugins, services=services) - - @register_api def get_examples_path(*pathsegments): import os @@ -94,5 +117,3 @@ def get_examples_path(*pathsegments): def open_examples_fs(*pathsegments): return open_fs(get_examples_path(*pathsegments)) - -print(__all__) diff --git a/bonobo/structs/bags.py b/bonobo/structs/bags.py index 8605b19..4d86f89 100644 --- a/bonobo/structs/bags.py +++ b/bonobo/structs/bags.py @@ -9,6 +9,30 @@ __all__ = [ class Bag: + """ + Bags are simple datastructures that holds arguments and keyword arguments together, that may be applied to a + callable. + + Example: + + >>> from bonobo import Bag + >>> def myfunc(foo, *, bar): + ... print(foo, bar) + ... + >>> bag = Bag('foo', bar='baz') + >>> bag.apply(myfunc) + foo baz + + A bag can inherit another bag, allowing to override only a few arguments without touching the parent. + + Example: + + >>> bag2 = Bag(bar='notbaz', _parent=bag) + >>> bag2.apply(myfunc) + foo notbaz + + """ + def __init__(self, *args, _flags=None, _parent=None, **kwargs): self._flags = _flags or () self._parent = _parent diff --git a/docs/contribute/index.rst b/docs/contribute/index.rst index 79d576b..6ee7e31 100644 --- a/docs/contribute/index.rst +++ b/docs/contribute/index.rst @@ -1,6 +1,21 @@ Contributing ============ +There's a lot of different ways you can contribute, and not all of them includes coding. Do not think that the codeless +contributions have less value, all contributions are very important. + +* You can contribute to the documentation. +* You can help reproducing errors and giving more infos in the issues. +* You can open issues with problems you're facing. +* You can help creating better presentation material. +* You can talk about it in your local python user group. +* You can enhance examples. +* You can enhance tests. +* etc. + +Code-related contributions (including tests and examples) +::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + Contributing to bonobo is usually done this way: * Discuss ideas in the `issue tracker `_ or on `Slack `_. diff --git a/docs/guide/index.rst b/docs/guide/index.rst index 02082aa..dab1b44 100644 --- a/docs/guide/index.rst +++ b/docs/guide/index.rst @@ -22,5 +22,6 @@ available as an optional extra dependency, and the maturity stage of each extens :maxdepth: 2 ext/docker + ext/jupyter ext/selenium ext/sqlalchemy diff --git a/docs/reference/api.rst b/docs/reference/api.rst index 13376ac..e401b9c 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -1,54 +1,13 @@ -Public API +Bonobo API ========== -All the "public api" callables, classes and other callables are available under the root :mod:`bonobo` package, even if -they are documented within their sub-namespace, for convenience. +The Bonobo API, available directly under the :mod:`bonobo` package, contains all the tools you need to get started with +bonobo. + +The :mod:`bonobo` package +::::::::::::::::::::::::: .. automodule:: bonobo - :members: create_strategy, get_examples_path, run - :undoc-members: - :show-inheritance: - -Config ------- - -.. automodule:: bonobo.config - :members: - :undoc-members: - :show-inheritance: - - -Context -------- - -.. automodule:: bonobo.context - :members: - :undoc-members: - :show-inheritance: - - -Core ----- - -.. automodule:: bonobo.core - :members: - :undoc-members: - :show-inheritance: - - -IO --- - -.. automodule:: bonobo.io - :members: - :undoc-members: - :show-inheritance: - - -Util ----- - -.. automodule:: bonobo.util :members: :undoc-members: :show-inheritance: diff --git a/docs/reference/api_config.rst b/docs/reference/api_config.rst new file mode 100644 index 0000000..c1010a7 --- /dev/null +++ b/docs/reference/api_config.rst @@ -0,0 +1,10 @@ +Config API +========== + +The Config API, located under the :mod:`bonobo.config` namespace, contains all the tools you need to create +configurable transformations, either class-based or function-based. + +.. automodule:: bonobo.config + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/reference/commands.rst b/docs/reference/commands.rst index 8d79fb6..f7c95af 100644 --- a/docs/reference/commands.rst +++ b/docs/reference/commands.rst @@ -1,5 +1,5 @@ -Commands Reference -================== +Command-line Reference +====================== Bonobo Init ::::::::::: diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 763a126..f255e22 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -1,11 +1,10 @@ References ========== -.. todo:: write the fucking doc! - .. toctree:: :maxdepth: 4 - commands api + api_config + commands examples diff --git a/docs/tutorial/index.rst b/docs/tutorial/index.rst index 324fed7..e57e154 100644 --- a/docs/tutorial/index.rst +++ b/docs/tutorial/index.rst @@ -10,7 +10,7 @@ We strongly advice that even if you're an advanced python developper, you go thr reasons: that should be sufficient to do anything possible with **Bonobo** and that's a good moment to learn the few concepts you'll see everywhere in the software. -If you're not familiar with python, you should first read :doc:`./python`. +If you're not familiar with python, you should first read :doc:`python`. .. toctree:: :maxdepth: 2 diff --git a/docs/tutorial/python.rst b/docs/tutorial/python.rst index dae49b8..d045031 100644 --- a/docs/tutorial/python.rst +++ b/docs/tutorial/python.rst @@ -6,16 +6,6 @@ Just enough Python for Bonobo This is a work in progress and it is not yet available. Please come back later or even better, help us write this guide! -This guide is intended to help programmers or enthusiasts to grasp the python basics necessary to use Bonobo. It should -definately not be considered as a general python introduction, neither a deep dive into details. - -.. toctree:: - :maxdepth: 2 - - python01 - python02 - python03 - python04 - python05 - + This guide is intended to help programmers or enthusiasts to grasp the python basics necessary to use Bonobo. It + should definately not be considered as a general python introduction, neither a deep dive into details. From 8fc0430a233db7bf9606ed56295ea2f008128a66 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 10:21:56 +0200 Subject: [PATCH 04/15] more documentage --- bonobo/examples/datasets/coffeeshops.py | 15 +++++++ bonobo/examples/datasets/fablabs.py | 16 +++++++ bonobo/examples/utils/count.py | 20 +++++++-- docs/_static/graphs.css | 2 +- docs/reference/commands.rst | 4 +- docs/reference/examples.rst | 56 +++++++++++++++++++++---- 6 files changed, 100 insertions(+), 13 deletions(-) diff --git a/bonobo/examples/datasets/coffeeshops.py b/bonobo/examples/datasets/coffeeshops.py index 9c6e187..dc3db52 100644 --- a/bonobo/examples/datasets/coffeeshops.py +++ b/bonobo/examples/datasets/coffeeshops.py @@ -1,3 +1,18 @@ +""" +Extracts a list of parisian bars where you can buy a coffee for a reasonable price, and store them in a flat text file. + +.. graphviz:: + + digraph { + rankdir = LR; + stylesheet = "../_static/graphs.css"; + + BEGIN [shape="point"]; + BEGIN -> "ODS()" -> "transform" -> "FileWriter()"; + } + +""" + import bonobo from bonobo.commands.run import get_default_services from bonobo.ext.opendatasoft import OpenDataSoftAPI diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index 1ed52d7..17ee841 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -1,3 +1,19 @@ +""" +Extracts a list of fablabs in the world, restrict to the ones in france, then format it both for a nice console output +and a flat txt file. + +.. graphviz:: + + digraph { + rankdir = LR; + stylesheet = "../_static/graphs.css"; + + BEGIN [shape="point"]; + BEGIN -> "ODS()" -> "normalize" -> "filter_france" -> "Tee()" -> "JsonWriter()"; + } + +""" + import json from colorama import Fore, Style diff --git a/bonobo/examples/utils/count.py b/bonobo/examples/utils/count.py index fdcc3bf..ea440a0 100644 --- a/bonobo/examples/utils/count.py +++ b/bonobo/examples/utils/count.py @@ -1,7 +1,21 @@ -import bonobo -import bonobo.basics +""" +Simple example of :func:`bonobo.count` usage. -graph = bonobo.Graph(range(42), bonobo.basics.count, print) +.. graphviz:: + + digraph { + rankdir = LR; + stylesheet = "../_static/graphs.css"; + + BEGIN [shape="point"]; + BEGIN -> "range()" -> "count" -> "print"; + } + +""" + +import bonobo + +graph = bonobo.Graph(range(42), bonobo.count, print) if __name__ == '__main__': bonobo.run(graph) diff --git a/docs/_static/graphs.css b/docs/_static/graphs.css index 873f3d6..abed45c 100644 --- a/docs/_static/graphs.css +++ b/docs/_static/graphs.css @@ -1,2 +1,2 @@ .node { -} \ No newline at end of file +} diff --git a/docs/reference/commands.rst b/docs/reference/commands.rst index f7c95af..dcd054a 100644 --- a/docs/reference/commands.rst +++ b/docs/reference/commands.rst @@ -1,5 +1,5 @@ -Command-line Reference -====================== +Command-line +============ Bonobo Init ::::::::::: diff --git a/docs/reference/examples.rst b/docs/reference/examples.rst index bddbe8b..e2f922d 100644 --- a/docs/reference/examples.rst +++ b/docs/reference/examples.rst @@ -1,13 +1,44 @@ Examples ======== -There are a few examples bundled with **bonobo**. You'll find them under the :mod:`bonobo.examples` package. +There are a few examples bundled with **bonobo**. You'll find them under the :mod:`bonobo.examples` package, and +you can try them in a clone of bonobo by typing:: + + $ bonobo run bonobo/examples/.../file.py + + +Datasets +:::::::: + + +.. module:: bonobo.examples.datasets + +The :mod:`bonobo.examples.datasets` package contains examples that generates datasets locally for other examples to +use. As of today, we commit the content of those datasets to git, even if that may be a bad idea, so all the examples +are easily runnable. Later, we'll see if we favor a "missing dependency exception" approach. + + +Coffeeshops +----------- + +.. automodule:: bonobo.examples.datasets.coffeeshops + :members: + :undoc-members: + :show-inheritance: + +Fablabs +------- + +.. automodule:: bonobo.examples.datasets.fablabs + :members: + :undoc-members: + :show-inheritance: Types ::::: -bonobo.examples.types.strings ------------------------------ +Strings +------- .. automodule:: bonobo.examples.types.strings :members: graph, extract, transform, load @@ -15,8 +46,8 @@ bonobo.examples.types.strings :show-inheritance: -bonobo.examples.types.dicts ---------------------------- +Dicts +----- .. automodule:: bonobo.examples.types.dicts :members: graph, extract, transform, load @@ -24,8 +55,8 @@ bonobo.examples.types.dicts :show-inheritance: -bonobo.examples.types.bags --------------------------- +Bags +---- .. automodule:: bonobo.examples.types.bags :members: graph, extract, transform, load @@ -33,4 +64,15 @@ bonobo.examples.types.bags :show-inheritance: +Utils +::::: + +Count +----- + +.. automodule:: bonobo.examples.utils.count + :members: + :undoc-members: + :show-inheritance: + From a164c76d04e606205472af7da1901152d4724472 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 10:28:51 +0200 Subject: [PATCH 05/15] ignoring examples and ext packages from codacy inspections. --- .codacy.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .codacy.yml diff --git a/.codacy.yml b/.codacy.yml new file mode 100644 index 0000000..9254b5f --- /dev/null +++ b/.codacy.yml @@ -0,0 +1,4 @@ +--- +exclude_paths: + - bonobo/examples + - bonobo/ext From 6be714e3f6deafb2a23402a3792eee1cfd0cc9a7 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 10:31:51 +0200 Subject: [PATCH 06/15] omit examples and extensions from coverage. --- .coveragerc | 3 +++ bonobo/__init__.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.coveragerc b/.coveragerc index 1d76a1f..44bc7af 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,8 @@ [run] branch = True +omit = + bonobo/examples/* + bonobo/ext/* [report] # Regexes for lines to exclude from consideration diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 11e17a1..3c15c18 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -14,4 +14,6 @@ from bonobo._api import __all__ from bonobo._version import __version__ __all__ = ['__version__'] + __all__ +__version__ = __version__ + del sys From a2f477319cff627af0a0a20ebc8d1175f9fa2f10 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 10:38:48 +0200 Subject: [PATCH 07/15] attempt to skip appveyor problem --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 0c33699..bb7f99a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,6 +24,8 @@ environment: PYTHON_VERSION: "3.6.1" PYTHON_ARCH: "64" +build: false + install: # If there is a newer build queued for the same PR, cancel this one. # The AppVeyor 'rollout builds' option is supposed to serve the same From 27f2ab2fcb07c4ea8032f3f29783dbdd4084a871 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 10:41:04 +0200 Subject: [PATCH 08/15] adds test dependencies to appveyor config --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index bb7f99a..0779d84 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -63,7 +63,7 @@ install: # compiled extensions and are not provided as pre-built wheel packages, # pip will build them from source using the MSVC compiler matching the # target Python version and architecture - - "%CMD_IN_ENV% pip install -e ." + - "%CMD_IN_ENV% pip install -e .[dev]" test_script: # Run the project tests From 92fa1fb7e224e0a362b76c2872b3c3159b4a21af Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 10:43:13 +0200 Subject: [PATCH 09/15] mock module is in unittest (py3) --- tests/ext/test_ods.py | 2 +- tests/structs/test_bags.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ext/test_ods.py b/tests/ext/test_ods.py index 85036f8..6b6a242 100644 --- a/tests/ext/test_ods.py +++ b/tests/ext/test_ods.py @@ -1,4 +1,4 @@ -from mock import patch +from unittest.mock import patch from bonobo.ext.opendatasoft import OpenDataSoftAPI from bonobo.util.objects import ValueHolder diff --git a/tests/structs/test_bags.py b/tests/structs/test_bags.py index 75afa93..14f49c5 100644 --- a/tests/structs/test_bags.py +++ b/tests/structs/test_bags.py @@ -1,4 +1,4 @@ -from mock import Mock +from unittest.mock import Mock from bonobo import Bag from bonobo.constants import INHERIT_INPUT From 204135e5e0a791e729e58c6f5cb9cb0d601da60d Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 10:46:23 +0200 Subject: [PATCH 10/15] remove dist build after windows tests. --- appveyor.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0779d84..f044adb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -69,12 +69,12 @@ test_script: # Run the project tests - "%CMD_IN_ENV% pytest --capture=no --cov=bonobo --cov-report html" -after_test: - # If tests are successful, create binary packages for the project. - - "%CMD_IN_ENV% python setup.py bdist_wheel" - - "%CMD_IN_ENV% python setup.py bdist_wininst" - - "%CMD_IN_ENV% python setup.py bdist_msi" - - ps: "ls dist" +#after_test: +# If tests are successful, create binary packages for the project. +# - "%CMD_IN_ENV% python setup.py bdist_wheel" +# - "%CMD_IN_ENV% python setup.py bdist_wininst" +# - "%CMD_IN_ENV% python setup.py bdist_msi" +# - ps: "ls dist" artifacts: # Archive the generated packages in the ci.appveyor.com build report. From 670f4e61d1c00130c2f83ce24736c2faaeb95166 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 11:02:28 +0200 Subject: [PATCH 11/15] fixing codacy issues --- bonobo/commands/init.py | 4 +--- bonobo/commands/run.py | 4 ++-- bonobo/config/configurables.py | 2 +- bonobo/config/processors.py | 6 +++--- bonobo/ext/edgy/project/feature.py | 4 ---- bonobo/ext/jupyter/plugin.py | 8 -------- bonobo/io/csv.py | 2 +- tests/io/test_file.py | 2 +- tests/test_commands.py | 2 +- 9 files changed, 10 insertions(+), 24 deletions(-) diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index a1ab700..55c4b6b 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -3,14 +3,12 @@ import os def execute(): try: - import edgy.project + from edgy.project.__main__ import handle_init except ImportError as exc: raise ImportError( 'You must install "edgy.project" to use this command.\n\n $ pip install edgy.project\n' ) from exc - from edgy.project.__main__ import handle_init - return handle_init(os.path.join(os.getcwd(), 'Projectfile')) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 62f15c4..b7872e2 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -20,7 +20,7 @@ def get_default_services(filename, services=None): } try: exec(code, context) - except Exception as exc: + except Exception: raise return { **context[DEFAULT_SERVICES_ATTR](), @@ -55,7 +55,7 @@ def execute(file, quiet=False): 'but it is something that will be implemented in the future.\n\nExpected: 1, got: {}.' ).format(len(graphs)) - name, graph = list(graphs.items())[0] + graph = list(graphs.values())[0] # todo if console and not quiet, then add the console plugin # todo when better console plugin, add it if console and just disable display diff --git a/bonobo/config/configurables.py b/bonobo/config/configurables.py index d5bdeb3..e3ad2cf 100644 --- a/bonobo/config/configurables.py +++ b/bonobo/config/configurables.py @@ -53,4 +53,4 @@ class Configurable(metaclass=ConfigurableMeta): ) for name, value in kwargs.items(): - setattr(self, name, kwargs[name]) + setattr(self, name, value) diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py index 64a4bee..9dac6d0 100644 --- a/bonobo/config/processors.py +++ b/bonobo/config/processors.py @@ -52,9 +52,9 @@ def contextual(cls_or_func): setattr(cls_or_func, _CONTEXT_PROCESSORS_ATTR, []) _processors = getattr(cls_or_func, _CONTEXT_PROCESSORS_ATTR) - for name, value in cls_or_func.__dict__.items(): - if isinstance(value, ContextProcessor): - _processors.append(value) + for processor in cls_or_func.__dict__.values(): + if isinstance(processor, ContextProcessor): + _processors.append(processor) # This is needed for python 3.5, python 3.6 should be fine, but it's considered an implementation detail. _processors.sort(key=lambda proc: proc._creation_counter) diff --git a/bonobo/ext/edgy/project/feature.py b/bonobo/ext/edgy/project/feature.py index ef75fea..5a01f97 100644 --- a/bonobo/ext/edgy/project/feature.py +++ b/bonobo/ext/edgy/project/feature.py @@ -14,10 +14,6 @@ from edgy.project.feature import Feature, SUPPORT_PRIORITY class BonoboFeature(Feature): requires = {'python'} - @subscribe('edgy.project.feature.make.on_generate', priority=SUPPORT_PRIORITY) - def on_make_generate(self, event): - makefile = event.makefile - @subscribe('edgy.project.on_start', priority=SUPPORT_PRIORITY) def on_start(self, event): package_path = event.setup['name'].replace('.', os.sep) diff --git a/bonobo/ext/jupyter/plugin.py b/bonobo/ext/jupyter/plugin.py index c49610d..fe907be 100644 --- a/bonobo/ext/jupyter/plugin.py +++ b/bonobo/ext/jupyter/plugin.py @@ -24,11 +24,3 @@ class JupyterOutputPlugin(Plugin): finalize = run -""" -TODO JUPYTER WIDGET -################### - -# close the widget? what does it do? -https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20Basics.html#Closing-widgets - -""" diff --git a/bonobo/io/csv.py b/bonobo/io/csv.py index 9192fc3..4922b43 100644 --- a/bonobo/io/csv.py +++ b/bonobo/io/csv.py @@ -50,7 +50,7 @@ class CsvReader(CsvHandler, FileReader): field_count = len(headers.value) if self.skip and self.skip > 0: - for i in range(0, self.skip): + for _ in range(0, self.skip): next(reader) for row in reader: diff --git a/tests/io/test_file.py b/tests/io/test_file.py index dc011a1..918b27f 100644 --- a/tests/io/test_file.py +++ b/tests/io/test_file.py @@ -21,7 +21,7 @@ def test_file_writer_in_context(tmpdir, lines, output): context.start() context.recv(BEGIN, *map(Bag, lines), END) - for i in range(len(lines)): + for _ in range(len(lines)): context.step() context.stop() diff --git a/tests/test_commands.py b/tests/test_commands.py index 8c33c6d..55032b2 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -18,7 +18,7 @@ def test_entrypoint(): def test_no_command(capsys): with pytest.raises(SystemExit): entrypoint([]) - out, err = capsys.readouterr() + _, err = capsys.readouterr() assert 'error: the following arguments are required: command' in err From e04a6261d016507f1ea3a20bc8cd5175a41891c8 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 11:06:17 +0200 Subject: [PATCH 12/15] codacy config --- .codacy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.codacy.yml b/.codacy.yml index 9254b5f..390e58f 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -1,4 +1,4 @@ --- exclude_paths: - - bonobo/examples - - bonobo/ext + - bonobo/examples/ + - bonobo/ext/ From 474999a87e6b3cf8033cc58b3df70434e6bf1911 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 15:12:48 +0200 Subject: [PATCH 13/15] Trying to fix unending transformations on start() error. --- .gitignore | 1 + bonobo/execution/base.py | 70 ++++++++++++----------------------- bonobo/execution/graph.py | 10 ++--- bonobo/execution/node.py | 15 +++++--- bonobo/strategies/executor.py | 28 ++++++++++---- bonobo/util/errors.py | 30 +++++++++++++++ 6 files changed, 90 insertions(+), 64 deletions(-) create mode 100644 bonobo/util/errors.py diff --git a/.gitignore b/.gitignore index 179ed2e..a88bd7b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ /.idea /.release /bonobo.iml +/bonobo/examples/work_in_progress/ /bonobo/ext/jupyter/js/node_modules/ /build/ /coverage.xml diff --git a/bonobo/execution/base.py b/bonobo/execution/base.py index 9a96cb1..e2dfb55 100644 --- a/bonobo/execution/base.py +++ b/bonobo/execution/base.py @@ -1,9 +1,9 @@ -import sys import traceback from time import sleep from bonobo.config import Container from bonobo.config.processors import resolve_processors +from bonobo.util.errors import print_error from bonobo.util.iterators import ensure_tuple from bonobo.util.objects import Wrapper @@ -43,16 +43,13 @@ class LoopingExecutionContext(Wrapper): False), ('{}.start() can only be called on a new node.').format(type(self).__name__) assert self._context is None self._started = True - try: - if self.parent: - self._context = self.parent.services.args_for(self.wrapped) - elif self.services: - self._context = self.services.args_for(self.wrapped) - else: - self._context = () - except Exception as exc: # pylint: disable=broad-except - self.handle_error(exc, traceback.format_exc()) - raise + + if self.parent: + self._context = self.parent.services.args_for(self.wrapped) + elif self.services: + self._context = self.services.args_for(self.wrapped) + else: + self._context = () for processor in resolve_processors(self.wrapped): try: @@ -80,41 +77,22 @@ class LoopingExecutionContext(Wrapper): if self._stopped: return - assert self._context is not None - self._stopped = True - while len(self._stack): - processor = self._stack.pop() - try: - # todo yield from ? how to ? - next(processor) - except StopIteration as exc: - # This is normal, and wanted. - pass - except Exception as exc: # pylint: disable=broad-except - self.handle_error(exc, traceback.format_exc()) - raise - else: - # No error ? We should have had StopIteration ... - raise RuntimeError('Context processors should not yield more than once.') + if self._context is not None: + while len(self._stack): + processor = self._stack.pop() + try: + # todo yield from ? how to ? + next(processor) + except StopIteration as exc: + # This is normal, and wanted. + pass + except Exception as exc: # pylint: disable=broad-except + self.handle_error(exc, traceback.format_exc()) + raise + else: + # No error ? We should have had StopIteration ... + raise RuntimeError('Context processors should not yield more than once.') def handle_error(self, exc, trace): - """ - Error handler. Whatever happens in a plugin or component, if it looks like an exception, taste like an exception - or somehow make me think it is an exception, I'll handle it. - - :param exc: the culprit - :param trace: Hercule Poirot's logbook. - :return: to hell - """ - - from colorama import Fore, Style - print( - Style.BRIGHT, - Fore.RED, - '\U0001F4A3 {} in {}'.format(type(exc).__name__, self.wrapped), - Style.RESET_ALL, - sep='', - file=sys.stderr, - ) - print(trace) + return print_error(exc, trace, context=self.wrapped) diff --git a/bonobo/execution/graph.py b/bonobo/execution/graph.py index d5e241f..6f95ac3 100644 --- a/bonobo/execution/graph.py +++ b/bonobo/execution/graph.py @@ -25,15 +25,15 @@ class GraphExecutionContext: self.plugins = [PluginExecutionContext(plugin, parent=self) for plugin in plugins or ()] self.services = Container(services) if services else Container() - for i, component_context in enumerate(self): + for i, node_context in enumerate(self): try: - component_context.outputs = [self[j].input for j in self.graph.outputs_of(i)] + node_context.outputs = [self[j].input for j in self.graph.outputs_of(i)] except KeyError: continue - component_context.input.on_begin = partial(component_context.send, BEGIN, _control=True) - component_context.input.on_end = partial(component_context.send, END, _control=True) - component_context.input.on_finalize = partial(component_context.stop) + node_context.input.on_begin = partial(node_context.send, BEGIN, _control=True) + node_context.input.on_end = partial(node_context.send, END, _control=True) + node_context.input.on_finalize = partial(node_context.stop) def __getitem__(self, item): return self.nodes[item] diff --git a/bonobo/execution/node.py b/bonobo/execution/node.py index 8822e73..4b52ec4 100644 --- a/bonobo/execution/node.py +++ b/bonobo/execution/node.py @@ -7,7 +7,8 @@ from bonobo.core.inputs import Input from bonobo.core.statistics import WithStatistics from bonobo.errors import InactiveReadableError from bonobo.execution.base import LoopingExecutionContext -from bonobo.structs.bags import Bag, ErrorBag +from bonobo.structs.bags import Bag +from bonobo.util.errors import is_error from bonobo.util.iterators import iter_if_not_sequence @@ -32,7 +33,13 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext): return (('+' if self.alive else '-') + ' ' + self.__name__ + ' ' + self.get_statistics_as_string()).strip() def __repr__(self): - return '<' + self.__str__() + '>' + stats = self.get_statistics_as_string().strip() + return '<{}({}{}){}>'.format( + type(self).__name__, + '+' if self.alive else '', + self.__name__, + (' ' + stats) if stats else '', + ) def recv(self, *messages): """ @@ -116,10 +123,6 @@ class NodeExecutionContext(WithStatistics, LoopingExecutionContext): self.push(_resolve(input_bag, result)) -def is_error(bag): - return isinstance(bag, ErrorBag) - - def _resolve(input_bag, output): # NotModified means to send the input unmodified to output. if output is NOT_MODIFIED: diff --git a/bonobo/strategies/executor.py b/bonobo/strategies/executor.py index f3306ea..d43f2cd 100644 --- a/bonobo/strategies/executor.py +++ b/bonobo/strategies/executor.py @@ -1,10 +1,12 @@ import time +import traceback from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor from bonobo.constants import BEGIN, END from bonobo.strategies.base import Strategy from bonobo.structs.bags import Bag +from bonobo.util.errors import print_error class ExecutorStrategy(Strategy): @@ -27,20 +29,32 @@ class ExecutorStrategy(Strategy): futures = [] for plugin_context in context.plugins: - def _runner(plugin_context=plugin_context): - plugin_context.start() - plugin_context.loop() - plugin_context.stop() + try: + plugin_context.start() + plugin_context.loop() + plugin_context.stop() + except Exception as exc: + print_error(exc, traceback.format_exc(), prefix='Error in plugin context', context=plugin_context) futures.append(executor.submit(_runner)) for node_context in context.nodes: def _runner(node_context=node_context): - node_context.start() - node_context.loop() - node_context.stop() + try: + node_context.start() + except Exception as exc: + print_error(exc, traceback.format_exc(), prefix='Could not start node context', + context=node_context) + node_context.input.on_end() + else: + node_context.loop() + + try: + node_context.stop() + except Exception as exc: + print_error(exc, traceback.format_exc(), prefix='Could not stop node context', context=node_context) futures.append(executor.submit(_runner)) diff --git a/bonobo/util/errors.py b/bonobo/util/errors.py new file mode 100644 index 0000000..468ba2f --- /dev/null +++ b/bonobo/util/errors.py @@ -0,0 +1,30 @@ +import sys + +from bonobo.structs.bags import ErrorBag + + +def is_error(bag): + return isinstance(bag, ErrorBag) + + +def print_error(exc, trace, context=None, prefix=''): + """ + Error handler. Whatever happens in a plugin or component, if it looks like an exception, taste like an exception + or somehow make me think it is an exception, I'll handle it. + + :param exc: the culprit + :param trace: Hercule Poirot's logbook. + :return: to hell + """ + + from colorama import Fore, Style + print( + Style.BRIGHT, + Fore.RED, + '\U0001F4A3 {}{}{}'.format((prefix + ': ') if prefix else '', type(exc).__name__, + ' in {!r}'.format(context) if context else ''), + Style.RESET_ALL, + sep='', + file=sys.stderr, + ) + print(trace) From 0feccb1aa96f7baa4546525857d91fb31d00b2c7 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 15:21:26 +0200 Subject: [PATCH 14/15] reformating code, and adding specific rules for examples so it shows up correctly on readthedocs, by default. --- bonobo/_api.py | 11 ++++-- bonobo/basics.py | 1 + bonobo/examples/.style.yapf | 4 +++ bonobo/examples/datasets/fablabs.py | 36 ++++++++++++++----- bonobo/examples/files/json_handlers.py | 2 ++ bonobo/examples/tutorials/tut02_01_read.py | 4 +-- bonobo/examples/tutorials/tut02_02_write.py | 4 +-- .../examples/tutorials/tut02_03_writeasmap.py | 4 +-- bonobo/ext/jupyter/plugin.py | 2 -- bonobo/strategies/executor.py | 6 ++-- bonobo/structs/bags.py | 2 +- bonobo/util/errors.py | 5 +-- setup.py | 32 ++++++++--------- 13 files changed, 70 insertions(+), 43 deletions(-) create mode 100644 bonobo/examples/.style.yapf diff --git a/bonobo/_api.py b/bonobo/_api.py index 7d0f15e..1c05e59 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -88,7 +88,15 @@ def open_fs(fs_url, *args, **kwargs): # bonobo.basics -register_api_group(Limit, PrettyPrint, Tee, count, identity, noop, pprint, ) +register_api_group( + Limit, + PrettyPrint, + Tee, + count, + identity, + noop, + pprint, +) # bonobo.io register_api_group(CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter) @@ -116,4 +124,3 @@ def get_examples_path(*pathsegments): @register_api def open_examples_fs(*pathsegments): return open_fs(get_examples_path(*pathsegments)) - diff --git a/bonobo/basics.py b/bonobo/basics.py index 6fd8c3d..97282ab 100644 --- a/bonobo/basics.py +++ b/bonobo/basics.py @@ -66,6 +66,7 @@ pprint = Tee(_pprint) def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True): from bonobo.constants import NOT_MODIFIED + def _pprint(*args, **kwargs): nonlocal title_keys, sort, print_values diff --git a/bonobo/examples/.style.yapf b/bonobo/examples/.style.yapf new file mode 100644 index 0000000..70f7590 --- /dev/null +++ b/bonobo/examples/.style.yapf @@ -0,0 +1,4 @@ +[style] +based_on_style = pep8 +column_limit = 80 +dedent_closing_brackets = true diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index 17ee841..ae74c0b 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -25,7 +25,9 @@ from bonobo.ext.opendatasoft import OpenDataSoftAPI try: import pycountry except ImportError as exc: - raise ImportError('You must install package "pycountry" to run this example.') from exc + raise ImportError( + 'You must install package "pycountry" to run this example.' + ) from exc API_DATASET = 'fablabs-in-the-world' API_NETLOC = 'datanova.laposte.fr' @@ -57,20 +59,38 @@ def display(row): address = list( filter( None, ( - ' '.join(filter(None, (row.get('postal_code', None), row.get('city', None)))), row.get('county', None), - row.get('country'), + ' '.join( + filter( + None, + (row.get('postal_code', None), row.get('city', None)) + ) + ), row.get('county', None), row.get('country'), ) ) ) - print(' - {}address{}: {address}'.format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address))) - print(' - {}links{}: {links}'.format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links']))) - print(' - {}geometry{}: {geometry}'.format(Fore.BLUE, Style.RESET_ALL, **row)) - print(' - {}source{}: {source}'.format(Fore.BLUE, Style.RESET_ALL, source='datanova/' + API_DATASET)) + print( + ' - {}address{}: {address}'. + format(Fore.BLUE, Style.RESET_ALL, address=', '.join(address)) + ) + print( + ' - {}links{}: {links}'. + format(Fore.BLUE, Style.RESET_ALL, links=', '.join(row['links'])) + ) + print( + ' - {}geometry{}: {geometry}'. + format(Fore.BLUE, Style.RESET_ALL, **row) + ) + print( + ' - {}source{}: {source}'. + format(Fore.BLUE, Style.RESET_ALL, source='datanova/' + API_DATASET) + ) graph = bonobo.Graph( - OpenDataSoftAPI(dataset=API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris'), + OpenDataSoftAPI( + dataset=API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris' + ), normalize, filter_france, bonobo.Tee(display), diff --git a/bonobo/examples/files/json_handlers.py b/bonobo/examples/files/json_handlers.py index 727a137..0bb0436 100644 --- a/bonobo/examples/files/json_handlers.py +++ b/bonobo/examples/files/json_handlers.py @@ -1,9 +1,11 @@ import bonobo from bonobo.commands.run import get_default_services + def get_fields(row): return row['fields'] + graph = bonobo.Graph( bonobo.JsonReader(path='datasets/theaters.json'), get_fields, diff --git a/bonobo/examples/tutorials/tut02_01_read.py b/bonobo/examples/tutorials/tut02_01_read.py index e934c79..28baa9e 100644 --- a/bonobo/examples/tutorials/tut02_01_read.py +++ b/bonobo/examples/tutorials/tut02_01_read.py @@ -6,6 +6,4 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={ - 'fs': bonobo.open_examples_fs('datasets') - }) + bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) diff --git a/bonobo/examples/tutorials/tut02_02_write.py b/bonobo/examples/tutorials/tut02_02_write.py index d63dd47..a3c4811 100644 --- a/bonobo/examples/tutorials/tut02_02_write.py +++ b/bonobo/examples/tutorials/tut02_02_write.py @@ -12,6 +12,4 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={ - 'fs': bonobo.open_examples_fs('datasets') - }) + bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) diff --git a/bonobo/examples/tutorials/tut02_03_writeasmap.py b/bonobo/examples/tutorials/tut02_03_writeasmap.py index e131acd..bdf3194 100644 --- a/bonobo/examples/tutorials/tut02_03_writeasmap.py +++ b/bonobo/examples/tutorials/tut02_03_writeasmap.py @@ -22,6 +22,4 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={ - 'fs': bonobo.open_examples_fs('datasets') - }) + bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) diff --git a/bonobo/ext/jupyter/plugin.py b/bonobo/ext/jupyter/plugin.py index fe907be..a418c83 100644 --- a/bonobo/ext/jupyter/plugin.py +++ b/bonobo/ext/jupyter/plugin.py @@ -22,5 +22,3 @@ class JupyterOutputPlugin(Plugin): self.widget.value = [repr(node) for node in self.context.parent.nodes] finalize = run - - diff --git a/bonobo/strategies/executor.py b/bonobo/strategies/executor.py index d43f2cd..3f34862 100644 --- a/bonobo/strategies/executor.py +++ b/bonobo/strategies/executor.py @@ -29,6 +29,7 @@ class ExecutorStrategy(Strategy): futures = [] for plugin_context in context.plugins: + def _runner(plugin_context=plugin_context): try: plugin_context.start() @@ -45,8 +46,9 @@ class ExecutorStrategy(Strategy): try: node_context.start() except Exception as exc: - print_error(exc, traceback.format_exc(), prefix='Could not start node context', - context=node_context) + print_error( + exc, traceback.format_exc(), prefix='Could not start node context', context=node_context + ) node_context.input.on_end() else: node_context.loop() diff --git a/bonobo/structs/bags.py b/bonobo/structs/bags.py index 4d86f89..1d4a7e8 100644 --- a/bonobo/structs/bags.py +++ b/bonobo/structs/bags.py @@ -32,7 +32,7 @@ class Bag: foo notbaz """ - + def __init__(self, *args, _flags=None, _parent=None, **kwargs): self._flags = _flags or () self._parent = _parent diff --git a/bonobo/util/errors.py b/bonobo/util/errors.py index 468ba2f..bd1f51f 100644 --- a/bonobo/util/errors.py +++ b/bonobo/util/errors.py @@ -21,8 +21,9 @@ def print_error(exc, trace, context=None, prefix=''): print( Style.BRIGHT, Fore.RED, - '\U0001F4A3 {}{}{}'.format((prefix + ': ') if prefix else '', type(exc).__name__, - ' in {!r}'.format(context) if context else ''), + '\U0001F4A3 {}{}{}'.format( + (prefix + ': ') if prefix else '', type(exc).__name__, ' in {!r}'.format(context) if context else '' + ), Style.RESET_ALL, sep='', file=sys.stderr, diff --git a/setup.py b/setup.py index 83bd632..8b02f65 100644 --- a/setup.py +++ b/setup.py @@ -45,39 +45,37 @@ setup( description='Bonobo', license='Apache License, Version 2.0', install_requires=[ - 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', - 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0' + 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0' ], version=version, long_description=read('README.rst'), classifiers=read('classifiers.txt', tolines), packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, - data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/ext/jupyter/static/extension.js', - 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map' - ])], + data_files=[ + ( + 'share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js', + 'bonobo/ext/jupyter/static/index.js.map' + ] + ) + ], extras_require={ 'dev': [ - 'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', - 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx', + 'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx', 'sphinx_rtd_theme', 'yapf' ], 'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5'] }, entry_points={ 'bonobo.commands': [ - 'init = bonobo.commands.init:register', - 'run = bonobo.commands.run:register', + 'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register' ], 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], - 'edgy.project.features': - ['bonobo = ' - 'bonobo.ext.edgy.project.feature:BonoboFeature'] + 'edgy.project.features': ['bonobo = ' + 'bonobo.ext.edgy.project.feature:BonoboFeature'] }, url='https://www.bonobo-project.org/', - download_url= - 'https://github.com/python-bonobo/bonobo/tarball/{version}'.format( - version=version), ) + download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), +) From bbd258c313fbd219ab6351cc5bd9ca41d08c184b Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Mon, 1 May 2017 15:22:32 +0200 Subject: [PATCH 15/15] 74 characters instead of 80 for examples, as it seems that desktop version display less characters than tablet on rtd theme. --- bonobo/examples/.style.yapf | 2 +- bonobo/examples/datasets/fablabs.py | 11 +++++++---- bonobo/examples/tutorials/tut02_01_read.py | 4 +++- bonobo/examples/tutorials/tut02_02_write.py | 4 +++- bonobo/examples/tutorials/tut02_03_writeasmap.py | 4 +++- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/bonobo/examples/.style.yapf b/bonobo/examples/.style.yapf index 70f7590..ffebd86 100644 --- a/bonobo/examples/.style.yapf +++ b/bonobo/examples/.style.yapf @@ -1,4 +1,4 @@ [style] based_on_style = pep8 -column_limit = 80 +column_limit = 74 dedent_closing_brackets = true diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index ae74c0b..be95fe1 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -61,8 +61,10 @@ def display(row): None, ( ' '.join( filter( - None, - (row.get('postal_code', None), row.get('city', None)) + None, ( + row.get('postal_code', None), + row.get('city', None) + ) ) ), row.get('county', None), row.get('country'), ) @@ -82,8 +84,9 @@ def display(row): format(Fore.BLUE, Style.RESET_ALL, **row) ) print( - ' - {}source{}: {source}'. - format(Fore.BLUE, Style.RESET_ALL, source='datanova/' + API_DATASET) + ' - {}source{}: {source}'.format( + Fore.BLUE, Style.RESET_ALL, source='datanova/' + API_DATASET + ) ) diff --git a/bonobo/examples/tutorials/tut02_01_read.py b/bonobo/examples/tutorials/tut02_01_read.py index 28baa9e..9806d43 100644 --- a/bonobo/examples/tutorials/tut02_01_read.py +++ b/bonobo/examples/tutorials/tut02_01_read.py @@ -6,4 +6,6 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) + bonobo.run( + graph, services={'fs': bonobo.open_examples_fs('datasets')} + ) diff --git a/bonobo/examples/tutorials/tut02_02_write.py b/bonobo/examples/tutorials/tut02_02_write.py index a3c4811..161c58e 100644 --- a/bonobo/examples/tutorials/tut02_02_write.py +++ b/bonobo/examples/tutorials/tut02_02_write.py @@ -12,4 +12,6 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) + bonobo.run( + graph, services={'fs': bonobo.open_examples_fs('datasets')} + ) diff --git a/bonobo/examples/tutorials/tut02_03_writeasmap.py b/bonobo/examples/tutorials/tut02_03_writeasmap.py index bdf3194..4598b56 100644 --- a/bonobo/examples/tutorials/tut02_03_writeasmap.py +++ b/bonobo/examples/tutorials/tut02_03_writeasmap.py @@ -22,4 +22,6 @@ graph = bonobo.Graph( ) if __name__ == '__main__': - bonobo.run(graph, services={'fs': bonobo.open_examples_fs('datasets')}) + bonobo.run( + graph, services={'fs': bonobo.open_examples_fs('datasets')} + )