Refactoring API, writing docs.

Makefile

@@ -1,4 +1,4 @@
-# Generated by Medikit 0.4.5 on 2017-12-13.
+# Generated by Medikit 0.4.5 on 2017-12-27.
 # All changes will be overriden.
 
 PACKAGE ?= bonobo

Projectfile

@@ -7,7 +7,7 @@ python = require('python')
 sphinx = require('sphinx')
 yapf = require('yapf')
 
-# python.set_versions('3.5', '3.6', '3.7')  --> not yet implemented
+# python.set_versions('3.5', '3.6', '3.7')  --> not yet implemented in medikit
 
 python.setup(
     name='bonobo',
@@ -43,34 +43,30 @@ python.setup(
 )
 
 python.add_requirements(
-    'fs >=2.0,<2.1',
+    'fs ~=2.0',
     'graphviz >=0.8,<0.9',
-    'jinja2 >=2.9,<3',
+    'jinja2 ~=2.9',
-    'mondrian >=0.6,<0.7',
+    'mondrian ~=0.6',
-    'packaging >=16,<17',
+    'packaging ~=16.0',
-    'psutil >=5.4,<6',
+    'psutil ~=5.4',
-    'python-slugify >=1.2,<1.3',
+    'python-slugify ~=1.2.0',
-    'requests >=2,<3',
+    'requests ~=2.0',
     'stevedore ~=1.27',
-    'whistle >=1.0,<1.1',
+    'whistle ~=1.0',
     dev=[
         'pytest-sugar >=0.9,<0.10',
-        'pytest-timeout >=1,<2',
+        'pytest-timeout ~=1.0',
     ],
     docker=[
-        'bonobo-docker >=0.5.0',
+        'bonobo-docker',
     ],
     jupyter=[
-        'ipywidgets >=6.0.0,<7',
+        'ipywidgets ~=6.0',
-        'jupyter >=1.0,<1.1',
+        'jupyter ~=1.0',
     ],
     sqlalchemy=[
-        'bonobo-sqlalchemy >=0.5.1',
+        'bonobo-sqlalchemy',
     ],
 )
 
-# Following requirements are not enforced, because some dependencies enforce them so we don't want to break
-# the packaging in case it changes in dep.
-python.add_requirements('colorama >=0.3')
-
 # vim: ft=python:

bonobo/_api.py

@@ -3,45 +3,15 @@ from bonobo.nodes import __all__ as _all_nodes
 from bonobo.nodes import *
 from bonobo.structs import Graph
 from bonobo.util import get_name
+from bonobo.util.api import ApiHelper
 from bonobo.util.environ import parse_args, get_argument_parser
 
 __all__ = []
 
-
+api = ApiHelper(__all__)
-def register_api(x, __all__=__all__):
-    """Register a function as being part of Bonobo's API, then returns the original function."""
-    __all__.append(get_name(x))
-    return x
 
 
-def register_graph_api(x, __all__=__all__):
+@api.register_graph
-    """
-    Register a function as being part of Bonobo's API, after checking that its signature contains the right parameters
-    to work correctly, then returns the original function.
-    """
-    from inspect import signature
-    parameters = list(signature(x).parameters)
-    required_parameters = {'plugins', 'services', 'strategy'}
-    assert parameters[0] == 'graph', 'First parameter of a graph api function must be "graph".'
-    assert required_parameters.intersection(
-        parameters
-    ) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join(
-        sorted(required_parameters)
-    )
-
-    return register_api(x, __all__=__all__)
-
-
-def register_api_group(*args, check=None):
-    check = set(check) if check else None
-    for attr in args:
-        register_api(attr)
-        if check:
-            check.remove(get_name(attr))
-    assert not (check and len(check))
-
-
-@register_graph_api
 def run(graph, *, plugins=None, services=None, strategy=None):
     """
     Main entry point of bonobo. It takes a graph and creates all the necessary plumbing around to execute it.
@@ -104,7 +74,7 @@ def _inspect_as_graph(graph):
 _inspect_formats = {'graph': _inspect_as_graph}
 
 
-@register_graph_api
+@api.register_graph
 def inspect(graph, *, plugins=None, services=None, strategy=None, format):
     if not format in _inspect_formats:
         raise NotImplementedError(
@@ -116,14 +86,14 @@ def inspect(graph, *, plugins=None, services=None, strategy=None, format):
 
 
 # data structures
-register_api_group(Graph)
+api.register_group(Graph)
 
 # execution strategies
-register_api(create_strategy)
+api.register_group(create_strategy)
 
 
 # Shortcut to filesystem2's open_fs, that we make available there for convenience.
-@register_api
+@api.register
 def open_fs(fs_url=None, *args, **kwargs):
     """
     Wraps :func:`fs.open_fs` function with a few candies.
@@ -148,7 +118,7 @@ def open_fs(fs_url=None, *args, **kwargs):
 
 
 # standard transformations
-register_api_group(
+api.register_group(
     CsvReader,
     CsvWriter,
     FileReader,
@@ -189,16 +159,16 @@ def _is_jupyter_notebook():
         return False
 
 
-@register_api
+@api.register
 def get_examples_path(*pathsegments):
     import os
     import pathlib
     return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments))
 
 
-@register_api
+@api.register
 def open_examples_fs(*pathsegments):
     return open_fs(get_examples_path(*pathsegments))
 
 
-register_api_group(get_argument_parser, parse_args)
+api.register_group(get_argument_parser, parse_args)

bonobo/util/api.py

@@ -0,0 +1,35 @@
+from bonobo.util import get_name
+
+
+class ApiHelper:
+    def __init__(self, __all__):
+        self.__all__ = __all__
+
+    def register(self, x, graph=False):
+        """Register a function as being part of an API, then returns the original function."""
+
+        if graph:
+            # This function must comply to the "graph" API interface, meaning it can bahave like bonobo.run.
+            from inspect import signature
+            parameters = list(signature(x).parameters)
+            required_parameters = {'plugins', 'services', 'strategy'}
+            assert parameters[0] == 'graph', 'First parameter of a graph api function must be "graph".'
+            assert required_parameters.intersection(
+                parameters
+            ) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join(
+                sorted(required_parameters)
+            )
+
+        self.__all__.append(get_name(x))
+        return x
+
+    def register_graph(self, x):
+        return self.register(x, graph=True)
+
+    def register_group(self, *args, check=None):
+        check = set(check) if check else None
+        for attr in args:
+            self.register(attr)
+            if check:
+                check.remove(get_name(attr))
+        assert not (check and len(check))

docs/tutorial/2-jobs.rst

@@ -31,9 +31,6 @@ We'll create a job to do the following
 * Display it (in the next step, we'll learn about writing the result to a file.
 
 
-
-
-
 Moving forward
 ::::::::::::::
 

docs/tutorial/4-services.rst

@@ -1,9 +1,6 @@
 Part 4: Services and Configurables
 ==================================
 
-.. note::
-
-    This section lacks completeness, sorry for that (but you can still read it!).
 
 In the last section, we used a few new tools.
 

docs/tutorial/5-packaging.rst

@@ -1,7 +1,20 @@
 Part 5: Projects and Packaging
 ==============================
 
-Until then, we worked with one file managing a job. But real life is about set of jobs working together within a project.
+Until then, we worked with one file managing a job.
+
+Real life often involves more complicated setups, with relations and imports between different files.
+
+This section will describe the options available to move this file into a package, either a new one or something
+that already exists in your own project.
+
+Data processing is something a wide variety of tools may want to include, and thus |bonobo| does not enforce any
+kind of project structure, as the targert structure will be dicated by the hosting project. For example, a `pipelines`
+sub-package would perfectly fit a django or flask project, or even a regular package, but it's up to you to chose the
+structure of your project.
+
+about using |bonobo| in a pyt
+ is about set of jobs working together within a project.
 
 Let's see how to move from the current status to a package.
 

docs/tutorial/python.rst

@@ -1,11 +0,0 @@
-Just enough Python for Bonobo
-=============================
-
-.. todo::
-
-    This is a work in progress and it is not yet available. Please come back later or even better, help us write this
-    guide!
-
-    This guide is intended to help programmers or enthusiasts to grasp the python basics necessary to use Bonobo. It
-    should definately not be considered as a general python introduction, neither a deep dive into details.
-

requirements-docker.txt

@@ -20,8 +20,8 @@ python-slugify==1.2.4
 pytz==2017.3
 requests==2.18.4
 six==1.11.0
-stevedore==1.27.1
+stevedore==1.28.0
 unidecode==0.4.21
 urllib3==1.22
-websocket-client==0.44.0
+websocket-client==0.45.0
 whistle==1.0.0

requirements-jupyter.txt

@@ -1,5 +1,6 @@
 -e .[jupyter]
 appnope==0.1.0
+attrs==17.3.0
 bleach==2.1.2
 decorator==4.1.2
 entrypoints==0.2.3
@@ -8,10 +9,10 @@ ipykernel==4.7.0
 ipython-genutils==0.2.0
 ipython==6.2.1
 ipywidgets==6.0.1
-jedi==0.11.0
+jedi==0.11.1
 jinja2==2.10
 jsonschema==2.6.0
-jupyter-client==5.1.0
+jupyter-client==5.2.0
 jupyter-console==5.2.0
 jupyter-core==4.4.0
 jupyter==1.0.0
@@ -21,12 +22,15 @@ nbconvert==5.3.1
 nbformat==4.4.0
 notebook==5.2.2
 pandocfilters==1.4.2
-parso==0.1.0
+parso==0.1.1
 pexpect==4.3.1
 pickleshare==0.7.4
+pluggy==0.6.0
 prompt-toolkit==1.0.15
 ptyprocess==0.5.2
+py==1.5.2
 pygments==2.2.0
+pytest==3.3.1
 python-dateutil==2.6.1
 pyzmq==17.0.0b3
 qtconsole==4.3.1

requirements-sqlalchemy.txt

@@ -19,7 +19,7 @@ pytz==2017.3
 requests==2.18.4
 six==1.11.0
 sqlalchemy==1.1.15
-stevedore==1.27.1
+stevedore==1.28.0
 unidecode==0.4.21
 urllib3==1.22
 whistle==1.0.0

setup.py

@@ -59,18 +59,17 @@ setup(
     packages=find_packages(exclude=['ez_setup', 'example', 'test']),
     include_package_data=True,
     install_requires=[
-        'colorama (>= 0.3)', 'fs (>= 2.0, < 2.1)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (>= 2.9, < 3)',
+        'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (~= 2.9)', 'mondrian (~= 0.6)', 'packaging (~= 16.0)',
-        'mondrian (>= 0.6, < 0.7)', 'packaging (>= 16, < 17)', 'psutil (>= 5.4, < 6)', 'python-slugify (>= 1.2, < 1.3)',
+        'psutil (~= 5.4)', 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)', 'stevedore (~= 1.27)', 'whistle (~= 1.0)'
-        'requests (>= 2, < 3)', 'stevedore (~= 1.27)', 'whistle (>= 1.0, < 1.1)'
     ],
     extras_require={
         'dev': [
             'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)',
-            'pytest-sugar (>= 0.9, < 0.10)', 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)', 'yapf'
+            'pytest-sugar (>= 0.9, < 0.10)', 'pytest-timeout (~= 1.0)', 'sphinx (>= 1.6, < 2.0)', 'yapf'
         ],
-        'docker': ['bonobo-docker (>= 0.5.0)'],
+        'docker': ['bonobo-docker'],
-        'jupyter': ['ipywidgets (>= 6.0.0, < 7)', 'jupyter (>= 1.0, < 1.1)'],
+        'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'],
-        'sqlalchemy': ['bonobo-sqlalchemy (>= 0.5.1)']
+        'sqlalchemy': ['bonobo-sqlalchemy']
     },
     entry_points={
         'bonobo.commands': [