From 46a8fd192ebf327fe89a6a27b0dc1c0d229919d1 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 27 Dec 2017 11:32:47 +0100 Subject: [PATCH] Refactoring API, writing docs. --- Makefile | 2 +- Projectfile | 32 ++++++++++----------- bonobo/_api.py | 52 ++++++++--------------------------- bonobo/util/api.py | 35 +++++++++++++++++++++++ docs/tutorial/2-jobs.rst | 3 -- docs/tutorial/4-services.rst | 3 -- docs/tutorial/5-packaging.rst | 15 +++++++++- docs/tutorial/python.rst | 11 -------- requirements-docker.txt | 4 +-- requirements-jupyter.txt | 10 +++++-- requirements-sqlalchemy.txt | 2 +- setup.py | 13 ++++----- 12 files changed, 91 insertions(+), 91 deletions(-) create mode 100644 bonobo/util/api.py delete mode 100644 docs/tutorial/python.rst diff --git a/Makefile b/Makefile index 29e4d63..bc29a5b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Generated by Medikit 0.4.5 on 2017-12-13. +# Generated by Medikit 0.4.5 on 2017-12-27. # All changes will be overriden. PACKAGE ?= bonobo diff --git a/Projectfile b/Projectfile index 6016415..5c880db 100644 --- a/Projectfile +++ b/Projectfile @@ -7,7 +7,7 @@ python = require('python') sphinx = require('sphinx') yapf = require('yapf') -# python.set_versions('3.5', '3.6', '3.7') --> not yet implemented +# python.set_versions('3.5', '3.6', '3.7') --> not yet implemented in medikit python.setup( name='bonobo', @@ -43,34 +43,30 @@ python.setup( ) python.add_requirements( - 'fs >=2.0,<2.1', + 'fs ~=2.0', 'graphviz >=0.8,<0.9', - 'jinja2 >=2.9,<3', - 'mondrian >=0.6,<0.7', - 'packaging >=16,<17', - 'psutil >=5.4,<6', - 'python-slugify >=1.2,<1.3', - 'requests >=2,<3', + 'jinja2 ~=2.9', + 'mondrian ~=0.6', + 'packaging ~=16.0', + 'psutil ~=5.4', + 'python-slugify ~=1.2.0', + 'requests ~=2.0', 'stevedore ~=1.27', - 'whistle >=1.0,<1.1', + 'whistle ~=1.0', dev=[ 'pytest-sugar >=0.9,<0.10', - 'pytest-timeout >=1,<2', + 'pytest-timeout ~=1.0', ], docker=[ - 'bonobo-docker >=0.5.0', + 'bonobo-docker', ], jupyter=[ - 'ipywidgets >=6.0.0,<7', - 'jupyter >=1.0,<1.1', + 'ipywidgets ~=6.0', + 'jupyter ~=1.0', ], sqlalchemy=[ - 'bonobo-sqlalchemy >=0.5.1', + 'bonobo-sqlalchemy', ], ) -# Following requirements are not enforced, because some dependencies enforce them so we don't want to break -# the packaging in case it changes in dep. -python.add_requirements('colorama >=0.3') - # vim: ft=python: diff --git a/bonobo/_api.py b/bonobo/_api.py index 58aab3d..df38004 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -3,45 +3,15 @@ from bonobo.nodes import __all__ as _all_nodes from bonobo.nodes import * from bonobo.structs import Graph from bonobo.util import get_name +from bonobo.util.api import ApiHelper from bonobo.util.environ import parse_args, get_argument_parser __all__ = [] - -def register_api(x, __all__=__all__): - """Register a function as being part of Bonobo's API, then returns the original function.""" - __all__.append(get_name(x)) - return x +api = ApiHelper(__all__) -def register_graph_api(x, __all__=__all__): - """ - Register a function as being part of Bonobo's API, after checking that its signature contains the right parameters - to work correctly, then returns the original function. - """ - from inspect import signature - parameters = list(signature(x).parameters) - required_parameters = {'plugins', 'services', 'strategy'} - assert parameters[0] == 'graph', 'First parameter of a graph api function must be "graph".' - assert required_parameters.intersection( - parameters - ) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join( - sorted(required_parameters) - ) - - return register_api(x, __all__=__all__) - - -def register_api_group(*args, check=None): - check = set(check) if check else None - for attr in args: - register_api(attr) - if check: - check.remove(get_name(attr)) - assert not (check and len(check)) - - -@register_graph_api +@api.register_graph def run(graph, *, plugins=None, services=None, strategy=None): """ Main entry point of bonobo. It takes a graph and creates all the necessary plumbing around to execute it. @@ -104,7 +74,7 @@ def _inspect_as_graph(graph): _inspect_formats = {'graph': _inspect_as_graph} -@register_graph_api +@api.register_graph def inspect(graph, *, plugins=None, services=None, strategy=None, format): if not format in _inspect_formats: raise NotImplementedError( @@ -116,14 +86,14 @@ def inspect(graph, *, plugins=None, services=None, strategy=None, format): # data structures -register_api_group(Graph) +api.register_group(Graph) # execution strategies -register_api(create_strategy) +api.register_group(create_strategy) # Shortcut to filesystem2's open_fs, that we make available there for convenience. -@register_api +@api.register def open_fs(fs_url=None, *args, **kwargs): """ Wraps :func:`fs.open_fs` function with a few candies. @@ -148,7 +118,7 @@ def open_fs(fs_url=None, *args, **kwargs): # standard transformations -register_api_group( +api.register_group( CsvReader, CsvWriter, FileReader, @@ -189,16 +159,16 @@ def _is_jupyter_notebook(): return False -@register_api +@api.register def get_examples_path(*pathsegments): import os import pathlib return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments)) -@register_api +@api.register def open_examples_fs(*pathsegments): return open_fs(get_examples_path(*pathsegments)) -register_api_group(get_argument_parser, parse_args) +api.register_group(get_argument_parser, parse_args) diff --git a/bonobo/util/api.py b/bonobo/util/api.py new file mode 100644 index 0000000..1acf5c8 --- /dev/null +++ b/bonobo/util/api.py @@ -0,0 +1,35 @@ +from bonobo.util import get_name + + +class ApiHelper: + def __init__(self, __all__): + self.__all__ = __all__ + + def register(self, x, graph=False): + """Register a function as being part of an API, then returns the original function.""" + + if graph: + # This function must comply to the "graph" API interface, meaning it can bahave like bonobo.run. + from inspect import signature + parameters = list(signature(x).parameters) + required_parameters = {'plugins', 'services', 'strategy'} + assert parameters[0] == 'graph', 'First parameter of a graph api function must be "graph".' + assert required_parameters.intersection( + parameters + ) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join( + sorted(required_parameters) + ) + + self.__all__.append(get_name(x)) + return x + + def register_graph(self, x): + return self.register(x, graph=True) + + def register_group(self, *args, check=None): + check = set(check) if check else None + for attr in args: + self.register(attr) + if check: + check.remove(get_name(attr)) + assert not (check and len(check)) diff --git a/docs/tutorial/2-jobs.rst b/docs/tutorial/2-jobs.rst index dd7e183..d2bbfe5 100644 --- a/docs/tutorial/2-jobs.rst +++ b/docs/tutorial/2-jobs.rst @@ -31,9 +31,6 @@ We'll create a job to do the following * Display it (in the next step, we'll learn about writing the result to a file. - - - Moving forward :::::::::::::: diff --git a/docs/tutorial/4-services.rst b/docs/tutorial/4-services.rst index f2ada53..097d0f6 100644 --- a/docs/tutorial/4-services.rst +++ b/docs/tutorial/4-services.rst @@ -1,9 +1,6 @@ Part 4: Services and Configurables ================================== -.. note:: - - This section lacks completeness, sorry for that (but you can still read it!). In the last section, we used a few new tools. diff --git a/docs/tutorial/5-packaging.rst b/docs/tutorial/5-packaging.rst index 7362311..d0e29d5 100644 --- a/docs/tutorial/5-packaging.rst +++ b/docs/tutorial/5-packaging.rst @@ -1,7 +1,20 @@ Part 5: Projects and Packaging ============================== -Until then, we worked with one file managing a job. But real life is about set of jobs working together within a project. +Until then, we worked with one file managing a job. + +Real life often involves more complicated setups, with relations and imports between different files. + +This section will describe the options available to move this file into a package, either a new one or something +that already exists in your own project. + +Data processing is something a wide variety of tools may want to include, and thus |bonobo| does not enforce any +kind of project structure, as the targert structure will be dicated by the hosting project. For example, a `pipelines` +sub-package would perfectly fit a django or flask project, or even a regular package, but it's up to you to chose the +structure of your project. + +about using |bonobo| in a pyt + is about set of jobs working together within a project. Let's see how to move from the current status to a package. diff --git a/docs/tutorial/python.rst b/docs/tutorial/python.rst deleted file mode 100644 index d045031..0000000 --- a/docs/tutorial/python.rst +++ /dev/null @@ -1,11 +0,0 @@ -Just enough Python for Bonobo -============================= - -.. todo:: - - This is a work in progress and it is not yet available. Please come back later or even better, help us write this - guide! - - This guide is intended to help programmers or enthusiasts to grasp the python basics necessary to use Bonobo. It - should definately not be considered as a general python introduction, neither a deep dive into details. - diff --git a/requirements-docker.txt b/requirements-docker.txt index 687cfc5..6fbec83 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -20,8 +20,8 @@ python-slugify==1.2.4 pytz==2017.3 requests==2.18.4 six==1.11.0 -stevedore==1.27.1 +stevedore==1.28.0 unidecode==0.4.21 urllib3==1.22 -websocket-client==0.44.0 +websocket-client==0.45.0 whistle==1.0.0 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 2689ba3..2f5b6fe 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -1,5 +1,6 @@ -e .[jupyter] appnope==0.1.0 +attrs==17.3.0 bleach==2.1.2 decorator==4.1.2 entrypoints==0.2.3 @@ -8,10 +9,10 @@ ipykernel==4.7.0 ipython-genutils==0.2.0 ipython==6.2.1 ipywidgets==6.0.1 -jedi==0.11.0 +jedi==0.11.1 jinja2==2.10 jsonschema==2.6.0 -jupyter-client==5.1.0 +jupyter-client==5.2.0 jupyter-console==5.2.0 jupyter-core==4.4.0 jupyter==1.0.0 @@ -21,12 +22,15 @@ nbconvert==5.3.1 nbformat==4.4.0 notebook==5.2.2 pandocfilters==1.4.2 -parso==0.1.0 +parso==0.1.1 pexpect==4.3.1 pickleshare==0.7.4 +pluggy==0.6.0 prompt-toolkit==1.0.15 ptyprocess==0.5.2 +py==1.5.2 pygments==2.2.0 +pytest==3.3.1 python-dateutil==2.6.1 pyzmq==17.0.0b3 qtconsole==4.3.1 diff --git a/requirements-sqlalchemy.txt b/requirements-sqlalchemy.txt index e89781a..d46e7aa 100644 --- a/requirements-sqlalchemy.txt +++ b/requirements-sqlalchemy.txt @@ -19,7 +19,7 @@ pytz==2017.3 requests==2.18.4 six==1.11.0 sqlalchemy==1.1.15 -stevedore==1.27.1 +stevedore==1.28.0 unidecode==0.4.21 urllib3==1.22 whistle==1.0.0 diff --git a/setup.py b/setup.py index 741883e..5d89c9a 100644 --- a/setup.py +++ b/setup.py @@ -59,18 +59,17 @@ setup( packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, install_requires=[ - 'colorama (>= 0.3)', 'fs (>= 2.0, < 2.1)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (>= 2.9, < 3)', - 'mondrian (>= 0.6, < 0.7)', 'packaging (>= 16, < 17)', 'psutil (>= 5.4, < 6)', 'python-slugify (>= 1.2, < 1.3)', - 'requests (>= 2, < 3)', 'stevedore (~= 1.27)', 'whistle (>= 1.0, < 1.1)' + 'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (~= 2.9)', 'mondrian (~= 0.6)', 'packaging (~= 16.0)', + 'psutil (~= 5.4)', 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)', 'stevedore (~= 1.27)', 'whistle (~= 1.0)' ], extras_require={ 'dev': [ 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)', - 'pytest-sugar (>= 0.9, < 0.10)', 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)', 'yapf' + 'pytest-sugar (>= 0.9, < 0.10)', 'pytest-timeout (~= 1.0)', 'sphinx (>= 1.6, < 2.0)', 'yapf' ], - 'docker': ['bonobo-docker (>= 0.5.0)'], - 'jupyter': ['ipywidgets (>= 6.0.0, < 7)', 'jupyter (>= 1.0, < 1.1)'], - 'sqlalchemy': ['bonobo-sqlalchemy (>= 0.5.1)'] + 'docker': ['bonobo-docker'], + 'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'], + 'sqlalchemy': ['bonobo-sqlalchemy'] }, entry_points={ 'bonobo.commands': [