+
+
+ {% if theme_logo_name|lower == 'true' %}
+
{{ project }}
+ {% endif %}
+
+
{{ theme_description }}
+{% endif %} + +{% if theme_github_user and theme_github_repo %} +{% if theme_github_button|lower == 'true' %} ++ +
+{% endif %} +{% endif %} + +{% if theme_travis_button|lower != 'false' %} +{% if theme_travis_button|lower == 'true' %} + {% set path = theme_github_user + '/' + theme_github_repo %} +{% else %} + {% set path = theme_travis_button %} +{% endif %} +
+
+ 
+
+
+
+ 
+
+
+Consider supporting the authors on Gratipay: + +
+{% endif %} diff --git a/docs/_templates/alabaster/layout.html b/docs/_templates/alabaster/layout.html new file mode 100644 index 0000000..459700f --- /dev/null +++ b/docs/_templates/alabaster/layout.html @@ -0,0 +1,82 @@ +{%- extends "basic/layout.html" %} + +{%- block extrahead %} + {{ super() }} + + {% if theme_touch_icon %} + + {% endif %} + {% if theme_canonical_url %} + + {% endif %} + +{% endblock %} + +{# Disable base theme's top+bottom related navs; we have our own in sidebar #} +{%- block relbar1 %}{% endblock %} +{%- block relbar2 %}{% endblock %} + +{# Nav should appear before content, not after #} +{%- block content %} +{%- if theme_fixed_sidebar|lower == 'true' %} + if theme_github_banner|lower != 'true' else 'https://s3.amazonaws.com/github/ribbons/forkme_right_darkblue_121621.png' }})
+
+ {% endif %}
+
+ {% if theme_analytics_id %}
+
+ {% endif %}
+{%- endblock %}
diff --git a/docs/_templates/alabaster/navigation.html b/docs/_templates/alabaster/navigation.html
new file mode 100644
index 0000000..760dcc9
--- /dev/null
+++ b/docs/_templates/alabaster/navigation.html
@@ -0,0 +1,10 @@
+
+
+{% endif %}
+
+{% if theme_analytics_id %}
+
+{% endif %}
+{%- endblock %}
diff --git a/docs/_templates/index.html b/docs/_templates/index.html
index 33c4116..35698e1 100644
--- a/docs/_templates/index.html
+++ b/docs/_templates/index.html
@@ -1,7 +1,8 @@
-{% extends "layout.html" %}
-{% set title = _('Bonobo — Data processing for humans') %}
-{% block body %}
+{% extends "base.html" %}
+{% set title = _('Bonobo — Data processing for humans') %}
+
+{% block body %}
 }} "Bonobo")
@@ -9,7 +10,7 @@
{% trans %} - Bonobo is an Extract Transform Load framework for the Python (3.5+) language. + Bonobo is an Extract Transform Load (or ETL) framework for the Python (3.5+) language. {% endtrans %}
diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 0000000..f155134 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,7 @@ +{%- extends "base.html" %} + +{%- block content %} +{{ relbar() }} +{{ super() }} +{%- endblock %} + diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 5b8a7da..9210c4f 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -1,10 +1,10 @@ -
Bonobo
- Data processing for human beings. +
+ Data processing for humans.
diff --git a/docs/conf.py b/docs/conf.py index e164f8d..52fb506 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -75,9 +75,9 @@ html_theme = 'alabaster' html_theme_options = { 'github_user': 'python-bonobo', 'github_repo': 'bonobo', - 'github_button': True, - 'show_powered_by': False, - 'show_related': True, + 'github_button': 'true', + 'show_powered_by': 'false', + 'show_related': 'true', } html_sidebars = { diff --git a/docs/genindex.rst b/docs/genindex.rst new file mode 100644 index 0000000..de2955c --- /dev/null +++ b/docs/genindex.rst @@ -0,0 +1,3 @@ +Full Index +========== + diff --git a/docs/guide/graphs.rst b/docs/guide/graphs.rst index 14af705..1c753a1 100644 --- a/docs/guide/graphs.rst +++ b/docs/guide/graphs.rst @@ -1,11 +1,211 @@ Graphs ====== -Writing graphs -:::::::::::::: +Graphs are the glue that ties transformations together. It's the only data-structure bonobo can execute directly. Graphs +must be acyclic, and can contain as much nodes as your system can handle. Although this number can be rather high in +theory, extreme practical cases usually do not exceed hundreds of nodes (and this is already extreme, really). -Debugging graphs + +Definitions +::::::::::: + +Graph + + A directed acyclic graph of transformations, that Bonobo can inspect and execute. + +Node + + A transformation within a graph. The transformations are stateless, and have no idea whether or not they are + included in a graph, multiple graph, or not at all. + + +Creating a graph :::::::::::::::: +Graphs should be instances of :class:`bonobo.Graph`. The :func:`bonobo.Graph.add_chain` method can take as many +positional parameters as you want. + +.. code-block:: python + + import bonobo + + graph = bonobo.Graph() + graph.add_chain(a, b, c) + +Resulting graph: + +.. graphviz:: + + digraph { + rankdir = LR; + stylesheet = "../_static/graphs.css"; + + BEGIN [shape="point"]; + BEGIN -> "a" -> "b" -> "c"; + } + +Non-linear graphs +::::::::::::::::: + +Divergences / forks +------------------- + +To create two or more divergent data streams ("fork"), you should specify `_input` kwarg to `add_chain`. + +.. code-block:: python + + import bonobo + + graph = bonobo.Graph() + graph.add_chain(a, b, c) + graph.add_chain(f, g, _input=b) + + +Resulting graph: + +.. graphviz:: + + digraph { + rankdir = LR; + stylesheet = "../_static/graphs.css"; + + BEGIN [shape="point"]; + BEGIN -> "a" -> "b" -> "c"; + "b" -> "f" -> "g"; + } + +.. note:: Both branch will receive the same data, at the same time. + +Convergences / merges +--------------------- + +To merge two data streams ("merge"), you can use the `_output` kwarg to `add_chain`, or use named nodes (see below). + + +.. code-block:: python + + import bonobo + + graph = bonobo.Graph() + + # Here we mark _input to None, so normalize won't get the "begin" impulsion. + graph.add_chain(normalize, store, _input=None) + + # Add two different chains + graph.add_chain(a, b, _output=normalize) + graph.add_chain(f, g, _output=normalize) + + +Resulting graph: + +.. graphviz:: + + digraph { + rankdir = LR; + stylesheet = "../_static/graphs.css"; + + BEGIN [shape="point"]; + BEGIN -> "a" -> "b" -> "normalize"; + + BEGIN2 [shape="point"]; + BEGIN2 -> "f" -> "g" -> "normalize"; + + "normalize" -> "store" + } + +.. note:: + + This is not a "join" or "cartesian product". Any data that comes from `b` or `g` will go through `normalize`, one at + a time. Think of the graph edges as data flow pipes. + + +Named nodes +::::::::::: + +Using above code to create convergences can lead to hard to read code, because you have to define the "target" stream +before the streams that logically goes to the beginning of the transformation graph. To overcome that, one can use +"named" nodes: + + graph.add_chain(x, y, z, _name='zed') + graph.add_chain(f, g, h, _input='zed') + +.. code-block:: python + + import bonobo + + graph = bonobo.Graph() + + # Add two different chains + graph.add_chain(a, b, _output="load") + graph.add_chain(f, g, _output="load") + + # Here we mark _input to None, so normalize won't get the "begin" impulsion. + graph.add_chain(normalize, store, _input=None, _name="load") + + +Resulting graph: + +.. graphviz:: + + digraph { + rankdir = LR; + stylesheet = "../_static/graphs.css"; + + BEGIN [shape="point"]; + BEGIN -> "a" -> "b" -> "normalize (load)"; + + BEGIN2 [shape="point"]; + BEGIN2 -> "f" -> "g" -> "normalize (load)"; + + "normalize (load)" -> "store" + } + + +Inspecting graphs +::::::::::::::::: + +Bonobo is bundled with an "inspector", that can use graphviz to let you visualize your graphs. + +Read `How to inspect and visualize your graph